diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,36033 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.244167962674961, + "eval_steps": 500, + "global_step": 2000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0006220839813374805, + "grad_norm": 19.98119354248047, + "learning_rate": 4.9975e-05, + "log_odds_chosen": -0.9538012742996216, + "log_odds_ratio": -1.4484777450561523, + "logits/chosen": -0.3500179052352905, + "logits/rejected": -0.4888765215873718, + "logps/chosen": -2.206789255142212, + "logps/rejected": -1.3497388362884521, + "loss": 7.9648, + "nll_loss": 7.819947242736816, + "rewards/accuracies": 0.125, + "rewards/chosen": -0.2206789255142212, + "rewards/margins": -0.08570503443479538, + "rewards/rejected": -0.13497388362884521, + "step": 1 + }, + { + "epoch": 0.001244167962674961, + "grad_norm": 13.379537582397461, + "learning_rate": 4.995e-05, + "log_odds_chosen": -0.23575380444526672, + "log_odds_ratio": -0.9271624088287354, + "logits/chosen": -0.40814173221588135, + "logits/rejected": -0.43905961513519287, + "logps/chosen": -1.7428513765335083, + "logps/rejected": -1.5162339210510254, + "loss": 6.0609, + "nll_loss": 5.96816873550415, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1742851585149765, + "rewards/margins": -0.02266174554824829, + "rewards/rejected": -0.15162339806556702, + "step": 2 + }, + { + "epoch": 0.0018662519440124418, + "grad_norm": 18.385046005249023, + "learning_rate": 4.992500000000001e-05, + "log_odds_chosen": -1.152876853942871, + "log_odds_ratio": -1.6186071634292603, + "logits/chosen": -0.18745173513889313, + "logits/rejected": -0.269758939743042, + "logps/chosen": -2.3206677436828613, + "logps/rejected": -1.3112003803253174, + "loss": 6.6148, + "nll_loss": 6.452923774719238, + "rewards/accuracies": 0.0, + "rewards/chosen": -0.2320667803287506, + "rewards/margins": -0.10094673931598663, + "rewards/rejected": -0.13112004101276398, + "step": 3 + }, + { + "epoch": 0.002488335925349922, + "grad_norm": 9.557136535644531, + "learning_rate": 4.99e-05, + "log_odds_chosen": -1.4854508638381958, + "log_odds_ratio": -2.0142252445220947, + "logits/chosen": -0.028350114822387695, + "logits/rejected": -0.2540076971054077, + "logps/chosen": -2.8356778621673584, + "logps/rejected": -1.4353848695755005, + "loss": 5.7077, + "nll_loss": 5.506240367889404, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.28356778621673584, + "rewards/margins": -0.14002928137779236, + "rewards/rejected": -0.1435384899377823, + "step": 4 + }, + { + "epoch": 0.003110419906687403, + "grad_norm": 1.9710413217544556, + "learning_rate": 4.9875000000000006e-05, + "log_odds_chosen": -0.064725860953331, + "log_odds_ratio": -0.7509320974349976, + "logits/chosen": -0.2338867336511612, + "logits/rejected": -0.143581822514534, + "logps/chosen": -1.6930336952209473, + "logps/rejected": -1.6086996793746948, + "loss": 3.6175, + "nll_loss": 3.5423970222473145, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.16930335760116577, + "rewards/margins": -0.008433390408754349, + "rewards/rejected": -0.16086997091770172, + "step": 5 + }, + { + "epoch": 0.0037325038880248835, + "grad_norm": 4.176668643951416, + "learning_rate": 4.9850000000000006e-05, + "log_odds_chosen": -1.303108811378479, + "log_odds_ratio": -1.8442931175231934, + "logits/chosen": 0.178592711687088, + "logits/rejected": -0.07974191009998322, + "logps/chosen": -2.8217928409576416, + "logps/rejected": -1.5813729763031006, + "loss": 5.1449, + "nll_loss": 4.96045446395874, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.2821792662143707, + "rewards/margins": -0.12404197454452515, + "rewards/rejected": -0.15813729166984558, + "step": 6 + }, + { + "epoch": 0.004354587869362364, + "grad_norm": 3.5636956691741943, + "learning_rate": 4.9825000000000005e-05, + "log_odds_chosen": -1.0302621126174927, + "log_odds_ratio": -1.533990502357483, + "logits/chosen": 0.1397887021303177, + "logits/rejected": -0.12103983759880066, + "logps/chosen": -2.5798606872558594, + "logps/rejected": -1.6471679210662842, + "loss": 5.2595, + "nll_loss": 5.1061482429504395, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.2579860985279083, + "rewards/margins": -0.09326928108930588, + "rewards/rejected": -0.16471679508686066, + "step": 7 + }, + { + "epoch": 0.004976671850699844, + "grad_norm": 1.424960732460022, + "learning_rate": 4.9800000000000004e-05, + "log_odds_chosen": -0.057843249291181564, + "log_odds_ratio": -0.7315274477005005, + "logits/chosen": -0.21581152081489563, + "logits/rejected": -0.2254290133714676, + "logps/chosen": -1.4299900531768799, + "logps/rejected": -1.376258134841919, + "loss": 3.5319, + "nll_loss": 3.458702802658081, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.14299900829792023, + "rewards/margins": -0.00537317618727684, + "rewards/rejected": -0.1376258283853531, + "step": 8 + }, + { + "epoch": 0.005598755832037325, + "grad_norm": 2.506531000137329, + "learning_rate": 4.9775000000000004e-05, + "log_odds_chosen": -1.3545101881027222, + "log_odds_ratio": -1.7200111150741577, + "logits/chosen": 0.09172062575817108, + "logits/rejected": -0.2259369194507599, + "logps/chosen": -2.604728937149048, + "logps/rejected": -1.421167016029358, + "loss": 5.1963, + "nll_loss": 5.0243120193481445, + "rewards/accuracies": 0.125, + "rewards/chosen": -0.2604728937149048, + "rewards/margins": -0.11835619807243347, + "rewards/rejected": -0.1421166956424713, + "step": 9 + }, + { + "epoch": 0.006220839813374806, + "grad_norm": 2.135577917098999, + "learning_rate": 4.975e-05, + "log_odds_chosen": -0.5466750264167786, + "log_odds_ratio": -1.1493580341339111, + "logits/chosen": -0.009926818311214447, + "logits/rejected": -0.12821561098098755, + "logps/chosen": -2.1568522453308105, + "logps/rejected": -1.6440058946609497, + "loss": 4.7799, + "nll_loss": 4.664962291717529, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.21568521857261658, + "rewards/margins": -0.05128464847803116, + "rewards/rejected": -0.16440057754516602, + "step": 10 + }, + { + "epoch": 0.006842923794712286, + "grad_norm": 2.0873756408691406, + "learning_rate": 4.9725e-05, + "log_odds_chosen": -0.22385576367378235, + "log_odds_ratio": -0.8634535074234009, + "logits/chosen": -0.07170040160417557, + "logits/rejected": -0.28907641768455505, + "logps/chosen": -1.4515858888626099, + "logps/rejected": -1.2991254329681396, + "loss": 4.4469, + "nll_loss": 4.360559463500977, + "rewards/accuracies": 0.125, + "rewards/chosen": -0.145158588886261, + "rewards/margins": -0.015246042050421238, + "rewards/rejected": -0.12991255521774292, + "step": 11 + }, + { + "epoch": 0.007465007776049767, + "grad_norm": 1.0876375436782837, + "learning_rate": 4.97e-05, + "log_odds_chosen": -0.17661024630069733, + "log_odds_ratio": -0.8128032088279724, + "logits/chosen": -0.3114607334136963, + "logits/rejected": -0.4201509356498718, + "logps/chosen": -1.3225839138031006, + "logps/rejected": -1.1948829889297485, + "loss": 3.5573, + "nll_loss": 3.476047992706299, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.13225840032100677, + "rewards/margins": -0.012770086526870728, + "rewards/rejected": -0.11948829889297485, + "step": 12 + }, + { + "epoch": 0.008087091757387248, + "grad_norm": 1.08712637424469, + "learning_rate": 4.967500000000001e-05, + "log_odds_chosen": -0.38383999466896057, + "log_odds_ratio": -0.93343186378479, + "logits/chosen": -0.17842403054237366, + "logits/rejected": -0.4043556749820709, + "logps/chosen": -1.5285923480987549, + "logps/rejected": -1.2196223735809326, + "loss": 4.0367, + "nll_loss": 3.943343162536621, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.15285924077033997, + "rewards/margins": -0.030896998941898346, + "rewards/rejected": -0.12196224182844162, + "step": 13 + }, + { + "epoch": 0.008709175738724729, + "grad_norm": 1.449515461921692, + "learning_rate": 4.965e-05, + "log_odds_chosen": -0.6121888160705566, + "log_odds_ratio": -1.3804981708526611, + "logits/chosen": -0.06745365262031555, + "logits/rejected": -0.2206701636314392, + "logps/chosen": -2.0546789169311523, + "logps/rejected": -1.398402214050293, + "loss": 4.3677, + "nll_loss": 4.229663372039795, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.20546789467334747, + "rewards/margins": -0.06562765687704086, + "rewards/rejected": -0.13984021544456482, + "step": 14 + }, + { + "epoch": 0.00933125972006221, + "grad_norm": 0.8825786709785461, + "learning_rate": 4.962500000000001e-05, + "log_odds_chosen": -0.13495028018951416, + "log_odds_ratio": -0.8009682893753052, + "logits/chosen": -0.20614610612392426, + "logits/rejected": -0.45154061913490295, + "logps/chosen": -1.570542812347412, + "logps/rejected": -1.476640224456787, + "loss": 3.9433, + "nll_loss": 3.863245964050293, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.15705429017543793, + "rewards/margins": -0.009390261955559254, + "rewards/rejected": -0.14766402542591095, + "step": 15 + }, + { + "epoch": 0.009953343701399688, + "grad_norm": 0.7910265922546387, + "learning_rate": 4.96e-05, + "log_odds_chosen": -0.36577627062797546, + "log_odds_ratio": -0.9311540126800537, + "logits/chosen": -0.22354616224765778, + "logits/rejected": -0.29362282156944275, + "logps/chosen": -1.65301513671875, + "logps/rejected": -1.361577033996582, + "loss": 3.606, + "nll_loss": 3.5129306316375732, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.16530153155326843, + "rewards/margins": -0.0291438065469265, + "rewards/rejected": -0.13615772128105164, + "step": 16 + }, + { + "epoch": 0.010575427682737169, + "grad_norm": 0.9194669127464294, + "learning_rate": 4.9575000000000006e-05, + "log_odds_chosen": -0.17109160125255585, + "log_odds_ratio": -0.8144399523735046, + "logits/chosen": -0.0705355554819107, + "logits/rejected": -0.2670852541923523, + "logps/chosen": -1.7514052391052246, + "logps/rejected": -1.6155729293823242, + "loss": 4.2639, + "nll_loss": 4.182419776916504, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.17514052987098694, + "rewards/margins": -0.013583235442638397, + "rewards/rejected": -0.16155728697776794, + "step": 17 + }, + { + "epoch": 0.01119751166407465, + "grad_norm": 0.7311789393424988, + "learning_rate": 4.9550000000000005e-05, + "log_odds_chosen": -0.04682855308055878, + "log_odds_ratio": -0.7275374531745911, + "logits/chosen": -0.08018438518047333, + "logits/rejected": -0.20024976134300232, + "logps/chosen": -1.3416996002197266, + "logps/rejected": -1.3203401565551758, + "loss": 3.7728, + "nll_loss": 3.7000904083251953, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13416996598243713, + "rewards/margins": -0.00213593989610672, + "rewards/rejected": -0.132034033536911, + "step": 18 + }, + { + "epoch": 0.01181959564541213, + "grad_norm": 0.7463477253913879, + "learning_rate": 4.9525000000000004e-05, + "log_odds_chosen": -0.6258882284164429, + "log_odds_ratio": -1.0997464656829834, + "logits/chosen": -0.12337462604045868, + "logits/rejected": -0.34859755635261536, + "logps/chosen": -1.7963664531707764, + "logps/rejected": -1.2740449905395508, + "loss": 3.714, + "nll_loss": 3.603994369506836, + "rewards/accuracies": 0.125, + "rewards/chosen": -0.1796366274356842, + "rewards/margins": -0.05223213881254196, + "rewards/rejected": -0.12740451097488403, + "step": 19 + }, + { + "epoch": 0.012441679626749611, + "grad_norm": 0.6245782375335693, + "learning_rate": 4.9500000000000004e-05, + "log_odds_chosen": -0.2411532998085022, + "log_odds_ratio": -0.8302733898162842, + "logits/chosen": -0.049967993050813675, + "logits/rejected": -0.2759324908256531, + "logps/chosen": -1.4726518392562866, + "logps/rejected": -1.3111158609390259, + "loss": 3.707, + "nll_loss": 3.6239380836486816, + "rewards/accuracies": 0.125, + "rewards/chosen": -0.14726518094539642, + "rewards/margins": -0.01615358516573906, + "rewards/rejected": -0.13111159205436707, + "step": 20 + }, + { + "epoch": 0.013063763608087092, + "grad_norm": 0.7757321000099182, + "learning_rate": 4.9475e-05, + "log_odds_chosen": 0.0013154447078704834, + "log_odds_ratio": -0.7458984851837158, + "logits/chosen": -0.016477234661579132, + "logits/rejected": -0.1882762759923935, + "logps/chosen": -1.4737496376037598, + "logps/rejected": -1.4141770601272583, + "loss": 3.9635, + "nll_loss": 3.8889150619506836, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1473749727010727, + "rewards/margins": -0.005957265384495258, + "rewards/rejected": -0.1414177119731903, + "step": 21 + }, + { + "epoch": 0.013685847589424573, + "grad_norm": 0.5803574323654175, + "learning_rate": 4.945e-05, + "log_odds_chosen": -0.024330340325832367, + "log_odds_ratio": -0.7314656972885132, + "logits/chosen": -0.0876406729221344, + "logits/rejected": -0.25811609625816345, + "logps/chosen": -1.4882334470748901, + "logps/rejected": -1.4596867561340332, + "loss": 3.7711, + "nll_loss": 3.6979050636291504, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1488233506679535, + "rewards/margins": -0.002854675054550171, + "rewards/rejected": -0.14596867561340332, + "step": 22 + }, + { + "epoch": 0.014307931570762053, + "grad_norm": 0.5226195454597473, + "learning_rate": 4.9425e-05, + "log_odds_chosen": -0.0417926162481308, + "log_odds_ratio": -0.7332164645195007, + "logits/chosen": -0.12541602551937103, + "logits/rejected": -0.2495168298482895, + "logps/chosen": -1.5487079620361328, + "logps/rejected": -1.5171284675598145, + "loss": 3.6381, + "nll_loss": 3.5647568702697754, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.15487080812454224, + "rewards/margins": -0.0031579509377479553, + "rewards/rejected": -0.15171284973621368, + "step": 23 + }, + { + "epoch": 0.014930015552099534, + "grad_norm": 0.49786579608917236, + "learning_rate": 4.94e-05, + "log_odds_chosen": -0.24082081019878387, + "log_odds_ratio": -0.8401607275009155, + "logits/chosen": -0.15435853600502014, + "logits/rejected": -0.3182606101036072, + "logps/chosen": -1.4637329578399658, + "logps/rejected": -1.2792115211486816, + "loss": 3.6258, + "nll_loss": 3.5417871475219727, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.14637331664562225, + "rewards/margins": -0.01845216006040573, + "rewards/rejected": -0.12792114913463593, + "step": 24 + }, + { + "epoch": 0.015552099533437015, + "grad_norm": 0.7141457200050354, + "learning_rate": 4.937500000000001e-05, + "log_odds_chosen": 0.11563286930322647, + "log_odds_ratio": -0.6847242116928101, + "logits/chosen": -0.05548834800720215, + "logits/rejected": -0.25137871503829956, + "logps/chosen": -1.3125858306884766, + "logps/rejected": -1.3853249549865723, + "loss": 3.9615, + "nll_loss": 3.8930606842041016, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13125859200954437, + "rewards/margins": 0.007273899391293526, + "rewards/rejected": -0.13853248953819275, + "step": 25 + }, + { + "epoch": 0.016174183514774496, + "grad_norm": 0.48132890462875366, + "learning_rate": 4.935e-05, + "log_odds_chosen": -0.3378981947898865, + "log_odds_ratio": -1.048970341682434, + "logits/chosen": -0.27093982696533203, + "logits/rejected": -0.29352226853370667, + "logps/chosen": -1.8172366619110107, + "logps/rejected": -1.4850423336029053, + "loss": 3.3072, + "nll_loss": 3.2022593021392822, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.18172365427017212, + "rewards/margins": -0.03321942687034607, + "rewards/rejected": -0.14850424230098724, + "step": 26 + }, + { + "epoch": 0.016796267496111975, + "grad_norm": 0.465388685464859, + "learning_rate": 4.9325000000000006e-05, + "log_odds_chosen": 0.26982247829437256, + "log_odds_ratio": -0.5840736031532288, + "logits/chosen": -0.13838329911231995, + "logits/rejected": -0.14500632882118225, + "logps/chosen": -1.2994801998138428, + "logps/rejected": -1.5056928396224976, + "loss": 3.2789, + "nll_loss": 3.220458984375, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12994801998138428, + "rewards/margins": 0.020621273666620255, + "rewards/rejected": -0.15056928992271423, + "step": 27 + }, + { + "epoch": 0.017418351477449457, + "grad_norm": 0.4851706922054291, + "learning_rate": 4.93e-05, + "log_odds_chosen": 0.0167313814163208, + "log_odds_ratio": -0.7801838517189026, + "logits/chosen": -0.12882889807224274, + "logits/rejected": -0.27155885100364685, + "logps/chosen": -1.5362133979797363, + "logps/rejected": -1.5006215572357178, + "loss": 3.7467, + "nll_loss": 3.6686904430389404, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1536213457584381, + "rewards/margins": -0.0035591907799243927, + "rewards/rejected": -0.15006214380264282, + "step": 28 + }, + { + "epoch": 0.018040435458786936, + "grad_norm": 0.4745447039604187, + "learning_rate": 4.9275000000000005e-05, + "log_odds_chosen": -0.4235059916973114, + "log_odds_ratio": -0.944256067276001, + "logits/chosen": -0.15738001465797424, + "logits/rejected": -0.28892505168914795, + "logps/chosen": -1.4211246967315674, + "logps/rejected": -1.1277490854263306, + "loss": 3.5365, + "nll_loss": 3.4420742988586426, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.14211246371269226, + "rewards/margins": -0.029337557032704353, + "rewards/rejected": -0.11277490854263306, + "step": 29 + }, + { + "epoch": 0.01866251944012442, + "grad_norm": 0.507114052772522, + "learning_rate": 4.9250000000000004e-05, + "log_odds_chosen": -0.3637525737285614, + "log_odds_ratio": -0.9062818884849548, + "logits/chosen": -0.2181130200624466, + "logits/rejected": -0.37557047605514526, + "logps/chosen": -1.2170381546020508, + "logps/rejected": -0.9694937467575073, + "loss": 2.9864, + "nll_loss": 2.8957455158233643, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.12170383334159851, + "rewards/margins": -0.024754449725151062, + "rewards/rejected": -0.09694937616586685, + "step": 30 + }, + { + "epoch": 0.019284603421461897, + "grad_norm": 0.40544337034225464, + "learning_rate": 4.9225000000000004e-05, + "log_odds_chosen": -0.1934043914079666, + "log_odds_ratio": -0.8023362159729004, + "logits/chosen": -0.21577346324920654, + "logits/rejected": -0.29102498292922974, + "logps/chosen": -1.170575737953186, + "logps/rejected": -1.035292625427246, + "loss": 3.3617, + "nll_loss": 3.281477928161621, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.11705759167671204, + "rewards/margins": -0.013528317213058472, + "rewards/rejected": -0.10352927446365356, + "step": 31 + }, + { + "epoch": 0.019906687402799376, + "grad_norm": 0.5559415221214294, + "learning_rate": 4.92e-05, + "log_odds_chosen": -0.3727276027202606, + "log_odds_ratio": -0.9165624976158142, + "logits/chosen": -0.17616789042949677, + "logits/rejected": -0.41072139143943787, + "logps/chosen": -1.4517570734024048, + "logps/rejected": -1.1750273704528809, + "loss": 3.7894, + "nll_loss": 3.6977787017822266, + "rewards/accuracies": 0.125, + "rewards/chosen": -0.14517571032047272, + "rewards/margins": -0.027672961354255676, + "rewards/rejected": -0.11750274896621704, + "step": 32 + }, + { + "epoch": 0.02052877138413686, + "grad_norm": 0.49967700242996216, + "learning_rate": 4.9175e-05, + "log_odds_chosen": -0.37599754333496094, + "log_odds_ratio": -1.035901427268982, + "logits/chosen": -0.19352321326732635, + "logits/rejected": -0.37395039200782776, + "logps/chosen": -1.4342128038406372, + "logps/rejected": -1.0744060277938843, + "loss": 3.4439, + "nll_loss": 3.3403570652008057, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.14342127740383148, + "rewards/margins": -0.03598066791892052, + "rewards/rejected": -0.10744060575962067, + "step": 33 + }, + { + "epoch": 0.021150855365474338, + "grad_norm": 0.40192580223083496, + "learning_rate": 4.915e-05, + "log_odds_chosen": 0.06716141849756241, + "log_odds_ratio": -0.68401700258255, + "logits/chosen": -0.2316037267446518, + "logits/rejected": -0.34950870275497437, + "logps/chosen": -1.3879081010818481, + "logps/rejected": -1.4275767803192139, + "loss": 3.3671, + "nll_loss": 3.2987279891967773, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1387908160686493, + "rewards/margins": 0.003966865129768848, + "rewards/rejected": -0.14275768399238586, + "step": 34 + }, + { + "epoch": 0.02177293934681182, + "grad_norm": 0.6308751106262207, + "learning_rate": 4.9125e-05, + "log_odds_chosen": 0.10806269943714142, + "log_odds_ratio": -0.6696207523345947, + "logits/chosen": -0.3126291334629059, + "logits/rejected": -0.3402310907840729, + "logps/chosen": -1.4202438592910767, + "logps/rejected": -1.5030903816223145, + "loss": 3.0162, + "nll_loss": 2.949201822280884, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14202438294887543, + "rewards/margins": 0.008284655399620533, + "rewards/rejected": -0.15030904114246368, + "step": 35 + }, + { + "epoch": 0.0223950233281493, + "grad_norm": 0.47672948241233826, + "learning_rate": 4.91e-05, + "log_odds_chosen": 0.6212818026542664, + "log_odds_ratio": -0.48152947425842285, + "logits/chosen": -0.2364703118801117, + "logits/rejected": -0.3322206139564514, + "logps/chosen": -0.9610673189163208, + "logps/rejected": -1.338320016860962, + "loss": 3.2005, + "nll_loss": 3.152357339859009, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09610673785209656, + "rewards/margins": 0.03772527724504471, + "rewards/rejected": -0.13383200764656067, + "step": 36 + }, + { + "epoch": 0.023017107309486782, + "grad_norm": 0.44931307435035706, + "learning_rate": 4.907500000000001e-05, + "log_odds_chosen": -0.29742100834846497, + "log_odds_ratio": -0.952938437461853, + "logits/chosen": -0.16727347671985626, + "logits/rejected": -0.3058803081512451, + "logps/chosen": -1.608022689819336, + "logps/rejected": -1.4014194011688232, + "loss": 3.4746, + "nll_loss": 3.3793392181396484, + "rewards/accuracies": 0.125, + "rewards/chosen": -0.16080228984355927, + "rewards/margins": -0.020660335198044777, + "rewards/rejected": -0.14014194905757904, + "step": 37 + }, + { + "epoch": 0.02363919129082426, + "grad_norm": 0.42830324172973633, + "learning_rate": 4.905e-05, + "log_odds_chosen": -0.4679420292377472, + "log_odds_ratio": -1.0487958192825317, + "logits/chosen": -0.1773906946182251, + "logits/rejected": -0.206269308924675, + "logps/chosen": -1.745017647743225, + "logps/rejected": -1.3269473314285278, + "loss": 3.3234, + "nll_loss": 3.2185542583465576, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.17450176179409027, + "rewards/margins": -0.04180702194571495, + "rewards/rejected": -0.13269473612308502, + "step": 38 + }, + { + "epoch": 0.024261275272161743, + "grad_norm": 0.4050178825855255, + "learning_rate": 4.9025000000000006e-05, + "log_odds_chosen": 0.10232503712177277, + "log_odds_ratio": -0.6785426139831543, + "logits/chosen": -0.18703876435756683, + "logits/rejected": -0.3412337899208069, + "logps/chosen": -1.0300055742263794, + "logps/rejected": -1.0844731330871582, + "loss": 3.1217, + "nll_loss": 3.0538713932037354, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.10300055146217346, + "rewards/margins": 0.0054467711597681046, + "rewards/rejected": -0.10844732820987701, + "step": 39 + }, + { + "epoch": 0.024883359253499222, + "grad_norm": 0.4841116666793823, + "learning_rate": 4.9e-05, + "log_odds_chosen": 0.02147568017244339, + "log_odds_ratio": -0.6917948722839355, + "logits/chosen": -0.10562430322170258, + "logits/rejected": -0.30269837379455566, + "logps/chosen": -1.4125561714172363, + "logps/rejected": -1.440596580505371, + "loss": 3.449, + "nll_loss": 3.3798322677612305, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.14125561714172363, + "rewards/margins": 0.0028040409088134766, + "rewards/rejected": -0.1440596580505371, + "step": 40 + }, + { + "epoch": 0.0255054432348367, + "grad_norm": 0.4915813207626343, + "learning_rate": 4.8975000000000005e-05, + "log_odds_chosen": -0.11176743358373642, + "log_odds_ratio": -0.7612941265106201, + "logits/chosen": -0.08788780868053436, + "logits/rejected": -0.3535291254520416, + "logps/chosen": -1.260170578956604, + "logps/rejected": -1.17002534866333, + "loss": 3.5663, + "nll_loss": 3.490180015563965, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.12601706385612488, + "rewards/margins": -0.009014511480927467, + "rewards/rejected": -0.11700254678726196, + "step": 41 + }, + { + "epoch": 0.026127527216174184, + "grad_norm": 0.4824346601963043, + "learning_rate": 4.8950000000000004e-05, + "log_odds_chosen": -0.18203996121883392, + "log_odds_ratio": -0.80300372838974, + "logits/chosen": -0.13452517986297607, + "logits/rejected": -0.14837010204792023, + "logps/chosen": -1.6312255859375, + "logps/rejected": -1.490962266921997, + "loss": 3.3183, + "nll_loss": 3.2380266189575195, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.16312256455421448, + "rewards/margins": -0.014026330783963203, + "rewards/rejected": -0.14909623563289642, + "step": 42 + }, + { + "epoch": 0.026749611197511663, + "grad_norm": 0.40107211470603943, + "learning_rate": 4.8925e-05, + "log_odds_chosen": 0.1854020357131958, + "log_odds_ratio": -0.6462830305099487, + "logits/chosen": -0.15604250133037567, + "logits/rejected": -0.18608345091342926, + "logps/chosen": -1.3835902214050293, + "logps/rejected": -1.5467376708984375, + "loss": 3.2676, + "nll_loss": 3.203000545501709, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13835902512073517, + "rewards/margins": 0.016314741224050522, + "rewards/rejected": -0.154673770070076, + "step": 43 + }, + { + "epoch": 0.027371695178849145, + "grad_norm": 0.516572117805481, + "learning_rate": 4.89e-05, + "log_odds_chosen": -0.15254731476306915, + "log_odds_ratio": -0.9058659672737122, + "logits/chosen": -0.07723084092140198, + "logits/rejected": -0.3161468803882599, + "logps/chosen": -1.5304224491119385, + "logps/rejected": -1.3918745517730713, + "loss": 3.5929, + "nll_loss": 3.502323865890503, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1530422568321228, + "rewards/margins": -0.013854792341589928, + "rewards/rejected": -0.13918745517730713, + "step": 44 + }, + { + "epoch": 0.027993779160186624, + "grad_norm": 0.4352788031101227, + "learning_rate": 4.8875e-05, + "log_odds_chosen": 0.12119769304990768, + "log_odds_ratio": -0.6822906136512756, + "logits/chosen": -0.08103634417057037, + "logits/rejected": -0.23941874504089355, + "logps/chosen": -1.2429063320159912, + "logps/rejected": -1.3138731718063354, + "loss": 3.4242, + "nll_loss": 3.3560192584991455, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.12429064512252808, + "rewards/margins": 0.007096678018569946, + "rewards/rejected": -0.13138732314109802, + "step": 45 + }, + { + "epoch": 0.028615863141524107, + "grad_norm": 0.45666632056236267, + "learning_rate": 4.885e-05, + "log_odds_chosen": 0.277486115694046, + "log_odds_ratio": -0.6156935095787048, + "logits/chosen": -0.16658517718315125, + "logits/rejected": -0.2931288778781891, + "logps/chosen": -1.0638997554779053, + "logps/rejected": -1.2447154521942139, + "loss": 2.8714, + "nll_loss": 2.8098669052124023, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10638997703790665, + "rewards/margins": 0.018081560730934143, + "rewards/rejected": -0.12447153031826019, + "step": 46 + }, + { + "epoch": 0.029237947122861586, + "grad_norm": 0.4158986806869507, + "learning_rate": 4.8825e-05, + "log_odds_chosen": -0.03135260194540024, + "log_odds_ratio": -0.7471761107444763, + "logits/chosen": 0.007953077554702759, + "logits/rejected": -0.1722358912229538, + "logps/chosen": -1.3186936378479004, + "logps/rejected": -1.2820833921432495, + "loss": 3.5286, + "nll_loss": 3.453866958618164, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.131869375705719, + "rewards/margins": -0.0036610299721360207, + "rewards/rejected": -0.12820833921432495, + "step": 47 + }, + { + "epoch": 0.029860031104199068, + "grad_norm": 0.5239469408988953, + "learning_rate": 4.88e-05, + "log_odds_chosen": -0.14184284210205078, + "log_odds_ratio": -0.7821545600891113, + "logits/chosen": -0.00846000388264656, + "logits/rejected": -0.24844269454479218, + "logps/chosen": -1.4144936800003052, + "logps/rejected": -1.2858200073242188, + "loss": 3.157, + "nll_loss": 3.07881236076355, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.14144936203956604, + "rewards/margins": -0.01286737434566021, + "rewards/rejected": -0.12858200073242188, + "step": 48 + }, + { + "epoch": 0.030482115085536547, + "grad_norm": 0.42745500802993774, + "learning_rate": 4.8775000000000007e-05, + "log_odds_chosen": -0.016907602548599243, + "log_odds_ratio": -0.7335056066513062, + "logits/chosen": 0.027836868539452553, + "logits/rejected": -0.16093496978282928, + "logps/chosen": -1.3329463005065918, + "logps/rejected": -1.3012332916259766, + "loss": 3.7355, + "nll_loss": 3.6621429920196533, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.13329462707042694, + "rewards/margins": -0.00317130284383893, + "rewards/rejected": -0.1301233321428299, + "step": 49 + }, + { + "epoch": 0.03110419906687403, + "grad_norm": 0.5647901296615601, + "learning_rate": 4.875e-05, + "log_odds_chosen": 0.05693642795085907, + "log_odds_ratio": -0.7423352003097534, + "logits/chosen": 0.07700448483228683, + "logits/rejected": -0.21037913858890533, + "logps/chosen": -1.3298025131225586, + "logps/rejected": -1.3227578401565552, + "loss": 3.6339, + "nll_loss": 3.55966854095459, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13298025727272034, + "rewards/margins": -0.0007044710218906403, + "rewards/rejected": -0.13227578997612, + "step": 50 + }, + { + "epoch": 0.031726283048211505, + "grad_norm": 0.5185202360153198, + "learning_rate": 4.8725000000000005e-05, + "log_odds_chosen": 0.271283358335495, + "log_odds_ratio": -0.5842689275741577, + "logits/chosen": -0.0015755696222186089, + "logits/rejected": -0.17162489891052246, + "logps/chosen": -1.039259672164917, + "logps/rejected": -1.1926697492599487, + "loss": 3.2335, + "nll_loss": 3.1750574111938477, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10392597317695618, + "rewards/margins": 0.015341001562774181, + "rewards/rejected": -0.11926697194576263, + "step": 51 + }, + { + "epoch": 0.03234836702954899, + "grad_norm": 0.3914552927017212, + "learning_rate": 4.87e-05, + "log_odds_chosen": -0.09510594606399536, + "log_odds_ratio": -0.7722680568695068, + "logits/chosen": -0.11492118239402771, + "logits/rejected": -0.1295318901538849, + "logps/chosen": -1.2744344472885132, + "logps/rejected": -1.1613863706588745, + "loss": 2.7643, + "nll_loss": 2.6871042251586914, + "rewards/accuracies": 0.125, + "rewards/chosen": -0.12744343280792236, + "rewards/margins": -0.011304810643196106, + "rewards/rejected": -0.11613863706588745, + "step": 52 + }, + { + "epoch": 0.03297045101088647, + "grad_norm": 0.42848122119903564, + "learning_rate": 4.8675000000000004e-05, + "log_odds_chosen": 0.13229113817214966, + "log_odds_ratio": -0.659803569316864, + "logits/chosen": -0.206315815448761, + "logits/rejected": -0.24223698675632477, + "logps/chosen": -1.1478495597839355, + "logps/rejected": -1.2279183864593506, + "loss": 2.5846, + "nll_loss": 2.5186636447906494, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11478495597839355, + "rewards/margins": 0.008006898686289787, + "rewards/rejected": -0.12279185652732849, + "step": 53 + }, + { + "epoch": 0.03359253499222395, + "grad_norm": 0.9718621373176575, + "learning_rate": 4.8650000000000003e-05, + "log_odds_chosen": 0.4774380326271057, + "log_odds_ratio": -0.5322307348251343, + "logits/chosen": 0.07056058198213577, + "logits/rejected": -0.18487195670604706, + "logps/chosen": -1.300857663154602, + "logps/rejected": -1.635850429534912, + "loss": 3.7447, + "nll_loss": 3.6914896965026855, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.130085751414299, + "rewards/margins": 0.033499278128147125, + "rewards/rejected": -0.16358503699302673, + "step": 54 + }, + { + "epoch": 0.03421461897356143, + "grad_norm": 0.341932475566864, + "learning_rate": 4.8625e-05, + "log_odds_chosen": 0.007643640041351318, + "log_odds_ratio": -0.7581207752227783, + "logits/chosen": -0.09210430085659027, + "logits/rejected": -0.0826970785856247, + "logps/chosen": -1.4047452211380005, + "logps/rejected": -1.3780653476715088, + "loss": 3.1252, + "nll_loss": 3.0493462085723877, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.14047454297542572, + "rewards/margins": -0.002668004482984543, + "rewards/rejected": -0.13780653476715088, + "step": 55 + }, + { + "epoch": 0.034836702954898914, + "grad_norm": 0.45145970582962036, + "learning_rate": 4.86e-05, + "log_odds_chosen": -0.24970951676368713, + "log_odds_ratio": -0.8379786014556885, + "logits/chosen": 0.09427239745855331, + "logits/rejected": 0.03530137240886688, + "logps/chosen": -1.4495787620544434, + "logps/rejected": -1.284151554107666, + "loss": 2.9932, + "nll_loss": 2.90944242477417, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.14495787024497986, + "rewards/margins": -0.0165427103638649, + "rewards/rejected": -0.12841516733169556, + "step": 56 + }, + { + "epoch": 0.03545878693623639, + "grad_norm": 0.7668087482452393, + "learning_rate": 4.8575e-05, + "log_odds_chosen": -0.44778311252593994, + "log_odds_ratio": -1.0893973112106323, + "logits/chosen": -0.056043416261672974, + "logits/rejected": -0.1815190464258194, + "logps/chosen": -1.829887866973877, + "logps/rejected": -1.4810354709625244, + "loss": 3.1517, + "nll_loss": 3.0427865982055664, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.18298880755901337, + "rewards/margins": -0.03488525003194809, + "rewards/rejected": -0.14810355007648468, + "step": 57 + }, + { + "epoch": 0.03608087091757387, + "grad_norm": 0.49874478578567505, + "learning_rate": 4.855e-05, + "log_odds_chosen": 0.3892138600349426, + "log_odds_ratio": -0.5497685670852661, + "logits/chosen": -0.11005600541830063, + "logits/rejected": -0.2566314935684204, + "logps/chosen": -1.1769773960113525, + "logps/rejected": -1.4603263139724731, + "loss": 3.0813, + "nll_loss": 3.026326894760132, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11769775301218033, + "rewards/margins": 0.028334885835647583, + "rewards/rejected": -0.14603263139724731, + "step": 58 + }, + { + "epoch": 0.03670295489891135, + "grad_norm": 0.3849422335624695, + "learning_rate": 4.8525e-05, + "log_odds_chosen": -0.050915684551000595, + "log_odds_ratio": -0.7625954151153564, + "logits/chosen": -0.010294832289218903, + "logits/rejected": -0.12012767791748047, + "logps/chosen": -1.289703607559204, + "logps/rejected": -1.2587721347808838, + "loss": 3.4305, + "nll_loss": 3.354198932647705, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.12897036969661713, + "rewards/margins": -0.0030931569635868073, + "rewards/rejected": -0.12587721645832062, + "step": 59 + }, + { + "epoch": 0.03732503888024884, + "grad_norm": 0.3733402192592621, + "learning_rate": 4.85e-05, + "log_odds_chosen": 0.9943278431892395, + "log_odds_ratio": -0.46127861738204956, + "logits/chosen": -0.16634932160377502, + "logits/rejected": -0.24419239163398743, + "logps/chosen": -0.8558648824691772, + "logps/rejected": -1.3538252115249634, + "loss": 3.0007, + "nll_loss": 2.954594612121582, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08558649569749832, + "rewards/margins": 0.049796026200056076, + "rewards/rejected": -0.1353825181722641, + "step": 60 + }, + { + "epoch": 0.037947122861586316, + "grad_norm": 0.47226205468177795, + "learning_rate": 4.8475000000000006e-05, + "log_odds_chosen": 0.36546510457992554, + "log_odds_ratio": -0.5973978042602539, + "logits/chosen": 0.09770157933235168, + "logits/rejected": -0.09729360044002533, + "logps/chosen": -1.3515472412109375, + "logps/rejected": -1.606210470199585, + "loss": 3.6102, + "nll_loss": 3.5504493713378906, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13515472412109375, + "rewards/margins": 0.025466332212090492, + "rewards/rejected": -0.1606210470199585, + "step": 61 + }, + { + "epoch": 0.038569206842923795, + "grad_norm": 0.38134610652923584, + "learning_rate": 4.845e-05, + "log_odds_chosen": 0.3819832503795624, + "log_odds_ratio": -0.6043448448181152, + "logits/chosen": 0.1279037743806839, + "logits/rejected": 0.05211859941482544, + "logps/chosen": -1.3050546646118164, + "logps/rejected": -1.5339863300323486, + "loss": 3.3049, + "nll_loss": 3.244431972503662, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13050545752048492, + "rewards/margins": 0.02289317362010479, + "rewards/rejected": -0.15339863300323486, + "step": 62 + }, + { + "epoch": 0.039191290824261274, + "grad_norm": 0.5376470685005188, + "learning_rate": 4.8425000000000005e-05, + "log_odds_chosen": 0.21296370029449463, + "log_odds_ratio": -0.6956855058670044, + "logits/chosen": 0.005937471985816956, + "logits/rejected": -0.13821996748447418, + "logps/chosen": -1.2709373235702515, + "logps/rejected": -1.3110651969909668, + "loss": 3.4905, + "nll_loss": 3.4209353923797607, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.12709373235702515, + "rewards/margins": 0.004012777470052242, + "rewards/rejected": -0.13110651075839996, + "step": 63 + }, + { + "epoch": 0.03981337480559875, + "grad_norm": 0.45539331436157227, + "learning_rate": 4.8400000000000004e-05, + "log_odds_chosen": -0.19902189075946808, + "log_odds_ratio": -0.8536033630371094, + "logits/chosen": 0.059451624751091, + "logits/rejected": -0.11644947528839111, + "logps/chosen": -1.4663325548171997, + "logps/rejected": -1.3308027982711792, + "loss": 3.6228, + "nll_loss": 3.5374748706817627, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.14663325250148773, + "rewards/margins": -0.013552968390285969, + "rewards/rejected": -0.13308028876781464, + "step": 64 + }, + { + "epoch": 0.04043545878693624, + "grad_norm": 0.3535555899143219, + "learning_rate": 4.8375000000000004e-05, + "log_odds_chosen": -0.03207793086767197, + "log_odds_ratio": -0.724827766418457, + "logits/chosen": 0.035917554050683975, + "logits/rejected": -0.06365230679512024, + "logps/chosen": -1.3977922201156616, + "logps/rejected": -1.3612167835235596, + "loss": 3.3147, + "nll_loss": 3.2421815395355225, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1397792100906372, + "rewards/margins": -0.0036575384438037872, + "rewards/rejected": -0.13612167537212372, + "step": 65 + }, + { + "epoch": 0.04105754276827372, + "grad_norm": 0.4546717405319214, + "learning_rate": 4.835e-05, + "log_odds_chosen": -0.03722512722015381, + "log_odds_ratio": -0.7913417220115662, + "logits/chosen": 0.09111925959587097, + "logits/rejected": -0.1404401957988739, + "logps/chosen": -1.3331866264343262, + "logps/rejected": -1.2620258331298828, + "loss": 3.2911, + "nll_loss": 3.2119736671447754, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.13331866264343262, + "rewards/margins": -0.007116083987057209, + "rewards/rejected": -0.12620258331298828, + "step": 66 + }, + { + "epoch": 0.0416796267496112, + "grad_norm": 0.30916666984558105, + "learning_rate": 4.8325e-05, + "log_odds_chosen": -0.3806450664997101, + "log_odds_ratio": -0.9495774507522583, + "logits/chosen": 0.22228950262069702, + "logits/rejected": 0.011185593903064728, + "logps/chosen": -1.36622953414917, + "logps/rejected": -1.1000258922576904, + "loss": 3.9335, + "nll_loss": 3.8385517597198486, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.13662296533584595, + "rewards/margins": -0.026620371267199516, + "rewards/rejected": -0.11000259220600128, + "step": 67 + }, + { + "epoch": 0.042301710730948676, + "grad_norm": 0.5843622088432312, + "learning_rate": 4.83e-05, + "log_odds_chosen": -0.04287657141685486, + "log_odds_ratio": -0.7503262758255005, + "logits/chosen": 0.12020996958017349, + "logits/rejected": -0.06287462264299393, + "logps/chosen": -1.2391448020935059, + "logps/rejected": -1.2382404804229736, + "loss": 2.7952, + "nll_loss": 2.72021484375, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.12391449511051178, + "rewards/margins": -9.043654426932335e-05, + "rewards/rejected": -0.12382405996322632, + "step": 68 + }, + { + "epoch": 0.04292379471228616, + "grad_norm": 0.40152105689048767, + "learning_rate": 4.8275e-05, + "log_odds_chosen": 0.48192113637924194, + "log_odds_ratio": -0.5793935060501099, + "logits/chosen": 0.024148011580109596, + "logits/rejected": -0.028608618304133415, + "logps/chosen": -1.2637081146240234, + "logps/rejected": -1.5951621532440186, + "loss": 3.0237, + "nll_loss": 2.9657363891601562, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12637081742286682, + "rewards/margins": 0.03314541280269623, + "rewards/rejected": -0.15951621532440186, + "step": 69 + }, + { + "epoch": 0.04354587869362364, + "grad_norm": 0.5005712509155273, + "learning_rate": 4.825e-05, + "log_odds_chosen": 0.16657370328903198, + "log_odds_ratio": -0.6632782220840454, + "logits/chosen": 0.0074752867221832275, + "logits/rejected": -0.15818460285663605, + "logps/chosen": -1.009189486503601, + "logps/rejected": -1.074352502822876, + "loss": 3.0056, + "nll_loss": 2.939265012741089, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1009189561009407, + "rewards/margins": 0.006516308058053255, + "rewards/rejected": -0.10743525624275208, + "step": 70 + }, + { + "epoch": 0.04416796267496112, + "grad_norm": 0.47246062755584717, + "learning_rate": 4.822500000000001e-05, + "log_odds_chosen": -0.008882712572813034, + "log_odds_ratio": -0.799690842628479, + "logits/chosen": 0.1656499207019806, + "logits/rejected": 0.012578403577208519, + "logps/chosen": -1.1688836812973022, + "logps/rejected": -1.1197009086608887, + "loss": 3.2172, + "nll_loss": 3.137253999710083, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1168883666396141, + "rewards/margins": -0.0049182698130607605, + "rewards/rejected": -0.11197009682655334, + "step": 71 + }, + { + "epoch": 0.0447900466562986, + "grad_norm": 0.5179511308670044, + "learning_rate": 4.82e-05, + "log_odds_chosen": -0.01626509428024292, + "log_odds_ratio": -0.7597264051437378, + "logits/chosen": -0.07426048815250397, + "logits/rejected": -0.21345210075378418, + "logps/chosen": -1.305432677268982, + "logps/rejected": -1.3045635223388672, + "loss": 2.7467, + "nll_loss": 2.6707706451416016, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13054326176643372, + "rewards/margins": -8.692499250173569e-05, + "rewards/rejected": -0.1304563581943512, + "step": 72 + }, + { + "epoch": 0.04541213063763608, + "grad_norm": 0.3978026211261749, + "learning_rate": 4.8175000000000005e-05, + "log_odds_chosen": -0.17721597850322723, + "log_odds_ratio": -0.8433041572570801, + "logits/chosen": -0.00016376003623008728, + "logits/rejected": -0.1693696677684784, + "logps/chosen": -1.6973767280578613, + "logps/rejected": -1.5059082508087158, + "loss": 3.4049, + "nll_loss": 3.3205349445343018, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.16973768174648285, + "rewards/margins": -0.019146855920553207, + "rewards/rejected": -0.15059083700180054, + "step": 73 + }, + { + "epoch": 0.046034214618973564, + "grad_norm": 0.44062405824661255, + "learning_rate": 4.815e-05, + "log_odds_chosen": 0.29165250062942505, + "log_odds_ratio": -0.6030845642089844, + "logits/chosen": 0.03590073063969612, + "logits/rejected": -0.05357559770345688, + "logps/chosen": -0.9286098480224609, + "logps/rejected": -1.074052333831787, + "loss": 3.1473, + "nll_loss": 3.0869500637054443, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09286098182201385, + "rewards/margins": 0.014544256031513214, + "rewards/rejected": -0.10740524530410767, + "step": 74 + }, + { + "epoch": 0.04665629860031104, + "grad_norm": 0.3442816436290741, + "learning_rate": 4.8125000000000004e-05, + "log_odds_chosen": 0.2888907194137573, + "log_odds_ratio": -0.6076911687850952, + "logits/chosen": -0.12151262164115906, + "logits/rejected": -0.12172101438045502, + "logps/chosen": -1.0215524435043335, + "logps/rejected": -1.1889455318450928, + "loss": 2.7404, + "nll_loss": 2.6796624660491943, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.10215524584054947, + "rewards/margins": 0.016739321872591972, + "rewards/rejected": -0.118894562125206, + "step": 75 + }, + { + "epoch": 0.04727838258164852, + "grad_norm": 0.3842046558856964, + "learning_rate": 4.8100000000000004e-05, + "log_odds_chosen": -0.25257712602615356, + "log_odds_ratio": -0.8657203912734985, + "logits/chosen": 0.023503951728343964, + "logits/rejected": -0.18105646967887878, + "logps/chosen": -1.4399148225784302, + "logps/rejected": -1.2502027750015259, + "loss": 3.2649, + "nll_loss": 3.178344488143921, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.14399148523807526, + "rewards/margins": -0.01897120475769043, + "rewards/rejected": -0.12502028048038483, + "step": 76 + }, + { + "epoch": 0.047900466562986, + "grad_norm": 0.3534882664680481, + "learning_rate": 4.8075e-05, + "log_odds_chosen": -0.21401476860046387, + "log_odds_ratio": -0.8372653722763062, + "logits/chosen": 0.14382417500019073, + "logits/rejected": -0.052027709782123566, + "logps/chosen": -1.4991471767425537, + "logps/rejected": -1.312038540840149, + "loss": 3.7097, + "nll_loss": 3.625964403152466, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.1499147266149521, + "rewards/margins": -0.01871085911989212, + "rewards/rejected": -0.13120386004447937, + "step": 77 + }, + { + "epoch": 0.04852255054432349, + "grad_norm": 0.8299946188926697, + "learning_rate": 4.805e-05, + "log_odds_chosen": -0.24282418191432953, + "log_odds_ratio": -0.8619076013565063, + "logits/chosen": -0.18960991501808167, + "logits/rejected": -0.34159693121910095, + "logps/chosen": -1.1464121341705322, + "logps/rejected": -0.992071270942688, + "loss": 2.5854, + "nll_loss": 2.4992449283599854, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.1146412119269371, + "rewards/margins": -0.015434084460139275, + "rewards/rejected": -0.09920713305473328, + "step": 78 + }, + { + "epoch": 0.049144634525660966, + "grad_norm": 0.5896576046943665, + "learning_rate": 4.8025e-05, + "log_odds_chosen": 0.14239203929901123, + "log_odds_ratio": -0.6772485375404358, + "logits/chosen": -0.02156006544828415, + "logits/rejected": -0.23830567300319672, + "logps/chosen": -1.3663321733474731, + "logps/rejected": -1.4884802103042603, + "loss": 3.0472, + "nll_loss": 2.9794466495513916, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13663321733474731, + "rewards/margins": 0.012214799411594868, + "rewards/rejected": -0.1488480269908905, + "step": 79 + }, + { + "epoch": 0.049766718506998445, + "grad_norm": 0.5111446976661682, + "learning_rate": 4.8e-05, + "log_odds_chosen": 0.06895725429058075, + "log_odds_ratio": -0.6709849834442139, + "logits/chosen": 0.03493687883019447, + "logits/rejected": -0.12141556292772293, + "logps/chosen": -1.3161884546279907, + "logps/rejected": -1.3568629026412964, + "loss": 2.9872, + "nll_loss": 2.9200947284698486, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13161885738372803, + "rewards/margins": 0.0040674470365047455, + "rewards/rejected": -0.13568630814552307, + "step": 80 + }, + { + "epoch": 0.050388802488335924, + "grad_norm": 0.5454034805297852, + "learning_rate": 4.7975e-05, + "log_odds_chosen": 0.013033639639616013, + "log_odds_ratio": -0.7662529945373535, + "logits/chosen": 0.09307534992694855, + "logits/rejected": -0.13599663972854614, + "logps/chosen": -1.2162799835205078, + "logps/rejected": -1.1834042072296143, + "loss": 3.0209, + "nll_loss": 2.9443132877349854, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12162800133228302, + "rewards/margins": -0.0032875724136829376, + "rewards/rejected": -0.11834041774272919, + "step": 81 + }, + { + "epoch": 0.0510108864696734, + "grad_norm": 0.4710577726364136, + "learning_rate": 4.795e-05, + "log_odds_chosen": 0.1277192234992981, + "log_odds_ratio": -0.6501528024673462, + "logits/chosen": 0.048733945935964584, + "logits/rejected": -0.10845337063074112, + "logps/chosen": -1.2608442306518555, + "logps/rejected": -1.3538391590118408, + "loss": 2.9852, + "nll_loss": 2.920222759246826, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12608441710472107, + "rewards/margins": 0.0092995036393404, + "rewards/rejected": -0.13538393378257751, + "step": 82 + }, + { + "epoch": 0.05163297045101089, + "grad_norm": 0.4011155068874359, + "learning_rate": 4.7925000000000006e-05, + "log_odds_chosen": 0.20483741164207458, + "log_odds_ratio": -0.6441969275474548, + "logits/chosen": 0.017529848963022232, + "logits/rejected": -0.17869825661182404, + "logps/chosen": -1.1669516563415527, + "logps/rejected": -1.279728889465332, + "loss": 2.7464, + "nll_loss": 2.6819615364074707, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11669516563415527, + "rewards/margins": 0.011277715675532818, + "rewards/rejected": -0.12797288596630096, + "step": 83 + }, + { + "epoch": 0.05225505443234837, + "grad_norm": 0.5642073750495911, + "learning_rate": 4.79e-05, + "log_odds_chosen": 0.16463297605514526, + "log_odds_ratio": -0.6702374219894409, + "logits/chosen": 0.25281310081481934, + "logits/rejected": -0.022312596440315247, + "logps/chosen": -1.236396074295044, + "logps/rejected": -1.3540095090866089, + "loss": 3.6312, + "nll_loss": 3.564155101776123, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.12363961338996887, + "rewards/margins": 0.01176133006811142, + "rewards/rejected": -0.1354009509086609, + "step": 84 + }, + { + "epoch": 0.05287713841368585, + "grad_norm": 0.42271995544433594, + "learning_rate": 4.7875000000000005e-05, + "log_odds_chosen": 0.29148802161216736, + "log_odds_ratio": -0.6428624987602234, + "logits/chosen": 0.16957850754261017, + "logits/rejected": -0.029354337602853775, + "logps/chosen": -1.3640344142913818, + "logps/rejected": -1.4766488075256348, + "loss": 3.5189, + "nll_loss": 3.45458984375, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13640344142913818, + "rewards/margins": 0.01126144826412201, + "rewards/rejected": -0.1476649045944214, + "step": 85 + }, + { + "epoch": 0.053499222395023326, + "grad_norm": 0.6085870265960693, + "learning_rate": 4.785e-05, + "log_odds_chosen": 0.020148158073425293, + "log_odds_ratio": -0.6988396644592285, + "logits/chosen": -0.01158811990171671, + "logits/rejected": -0.15423518419265747, + "logps/chosen": -1.4441418647766113, + "logps/rejected": -1.4638988971710205, + "loss": 2.5733, + "nll_loss": 2.5033936500549316, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.14441418647766113, + "rewards/margins": 0.0019757067784667015, + "rewards/rejected": -0.146389901638031, + "step": 86 + }, + { + "epoch": 0.05412130637636081, + "grad_norm": 0.4892805814743042, + "learning_rate": 4.7825000000000004e-05, + "log_odds_chosen": 0.8066454529762268, + "log_odds_ratio": -0.4088033139705658, + "logits/chosen": 0.12198805809020996, + "logits/rejected": -0.12190286070108414, + "logps/chosen": -0.8199340105056763, + "logps/rejected": -1.2292183637619019, + "loss": 2.7766, + "nll_loss": 2.73576021194458, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08199340105056763, + "rewards/margins": 0.040928442031145096, + "rewards/rejected": -0.12292183935642242, + "step": 87 + }, + { + "epoch": 0.05474339035769829, + "grad_norm": 0.3810538351535797, + "learning_rate": 4.78e-05, + "log_odds_chosen": -0.106544628739357, + "log_odds_ratio": -0.7797843217849731, + "logits/chosen": 0.23251253366470337, + "logits/rejected": 0.10770893096923828, + "logps/chosen": -1.1218090057373047, + "logps/rejected": -1.0476629734039307, + "loss": 3.2626, + "nll_loss": 3.1846585273742676, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11218090355396271, + "rewards/margins": -0.007414607331156731, + "rewards/rejected": -0.10476629436016083, + "step": 88 + }, + { + "epoch": 0.05536547433903577, + "grad_norm": 0.5943082571029663, + "learning_rate": 4.7775e-05, + "log_odds_chosen": 0.18490558862686157, + "log_odds_ratio": -0.6679292917251587, + "logits/chosen": 0.14844851195812225, + "logits/rejected": -0.0743369460105896, + "logps/chosen": -1.268092155456543, + "logps/rejected": -1.3130736351013184, + "loss": 3.2991, + "nll_loss": 3.232351779937744, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12680920958518982, + "rewards/margins": 0.0044981567189097404, + "rewards/rejected": -0.13130736351013184, + "step": 89 + }, + { + "epoch": 0.05598755832037325, + "grad_norm": 0.5314677357673645, + "learning_rate": 4.775e-05, + "log_odds_chosen": 0.2689226269721985, + "log_odds_ratio": -0.6226869225502014, + "logits/chosen": 0.06861215829849243, + "logits/rejected": -0.163130983710289, + "logps/chosen": -1.3787274360656738, + "logps/rejected": -1.6074514389038086, + "loss": 2.6911, + "nll_loss": 2.6288278102874756, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13787275552749634, + "rewards/margins": 0.022872384637594223, + "rewards/rejected": -0.16074511408805847, + "step": 90 + }, + { + "epoch": 0.05660964230171073, + "grad_norm": 0.4301508367061615, + "learning_rate": 4.7725e-05, + "log_odds_chosen": 0.3461243510246277, + "log_odds_ratio": -0.646808385848999, + "logits/chosen": 0.24266819655895233, + "logits/rejected": 0.029673846438527107, + "logps/chosen": -1.2074958086013794, + "logps/rejected": -1.411824345588684, + "loss": 3.5601, + "nll_loss": 3.495389938354492, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1207495778799057, + "rewards/margins": 0.02043285220861435, + "rewards/rejected": -0.14118242263793945, + "step": 91 + }, + { + "epoch": 0.05723172628304821, + "grad_norm": 0.4627557694911957, + "learning_rate": 4.77e-05, + "log_odds_chosen": -0.28584611415863037, + "log_odds_ratio": -0.9114346504211426, + "logits/chosen": 0.21556691825389862, + "logits/rejected": 0.014135261997580528, + "logps/chosen": -1.367776870727539, + "logps/rejected": -1.1591300964355469, + "loss": 3.5889, + "nll_loss": 3.497756004333496, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.13677769899368286, + "rewards/margins": -0.02086469531059265, + "rewards/rejected": -0.11591300368309021, + "step": 92 + }, + { + "epoch": 0.05785381026438569, + "grad_norm": 0.37699708342552185, + "learning_rate": 4.7675e-05, + "log_odds_chosen": 0.3036189377307892, + "log_odds_ratio": -0.5816516876220703, + "logits/chosen": 0.01264739129692316, + "logits/rejected": -0.06274904310703278, + "logps/chosen": -0.9731736183166504, + "logps/rejected": -1.1666101217269897, + "loss": 3.0008, + "nll_loss": 2.942596673965454, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09731736779212952, + "rewards/margins": 0.019343651831150055, + "rewards/rejected": -0.11666101962327957, + "step": 93 + }, + { + "epoch": 0.05847589424572317, + "grad_norm": 0.4950994551181793, + "learning_rate": 4.765e-05, + "log_odds_chosen": -0.003652891144156456, + "log_odds_ratio": -0.7005108594894409, + "logits/chosen": 0.1562429666519165, + "logits/rejected": -0.07231743633747101, + "logps/chosen": -1.5042771100997925, + "logps/rejected": -1.506368637084961, + "loss": 3.2415, + "nll_loss": 3.1714537143707275, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1504277139902115, + "rewards/margins": 0.0002091517671942711, + "rewards/rejected": -0.15063685178756714, + "step": 94 + }, + { + "epoch": 0.05909797822706065, + "grad_norm": 0.5913958549499512, + "learning_rate": 4.7625000000000006e-05, + "log_odds_chosen": -0.5333858132362366, + "log_odds_ratio": -1.058704137802124, + "logits/chosen": 0.2847288250923157, + "logits/rejected": -0.04206930845975876, + "logps/chosen": -2.0648934841156006, + "logps/rejected": -1.5971152782440186, + "loss": 3.879, + "nll_loss": 3.7731385231018066, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.20648935437202454, + "rewards/margins": -0.04677780717611313, + "rewards/rejected": -0.1597115397453308, + "step": 95 + }, + { + "epoch": 0.059720062208398136, + "grad_norm": 0.46030476689338684, + "learning_rate": 4.76e-05, + "log_odds_chosen": -0.05350463092327118, + "log_odds_ratio": -0.7248205542564392, + "logits/chosen": 0.07910023629665375, + "logits/rejected": -0.06675226986408234, + "logps/chosen": -1.3112492561340332, + "logps/rejected": -1.2683042287826538, + "loss": 2.5608, + "nll_loss": 2.4883413314819336, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13112492859363556, + "rewards/margins": -0.0042945025488734245, + "rewards/rejected": -0.12683042883872986, + "step": 96 + }, + { + "epoch": 0.060342146189735615, + "grad_norm": 0.8850117921829224, + "learning_rate": 4.7575000000000004e-05, + "log_odds_chosen": -0.16680529713630676, + "log_odds_ratio": -0.918777585029602, + "logits/chosen": 0.09566190838813782, + "logits/rejected": -0.09266671538352966, + "logps/chosen": -1.5960246324539185, + "logps/rejected": -1.3469147682189941, + "loss": 3.1828, + "nll_loss": 3.0909605026245117, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.15960246324539185, + "rewards/margins": -0.024910978972911835, + "rewards/rejected": -0.13469147682189941, + "step": 97 + }, + { + "epoch": 0.060964230171073094, + "grad_norm": 0.43437162041664124, + "learning_rate": 4.755e-05, + "log_odds_chosen": 0.30574995279312134, + "log_odds_ratio": -0.6281970143318176, + "logits/chosen": 0.08859042823314667, + "logits/rejected": -0.02086031809449196, + "logps/chosen": -1.3456239700317383, + "logps/rejected": -1.5943999290466309, + "loss": 3.2651, + "nll_loss": 3.2022864818573, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1345623880624771, + "rewards/margins": 0.02487758919596672, + "rewards/rejected": -0.15943998098373413, + "step": 98 + }, + { + "epoch": 0.06158631415241057, + "grad_norm": 0.6766498684883118, + "learning_rate": 4.7525e-05, + "log_odds_chosen": -0.11186225712299347, + "log_odds_ratio": -0.7736210823059082, + "logits/chosen": 0.04021189361810684, + "logits/rejected": 0.03945222869515419, + "logps/chosen": -1.3682116270065308, + "logps/rejected": -1.301282286643982, + "loss": 2.6354, + "nll_loss": 2.5580055713653564, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13682116568088531, + "rewards/margins": -0.006692924536764622, + "rewards/rejected": -0.13012823462486267, + "step": 99 + }, + { + "epoch": 0.06220839813374806, + "grad_norm": 0.5133693218231201, + "learning_rate": 4.75e-05, + "log_odds_chosen": -0.17678174376487732, + "log_odds_ratio": -0.8037921786308289, + "logits/chosen": 0.03568641096353531, + "logits/rejected": -0.08334266394376755, + "logps/chosen": -1.4727808237075806, + "logps/rejected": -1.3427841663360596, + "loss": 2.8091, + "nll_loss": 2.7287254333496094, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.1472780704498291, + "rewards/margins": -0.012999659404158592, + "rewards/rejected": -0.13427841663360596, + "step": 100 + }, + { + "epoch": 0.06283048211508553, + "grad_norm": 0.437914103269577, + "learning_rate": 4.7475e-05, + "log_odds_chosen": -0.055506929755210876, + "log_odds_ratio": -0.7757474184036255, + "logits/chosen": 0.13743720948696136, + "logits/rejected": 0.028182677924633026, + "logps/chosen": -1.2761411666870117, + "logps/rejected": -1.158294677734375, + "loss": 3.2391, + "nll_loss": 3.161520481109619, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.1276141107082367, + "rewards/margins": -0.01178464200347662, + "rewards/rejected": -0.1158294752240181, + "step": 101 + }, + { + "epoch": 0.06345256609642301, + "grad_norm": 0.4476625919342041, + "learning_rate": 4.745e-05, + "log_odds_chosen": 0.2342412918806076, + "log_odds_ratio": -0.6413729786872864, + "logits/chosen": 0.168337881565094, + "logits/rejected": -0.14545048773288727, + "logps/chosen": -1.2778122425079346, + "logps/rejected": -1.4140448570251465, + "loss": 3.7524, + "nll_loss": 3.6882784366607666, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1277812272310257, + "rewards/margins": 0.013623261824250221, + "rewards/rejected": -0.14140449464321136, + "step": 102 + }, + { + "epoch": 0.0640746500777605, + "grad_norm": 0.5129625201225281, + "learning_rate": 4.7425e-05, + "log_odds_chosen": 0.08128762245178223, + "log_odds_ratio": -0.6819002628326416, + "logits/chosen": 0.10714941471815109, + "logits/rejected": -0.07025566697120667, + "logps/chosen": -1.333043098449707, + "logps/rejected": -1.3708577156066895, + "loss": 3.0904, + "nll_loss": 3.0221877098083496, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13330431282520294, + "rewards/margins": 0.0037814658135175705, + "rewards/rejected": -0.13708576560020447, + "step": 103 + }, + { + "epoch": 0.06469673405909798, + "grad_norm": 0.5524664521217346, + "learning_rate": 4.74e-05, + "log_odds_chosen": -0.28195393085479736, + "log_odds_ratio": -0.8670526742935181, + "logits/chosen": -0.04569420963525772, + "logits/rejected": -0.1721382588148117, + "logps/chosen": -1.4118688106536865, + "logps/rejected": -1.197587490081787, + "loss": 2.7953, + "nll_loss": 2.7085652351379395, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.1411868780851364, + "rewards/margins": -0.021428123116493225, + "rewards/rejected": -0.11975875496864319, + "step": 104 + }, + { + "epoch": 0.06531881804043546, + "grad_norm": 0.4822681248188019, + "learning_rate": 4.7375e-05, + "log_odds_chosen": 0.38326454162597656, + "log_odds_ratio": -0.5975579619407654, + "logits/chosen": -0.007261446211487055, + "logits/rejected": -0.030812345445156097, + "logps/chosen": -1.179614782333374, + "logps/rejected": -1.4915132522583008, + "loss": 2.6517, + "nll_loss": 2.5919699668884277, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11796149611473083, + "rewards/margins": 0.031189851462841034, + "rewards/rejected": -0.14915132522583008, + "step": 105 + }, + { + "epoch": 0.06594090202177294, + "grad_norm": 0.5283992886543274, + "learning_rate": 4.735e-05, + "log_odds_chosen": 0.45969775319099426, + "log_odds_ratio": -0.5704058408737183, + "logits/chosen": -0.005958788096904755, + "logits/rejected": -0.1403818130493164, + "logps/chosen": -1.1262407302856445, + "logps/rejected": -1.365699291229248, + "loss": 3.3859, + "nll_loss": 3.328857183456421, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11262409389019012, + "rewards/margins": 0.023945845663547516, + "rewards/rejected": -0.13656993210315704, + "step": 106 + }, + { + "epoch": 0.06656298600311042, + "grad_norm": 0.416676789522171, + "learning_rate": 4.7325000000000005e-05, + "log_odds_chosen": 0.1313173770904541, + "log_odds_ratio": -0.6711844205856323, + "logits/chosen": 0.157118558883667, + "logits/rejected": 0.035575054585933685, + "logps/chosen": -1.4387624263763428, + "logps/rejected": -1.5632802248001099, + "loss": 3.2781, + "nll_loss": 3.210995674133301, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.14387625455856323, + "rewards/margins": 0.012451781891286373, + "rewards/rejected": -0.156328022480011, + "step": 107 + }, + { + "epoch": 0.0671850699844479, + "grad_norm": 0.5572922229766846, + "learning_rate": 4.73e-05, + "log_odds_chosen": -0.12381504476070404, + "log_odds_ratio": -0.770376443862915, + "logits/chosen": 0.04753262549638748, + "logits/rejected": -0.051002245396375656, + "logps/chosen": -1.5710325241088867, + "logps/rejected": -1.46817147731781, + "loss": 2.8265, + "nll_loss": 2.749462127685547, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.15710324048995972, + "rewards/margins": -0.010286085307598114, + "rewards/rejected": -0.146817147731781, + "step": 108 + }, + { + "epoch": 0.06780715396578538, + "grad_norm": 0.3513568937778473, + "learning_rate": 4.7275000000000004e-05, + "log_odds_chosen": 0.05175573006272316, + "log_odds_ratio": -0.6735912561416626, + "logits/chosen": 0.08347096294164658, + "logits/rejected": 0.05597352236509323, + "logps/chosen": -1.2763020992279053, + "logps/rejected": -1.3162968158721924, + "loss": 2.8374, + "nll_loss": 2.77008056640625, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12763021886348724, + "rewards/margins": 0.0039994558319449425, + "rewards/rejected": -0.13162967562675476, + "step": 109 + }, + { + "epoch": 0.06842923794712286, + "grad_norm": 0.48008182644844055, + "learning_rate": 4.7249999999999997e-05, + "log_odds_chosen": 0.2633849084377289, + "log_odds_ratio": -0.5829953551292419, + "logits/chosen": 0.0013431366533041, + "logits/rejected": -0.17736227810382843, + "logps/chosen": -1.2399024963378906, + "logps/rejected": -1.4264099597930908, + "loss": 2.5525, + "nll_loss": 2.4942123889923096, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1239902600646019, + "rewards/margins": 0.018650764599442482, + "rewards/rejected": -0.14264102280139923, + "step": 110 + }, + { + "epoch": 0.06905132192846034, + "grad_norm": 0.4274824857711792, + "learning_rate": 4.7225e-05, + "log_odds_chosen": -0.1062813401222229, + "log_odds_ratio": -0.809943675994873, + "logits/chosen": 0.17697763442993164, + "logits/rejected": -0.11846765875816345, + "logps/chosen": -1.3306231498718262, + "logps/rejected": -1.2187504768371582, + "loss": 3.1534, + "nll_loss": 3.0723817348480225, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.13306231796741486, + "rewards/margins": -0.011187261901795864, + "rewards/rejected": -0.12187506258487701, + "step": 111 + }, + { + "epoch": 0.06967340590979783, + "grad_norm": 0.44988536834716797, + "learning_rate": 4.72e-05, + "log_odds_chosen": 0.298068106174469, + "log_odds_ratio": -0.5905066132545471, + "logits/chosen": 0.060463353991508484, + "logits/rejected": -0.10935623943805695, + "logps/chosen": -1.1479036808013916, + "logps/rejected": -1.342679738998413, + "loss": 2.709, + "nll_loss": 2.6499881744384766, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11479037255048752, + "rewards/margins": 0.01947760209441185, + "rewards/rejected": -0.13426797091960907, + "step": 112 + }, + { + "epoch": 0.07029548989113531, + "grad_norm": 0.4988582134246826, + "learning_rate": 4.7175e-05, + "log_odds_chosen": -0.08553829789161682, + "log_odds_ratio": -0.8603007793426514, + "logits/chosen": 0.39943423867225647, + "logits/rejected": 0.06061486154794693, + "logps/chosen": -1.4535963535308838, + "logps/rejected": -1.3087565898895264, + "loss": 3.8892, + "nll_loss": 3.8031439781188965, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14535963535308838, + "rewards/margins": -0.014483977109193802, + "rewards/rejected": -0.13087564706802368, + "step": 113 + }, + { + "epoch": 0.07091757387247279, + "grad_norm": 0.46725013852119446, + "learning_rate": 4.715e-05, + "log_odds_chosen": 0.641747772693634, + "log_odds_ratio": -0.5314986705780029, + "logits/chosen": 0.09639215469360352, + "logits/rejected": -0.023075614124536514, + "logps/chosen": -0.903424859046936, + "logps/rejected": -1.2310090065002441, + "loss": 3.0129, + "nll_loss": 2.9597768783569336, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09034248441457748, + "rewards/margins": 0.032758425921201706, + "rewards/rejected": -0.12310090661048889, + "step": 114 + }, + { + "epoch": 0.07153965785381027, + "grad_norm": 0.4860602617263794, + "learning_rate": 4.7125e-05, + "log_odds_chosen": -0.0031689107418060303, + "log_odds_ratio": -0.7295066118240356, + "logits/chosen": 0.21932338178157806, + "logits/rejected": 0.0026386789977550507, + "logps/chosen": -1.5008599758148193, + "logps/rejected": -1.5008916854858398, + "loss": 3.3598, + "nll_loss": 3.2868006229400635, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.15008598566055298, + "rewards/margins": 3.1907111406326294e-06, + "rewards/rejected": -0.15008917450904846, + "step": 115 + }, + { + "epoch": 0.07216174183514774, + "grad_norm": 0.8297399282455444, + "learning_rate": 4.71e-05, + "log_odds_chosen": -0.01504303514957428, + "log_odds_ratio": -0.7895686030387878, + "logits/chosen": 0.08432838320732117, + "logits/rejected": -0.08494340628385544, + "logps/chosen": -1.3483591079711914, + "logps/rejected": -1.3111402988433838, + "loss": 3.046, + "nll_loss": 2.9669947624206543, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13483591377735138, + "rewards/margins": -0.0037218770012259483, + "rewards/rejected": -0.13111403584480286, + "step": 116 + }, + { + "epoch": 0.07278382581648522, + "grad_norm": 0.4088709354400635, + "learning_rate": 4.7075e-05, + "log_odds_chosen": 0.20638790726661682, + "log_odds_ratio": -0.6793928146362305, + "logits/chosen": 0.2052862048149109, + "logits/rejected": 0.044292062520980835, + "logps/chosen": -1.2724294662475586, + "logps/rejected": -1.4322636127471924, + "loss": 3.539, + "nll_loss": 3.471055030822754, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.12724295258522034, + "rewards/margins": 0.01598340831696987, + "rewards/rejected": -0.14322635531425476, + "step": 117 + }, + { + "epoch": 0.0734059097978227, + "grad_norm": 0.42515477538108826, + "learning_rate": 4.705e-05, + "log_odds_chosen": -0.019400358200073242, + "log_odds_ratio": -0.7594976425170898, + "logits/chosen": 0.27322930097579956, + "logits/rejected": 0.07737819850444794, + "logps/chosen": -1.2583709955215454, + "logps/rejected": -1.2143146991729736, + "loss": 3.5821, + "nll_loss": 3.5061068534851074, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.12583710253238678, + "rewards/margins": -0.004405634477734566, + "rewards/rejected": -0.12143146991729736, + "step": 118 + }, + { + "epoch": 0.07402799377916018, + "grad_norm": 0.5038117170333862, + "learning_rate": 4.7025000000000005e-05, + "log_odds_chosen": 0.38098299503326416, + "log_odds_ratio": -0.5279546976089478, + "logits/chosen": 0.15353046357631683, + "logits/rejected": -0.20058253407478333, + "logps/chosen": -1.2580475807189941, + "logps/rejected": -1.5402312278747559, + "loss": 3.1463, + "nll_loss": 3.0934572219848633, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12580475211143494, + "rewards/margins": 0.028218373656272888, + "rewards/rejected": -0.15402314066886902, + "step": 119 + }, + { + "epoch": 0.07465007776049767, + "grad_norm": 0.4213177561759949, + "learning_rate": 4.7e-05, + "log_odds_chosen": 0.2627849876880646, + "log_odds_ratio": -0.6001940965652466, + "logits/chosen": 0.3041577935218811, + "logits/rejected": -0.03829241171479225, + "logps/chosen": -1.1501599550247192, + "logps/rejected": -1.3457529544830322, + "loss": 3.6689, + "nll_loss": 3.6088767051696777, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11501598358154297, + "rewards/margins": 0.019559307023882866, + "rewards/rejected": -0.13457529246807098, + "step": 120 + }, + { + "epoch": 0.07527216174183515, + "grad_norm": 0.5030676126480103, + "learning_rate": 4.6975000000000003e-05, + "log_odds_chosen": 0.1841212958097458, + "log_odds_ratio": -0.6212969422340393, + "logits/chosen": 0.10191065073013306, + "logits/rejected": 0.07480275630950928, + "logps/chosen": -1.2450635433197021, + "logps/rejected": -1.3654481172561646, + "loss": 2.7331, + "nll_loss": 2.670966625213623, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12450635433197021, + "rewards/margins": 0.012038461863994598, + "rewards/rejected": -0.13654480874538422, + "step": 121 + }, + { + "epoch": 0.07589424572317263, + "grad_norm": 0.47793930768966675, + "learning_rate": 4.695e-05, + "log_odds_chosen": 0.22587494552135468, + "log_odds_ratio": -0.6521732211112976, + "logits/chosen": 0.14298784732818604, + "logits/rejected": 0.07297507673501968, + "logps/chosen": -1.1125988960266113, + "logps/rejected": -1.2532191276550293, + "loss": 3.0589, + "nll_loss": 2.993687391281128, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11125989258289337, + "rewards/margins": 0.014062018133699894, + "rewards/rejected": -0.12532192468643188, + "step": 122 + }, + { + "epoch": 0.07651632970451011, + "grad_norm": 0.585175096988678, + "learning_rate": 4.6925e-05, + "log_odds_chosen": 0.2622927725315094, + "log_odds_ratio": -0.598949670791626, + "logits/chosen": 0.06829071789979935, + "logits/rejected": -0.06984008848667145, + "logps/chosen": -1.279144287109375, + "logps/rejected": -1.4879951477050781, + "loss": 2.6114, + "nll_loss": 2.5515384674072266, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1279144287109375, + "rewards/margins": 0.020885098725557327, + "rewards/rejected": -0.14879952371120453, + "step": 123 + }, + { + "epoch": 0.07713841368584759, + "grad_norm": 0.5636870861053467, + "learning_rate": 4.69e-05, + "log_odds_chosen": 0.9784868955612183, + "log_odds_ratio": -0.40459758043289185, + "logits/chosen": 0.18325799703598022, + "logits/rejected": -0.04458653926849365, + "logps/chosen": -0.9027924537658691, + "logps/rejected": -1.5085422992706299, + "loss": 3.4475, + "nll_loss": 3.4070000648498535, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09027925133705139, + "rewards/margins": 0.060574986040592194, + "rewards/rejected": -0.150854229927063, + "step": 124 + }, + { + "epoch": 0.07776049766718507, + "grad_norm": 0.5631504654884338, + "learning_rate": 4.6875e-05, + "log_odds_chosen": 0.8957180976867676, + "log_odds_ratio": -0.46610015630722046, + "logits/chosen": 0.13964581489562988, + "logits/rejected": 0.029128514230251312, + "logps/chosen": -0.8781135678291321, + "logps/rejected": -1.3816094398498535, + "loss": 2.6724, + "nll_loss": 2.6258134841918945, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08781135827302933, + "rewards/margins": 0.05034959316253662, + "rewards/rejected": -0.13816094398498535, + "step": 125 + }, + { + "epoch": 0.07838258164852255, + "grad_norm": 0.3702596426010132, + "learning_rate": 4.685000000000001e-05, + "log_odds_chosen": 0.15894243121147156, + "log_odds_ratio": -0.6334323883056641, + "logits/chosen": 0.2041868418455124, + "logits/rejected": 0.06562945991754532, + "logps/chosen": -1.257232427597046, + "logps/rejected": -1.3767375946044922, + "loss": 3.3947, + "nll_loss": 3.3313791751861572, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12572325766086578, + "rewards/margins": 0.011950517073273659, + "rewards/rejected": -0.1376737654209137, + "step": 126 + }, + { + "epoch": 0.07900466562986003, + "grad_norm": 0.43313437700271606, + "learning_rate": 4.6825e-05, + "log_odds_chosen": 0.21152283251285553, + "log_odds_ratio": -0.6254419684410095, + "logits/chosen": -0.06763223558664322, + "logits/rejected": -0.05511503666639328, + "logps/chosen": -1.2304375171661377, + "logps/rejected": -1.3703246116638184, + "loss": 2.5647, + "nll_loss": 2.502199411392212, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12304375320672989, + "rewards/margins": 0.013988707214593887, + "rewards/rejected": -0.13703244924545288, + "step": 127 + }, + { + "epoch": 0.0796267496111975, + "grad_norm": 0.4836374521255493, + "learning_rate": 4.6800000000000006e-05, + "log_odds_chosen": -0.18248885869979858, + "log_odds_ratio": -0.8107894659042358, + "logits/chosen": 0.10594216734170914, + "logits/rejected": -0.056509390473365784, + "logps/chosen": -1.2352960109710693, + "logps/rejected": -1.1191729307174683, + "loss": 3.1493, + "nll_loss": 3.0681915283203125, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1235295981168747, + "rewards/margins": -0.011612308211624622, + "rewards/rejected": -0.11191728711128235, + "step": 128 + }, + { + "epoch": 0.080248833592535, + "grad_norm": 0.41119441390037537, + "learning_rate": 4.6775000000000005e-05, + "log_odds_chosen": 0.48518961668014526, + "log_odds_ratio": -0.49193328619003296, + "logits/chosen": 0.09460698068141937, + "logits/rejected": 0.13982588052749634, + "logps/chosen": -1.2239277362823486, + "logps/rejected": -1.5707978010177612, + "loss": 2.8458, + "nll_loss": 2.796567916870117, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12239278107881546, + "rewards/margins": 0.03468699753284454, + "rewards/rejected": -0.1570797860622406, + "step": 129 + }, + { + "epoch": 0.08087091757387248, + "grad_norm": 0.5971326231956482, + "learning_rate": 4.6750000000000005e-05, + "log_odds_chosen": 0.3527142405509949, + "log_odds_ratio": -0.6361883878707886, + "logits/chosen": 0.3607098460197449, + "logits/rejected": -0.14897310733795166, + "logps/chosen": -1.3266918659210205, + "logps/rejected": -1.5898045301437378, + "loss": 3.5671, + "nll_loss": 3.5034427642822266, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13266919553279877, + "rewards/margins": 0.02631126344203949, + "rewards/rejected": -0.15898045897483826, + "step": 130 + }, + { + "epoch": 0.08149300155520996, + "grad_norm": 0.33660298585891724, + "learning_rate": 4.6725000000000004e-05, + "log_odds_chosen": 0.866208553314209, + "log_odds_ratio": -0.4489937126636505, + "logits/chosen": 0.11533968150615692, + "logits/rejected": 0.0789806991815567, + "logps/chosen": -0.9988769888877869, + "logps/rejected": -1.605279803276062, + "loss": 2.9545, + "nll_loss": 2.9095656871795654, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09988771378993988, + "rewards/margins": 0.06064027547836304, + "rewards/rejected": -0.16052797436714172, + "step": 131 + }, + { + "epoch": 0.08211508553654744, + "grad_norm": 0.31415626406669617, + "learning_rate": 4.6700000000000003e-05, + "log_odds_chosen": 0.2750133275985718, + "log_odds_ratio": -0.668391227722168, + "logits/chosen": 0.2666417360305786, + "logits/rejected": -0.017590831965208054, + "logps/chosen": -1.0113110542297363, + "logps/rejected": -1.1601898670196533, + "loss": 3.5387, + "nll_loss": 3.471817970275879, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.10113111138343811, + "rewards/margins": 0.014887874014675617, + "rewards/rejected": -0.11601898819208145, + "step": 132 + }, + { + "epoch": 0.08273716951788491, + "grad_norm": 0.5030128955841064, + "learning_rate": 4.6675e-05, + "log_odds_chosen": 1.185779094696045, + "log_odds_ratio": -0.3972381353378296, + "logits/chosen": 0.14357468485832214, + "logits/rejected": 0.00014878623187541962, + "logps/chosen": -1.0277682542800903, + "logps/rejected": -1.820441722869873, + "loss": 3.2187, + "nll_loss": 3.178988456726074, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10277682542800903, + "rewards/margins": 0.07926735281944275, + "rewards/rejected": -0.1820441633462906, + "step": 133 + }, + { + "epoch": 0.0833592534992224, + "grad_norm": 0.4640537202358246, + "learning_rate": 4.665e-05, + "log_odds_chosen": -0.1519235223531723, + "log_odds_ratio": -0.8262673616409302, + "logits/chosen": 0.11444417387247086, + "logits/rejected": -0.012055326253175735, + "logps/chosen": -1.376497745513916, + "logps/rejected": -1.2670477628707886, + "loss": 2.9043, + "nll_loss": 2.821722984313965, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1376497745513916, + "rewards/margins": -0.010944999754428864, + "rewards/rejected": -0.12670478224754333, + "step": 134 + }, + { + "epoch": 0.08398133748055987, + "grad_norm": 0.4529246389865875, + "learning_rate": 4.6625e-05, + "log_odds_chosen": 0.1725572943687439, + "log_odds_ratio": -0.6657087206840515, + "logits/chosen": 0.08027800917625427, + "logits/rejected": -0.09482064098119736, + "logps/chosen": -1.1149572134017944, + "logps/rejected": -1.2249038219451904, + "loss": 2.7318, + "nll_loss": 2.6652512550354004, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1114957183599472, + "rewards/margins": 0.01099465787410736, + "rewards/rejected": -0.12249037623405457, + "step": 135 + }, + { + "epoch": 0.08460342146189735, + "grad_norm": 0.4171338677406311, + "learning_rate": 4.660000000000001e-05, + "log_odds_chosen": -0.09017062187194824, + "log_odds_ratio": -0.7511317133903503, + "logits/chosen": 0.2360968291759491, + "logits/rejected": -0.010965634137392044, + "logps/chosen": -1.1789023876190186, + "logps/rejected": -1.0944397449493408, + "loss": 3.3572, + "nll_loss": 3.2820589542388916, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.11789023876190186, + "rewards/margins": -0.008446259424090385, + "rewards/rejected": -0.10944397747516632, + "step": 136 + }, + { + "epoch": 0.08522550544323483, + "grad_norm": 0.547868013381958, + "learning_rate": 4.6575e-05, + "log_odds_chosen": 0.7603722810745239, + "log_odds_ratio": -0.49443089962005615, + "logits/chosen": -0.03645198792219162, + "logits/rejected": -0.07120641320943832, + "logps/chosen": -1.1619699001312256, + "logps/rejected": -1.7555570602416992, + "loss": 2.7566, + "nll_loss": 2.707111120223999, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11619699001312256, + "rewards/margins": 0.05935873091220856, + "rewards/rejected": -0.17555572092533112, + "step": 137 + }, + { + "epoch": 0.08584758942457232, + "grad_norm": 0.3483281433582306, + "learning_rate": 4.655000000000001e-05, + "log_odds_chosen": 0.3946763873100281, + "log_odds_ratio": -0.6060354113578796, + "logits/chosen": 0.09304636716842651, + "logits/rejected": 0.01444307342171669, + "logps/chosen": -1.247818946838379, + "logps/rejected": -1.4207770824432373, + "loss": 3.0866, + "nll_loss": 3.025946855545044, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.12478190660476685, + "rewards/margins": 0.017295803874731064, + "rewards/rejected": -0.1420777142047882, + "step": 138 + }, + { + "epoch": 0.0864696734059098, + "grad_norm": 0.4377744495868683, + "learning_rate": 4.6525e-05, + "log_odds_chosen": 0.5676741003990173, + "log_odds_ratio": -0.4999235272407532, + "logits/chosen": 0.18979693949222565, + "logits/rejected": -0.09224162995815277, + "logps/chosen": -1.0275253057479858, + "logps/rejected": -1.4091662168502808, + "loss": 3.1186, + "nll_loss": 3.068612575531006, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10275252908468246, + "rewards/margins": 0.03816410154104233, + "rewards/rejected": -0.140916645526886, + "step": 139 + }, + { + "epoch": 0.08709175738724728, + "grad_norm": 0.5015956163406372, + "learning_rate": 4.6500000000000005e-05, + "log_odds_chosen": 0.40913963317871094, + "log_odds_ratio": -0.5399808883666992, + "logits/chosen": 0.13208572566509247, + "logits/rejected": -0.042870864272117615, + "logps/chosen": -1.4589378833770752, + "logps/rejected": -1.76136314868927, + "loss": 2.8749, + "nll_loss": 2.8209028244018555, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14589379727840424, + "rewards/margins": 0.03024252876639366, + "rewards/rejected": -0.176136314868927, + "step": 140 + }, + { + "epoch": 0.08771384136858476, + "grad_norm": 0.5315511226654053, + "learning_rate": 4.6475000000000005e-05, + "log_odds_chosen": 0.23443233966827393, + "log_odds_ratio": -0.6079375743865967, + "logits/chosen": 0.10196913778781891, + "logits/rejected": -0.09439704567193985, + "logps/chosen": -1.2660562992095947, + "logps/rejected": -1.440688133239746, + "loss": 2.9088, + "nll_loss": 2.8479607105255127, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.12660562992095947, + "rewards/margins": 0.01746317744255066, + "rewards/rejected": -0.14406880736351013, + "step": 141 + }, + { + "epoch": 0.08833592534992224, + "grad_norm": 0.48670142889022827, + "learning_rate": 4.6450000000000004e-05, + "log_odds_chosen": 0.20135235786437988, + "log_odds_ratio": -0.6504536271095276, + "logits/chosen": 0.057795051485300064, + "logits/rejected": -0.16002613306045532, + "logps/chosen": -1.202336072921753, + "logps/rejected": -1.336153268814087, + "loss": 2.9102, + "nll_loss": 2.8451108932495117, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.12023360282182693, + "rewards/margins": 0.01338173821568489, + "rewards/rejected": -0.13361534476280212, + "step": 142 + }, + { + "epoch": 0.08895800933125972, + "grad_norm": 0.4415438771247864, + "learning_rate": 4.6425000000000004e-05, + "log_odds_chosen": 0.61316978931427, + "log_odds_ratio": -0.4622770845890045, + "logits/chosen": 0.04670334979891777, + "logits/rejected": 0.007671605795621872, + "logps/chosen": -1.3000093698501587, + "logps/rejected": -1.7552378177642822, + "loss": 2.9155, + "nll_loss": 2.869236707687378, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13000094890594482, + "rewards/margins": 0.045522838830947876, + "rewards/rejected": -0.1755237877368927, + "step": 143 + }, + { + "epoch": 0.0895800933125972, + "grad_norm": 0.5169774889945984, + "learning_rate": 4.64e-05, + "log_odds_chosen": 0.6011465787887573, + "log_odds_ratio": -0.5140647292137146, + "logits/chosen": 0.007771043106913567, + "logits/rejected": -0.15149809420108795, + "logps/chosen": -1.184515118598938, + "logps/rejected": -1.645982027053833, + "loss": 2.7975, + "nll_loss": 2.7460548877716064, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11845151335000992, + "rewards/margins": 0.0461466908454895, + "rewards/rejected": -0.16459819674491882, + "step": 144 + }, + { + "epoch": 0.09020217729393468, + "grad_norm": 0.6221879720687866, + "learning_rate": 4.6375e-05, + "log_odds_chosen": 0.6371638774871826, + "log_odds_ratio": -0.5189849138259888, + "logits/chosen": 0.051749397069215775, + "logits/rejected": -0.1522921919822693, + "logps/chosen": -0.8362572193145752, + "logps/rejected": -1.1811851263046265, + "loss": 3.1062, + "nll_loss": 3.0543196201324463, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08362571895122528, + "rewards/margins": 0.034492794424295425, + "rewards/rejected": -0.1181185245513916, + "step": 145 + }, + { + "epoch": 0.09082426127527216, + "grad_norm": 0.4802875518798828, + "learning_rate": 4.635e-05, + "log_odds_chosen": 0.3668875992298126, + "log_odds_ratio": -0.5806620717048645, + "logits/chosen": 0.10094377398490906, + "logits/rejected": -0.02433057874441147, + "logps/chosen": -1.2760828733444214, + "logps/rejected": -1.5276360511779785, + "loss": 2.9177, + "nll_loss": 2.859609842300415, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1276082843542099, + "rewards/margins": 0.02515532448887825, + "rewards/rejected": -0.15276360511779785, + "step": 146 + }, + { + "epoch": 0.09144634525660965, + "grad_norm": 0.4567488133907318, + "learning_rate": 4.6325e-05, + "log_odds_chosen": 0.6026850938796997, + "log_odds_ratio": -0.4573545753955841, + "logits/chosen": 0.19558559358119965, + "logits/rejected": 0.03107052482664585, + "logps/chosen": -1.185868263244629, + "logps/rejected": -1.6299797296524048, + "loss": 3.3654, + "nll_loss": 3.3196253776550293, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11858682334423065, + "rewards/margins": 0.044411152601242065, + "rewards/rejected": -0.16299797594547272, + "step": 147 + }, + { + "epoch": 0.09206842923794713, + "grad_norm": 0.5302126407623291, + "learning_rate": 4.630000000000001e-05, + "log_odds_chosen": -0.01642867922782898, + "log_odds_ratio": -0.7874202728271484, + "logits/chosen": 0.11570745706558228, + "logits/rejected": -0.1371677815914154, + "logps/chosen": -1.3408689498901367, + "logps/rejected": -1.3910057544708252, + "loss": 3.5827, + "nll_loss": 3.5039186477661133, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13408689200878143, + "rewards/margins": 0.005013669840991497, + "rewards/rejected": -0.1391005516052246, + "step": 148 + }, + { + "epoch": 0.0926905132192846, + "grad_norm": 0.4184373617172241, + "learning_rate": 4.6275e-05, + "log_odds_chosen": 0.05463185906410217, + "log_odds_ratio": -0.7645725607872009, + "logits/chosen": 0.1671661138534546, + "logits/rejected": 0.034170206636190414, + "logps/chosen": -1.3616355657577515, + "logps/rejected": -1.3897101879119873, + "loss": 3.3596, + "nll_loss": 3.283134698867798, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.13616356253623962, + "rewards/margins": 0.002807457000017166, + "rewards/rejected": -0.13897103071212769, + "step": 149 + }, + { + "epoch": 0.09331259720062209, + "grad_norm": 0.42127570509910583, + "learning_rate": 4.6250000000000006e-05, + "log_odds_chosen": 0.08022980391979218, + "log_odds_ratio": -0.7030846476554871, + "logits/chosen": 0.0035371780395507812, + "logits/rejected": 0.003839358687400818, + "logps/chosen": -1.3736772537231445, + "logps/rejected": -1.378920078277588, + "loss": 2.8468, + "nll_loss": 2.7764601707458496, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13736772537231445, + "rewards/margins": 0.0005242684856057167, + "rewards/rejected": -0.1378920078277588, + "step": 150 + }, + { + "epoch": 0.09393468118195956, + "grad_norm": 0.4007570147514343, + "learning_rate": 4.6225e-05, + "log_odds_chosen": 0.06232370063662529, + "log_odds_ratio": -0.6894880533218384, + "logits/chosen": 0.028553307056427002, + "logits/rejected": 0.13209117949008942, + "logps/chosen": -1.5964021682739258, + "logps/rejected": -1.6478145122528076, + "loss": 2.7767, + "nll_loss": 2.707792282104492, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15964022278785706, + "rewards/margins": 0.005141226574778557, + "rewards/rejected": -0.16478145122528076, + "step": 151 + }, + { + "epoch": 0.09455676516329704, + "grad_norm": 0.506729006767273, + "learning_rate": 4.6200000000000005e-05, + "log_odds_chosen": 0.44813084602355957, + "log_odds_ratio": -0.6032027006149292, + "logits/chosen": 0.19062553346157074, + "logits/rejected": -0.01674717850983143, + "logps/chosen": -1.1239427328109741, + "logps/rejected": -1.383901834487915, + "loss": 3.1411, + "nll_loss": 3.0807762145996094, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11239427328109741, + "rewards/margins": 0.025995904579758644, + "rewards/rejected": -0.1383901834487915, + "step": 152 + }, + { + "epoch": 0.09517884914463452, + "grad_norm": 0.4609549641609192, + "learning_rate": 4.6175000000000004e-05, + "log_odds_chosen": 0.4586949646472931, + "log_odds_ratio": -0.5538212060928345, + "logits/chosen": 0.25490209460258484, + "logits/rejected": 0.08342747390270233, + "logps/chosen": -1.3676748275756836, + "logps/rejected": -1.6974760293960571, + "loss": 3.281, + "nll_loss": 3.2256217002868652, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13676749169826508, + "rewards/margins": 0.032980117946863174, + "rewards/rejected": -0.16974762082099915, + "step": 153 + }, + { + "epoch": 0.095800933125972, + "grad_norm": 0.3721259832382202, + "learning_rate": 4.6150000000000004e-05, + "log_odds_chosen": 0.5167128443717957, + "log_odds_ratio": -0.5634645223617554, + "logits/chosen": 0.06376560777425766, + "logits/rejected": -0.0771709680557251, + "logps/chosen": -1.1290371417999268, + "logps/rejected": -1.432267189025879, + "loss": 2.8284, + "nll_loss": 2.7720093727111816, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11290371417999268, + "rewards/margins": 0.03032299503684044, + "rewards/rejected": -0.1432267129421234, + "step": 154 + }, + { + "epoch": 0.09642301710730948, + "grad_norm": 0.43300989270210266, + "learning_rate": 4.6125e-05, + "log_odds_chosen": 0.20810005068778992, + "log_odds_ratio": -0.7468218207359314, + "logits/chosen": 0.2738891839981079, + "logits/rejected": 0.0035995468497276306, + "logps/chosen": -1.1184593439102173, + "logps/rejected": -1.2356008291244507, + "loss": 3.6973, + "nll_loss": 3.6226534843444824, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1118459403514862, + "rewards/margins": 0.011714148335158825, + "rewards/rejected": -0.12356008589267731, + "step": 155 + }, + { + "epoch": 0.09704510108864697, + "grad_norm": 0.4422686994075775, + "learning_rate": 4.61e-05, + "log_odds_chosen": 0.38565486669540405, + "log_odds_ratio": -0.5663627982139587, + "logits/chosen": 0.1406802535057068, + "logits/rejected": -0.10811451077461243, + "logps/chosen": -1.2533131837844849, + "logps/rejected": -1.501025915145874, + "loss": 2.9378, + "nll_loss": 2.881122589111328, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.125331312417984, + "rewards/margins": 0.02477126568555832, + "rewards/rejected": -0.15010258555412292, + "step": 156 + }, + { + "epoch": 0.09766718506998445, + "grad_norm": 0.43218180537223816, + "learning_rate": 4.6075e-05, + "log_odds_chosen": 0.7944599390029907, + "log_odds_ratio": -0.4741981029510498, + "logits/chosen": 0.13778428733348846, + "logits/rejected": 0.048661183565855026, + "logps/chosen": -1.2979130744934082, + "logps/rejected": -1.8968315124511719, + "loss": 3.1609, + "nll_loss": 3.1134564876556396, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12979131937026978, + "rewards/margins": 0.05989184230566025, + "rewards/rejected": -0.18968316912651062, + "step": 157 + }, + { + "epoch": 0.09828926905132193, + "grad_norm": 0.3956831991672516, + "learning_rate": 4.605e-05, + "log_odds_chosen": 0.27946868538856506, + "log_odds_ratio": -0.6237690448760986, + "logits/chosen": 0.3799353837966919, + "logits/rejected": 0.11462019383907318, + "logps/chosen": -1.2597386837005615, + "logps/rejected": -1.4242260456085205, + "loss": 3.4372, + "nll_loss": 3.374814987182617, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1259738653898239, + "rewards/margins": 0.016448739916086197, + "rewards/rejected": -0.142422616481781, + "step": 158 + }, + { + "epoch": 0.09891135303265941, + "grad_norm": 0.40490007400512695, + "learning_rate": 4.6025e-05, + "log_odds_chosen": 1.3628696203231812, + "log_odds_ratio": -0.38135695457458496, + "logits/chosen": 0.16640058159828186, + "logits/rejected": 0.0424039289355278, + "logps/chosen": -0.7394822835922241, + "logps/rejected": -1.6354302167892456, + "loss": 3.0489, + "nll_loss": 3.010765552520752, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07394823431968689, + "rewards/margins": 0.08959478884935379, + "rewards/rejected": -0.16354301571846008, + "step": 159 + }, + { + "epoch": 0.09953343701399689, + "grad_norm": 0.522266685962677, + "learning_rate": 4.600000000000001e-05, + "log_odds_chosen": 0.31891077756881714, + "log_odds_ratio": -0.6262969374656677, + "logits/chosen": 0.2351851910352707, + "logits/rejected": 0.0422259159386158, + "logps/chosen": -1.1578633785247803, + "logps/rejected": -1.3393428325653076, + "loss": 2.8211, + "nll_loss": 2.7584638595581055, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11578632891178131, + "rewards/margins": 0.018147945404052734, + "rewards/rejected": -0.13393428921699524, + "step": 160 + }, + { + "epoch": 0.10015552099533437, + "grad_norm": 0.4992579221725464, + "learning_rate": 4.5975e-05, + "log_odds_chosen": 0.36979615688323975, + "log_odds_ratio": -0.5988181829452515, + "logits/chosen": 0.2722373604774475, + "logits/rejected": 0.13797958195209503, + "logps/chosen": -1.223292589187622, + "logps/rejected": -1.4450644254684448, + "loss": 3.065, + "nll_loss": 3.0051493644714355, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.12232926487922668, + "rewards/margins": 0.022177184000611305, + "rewards/rejected": -0.14450645446777344, + "step": 161 + }, + { + "epoch": 0.10077760497667185, + "grad_norm": 0.48701006174087524, + "learning_rate": 4.5950000000000006e-05, + "log_odds_chosen": 0.34209126234054565, + "log_odds_ratio": -0.550736665725708, + "logits/chosen": 0.14484034478664398, + "logits/rejected": -0.13392439484596252, + "logps/chosen": -1.1791408061981201, + "logps/rejected": -1.4220118522644043, + "loss": 2.7731, + "nll_loss": 2.718043088912964, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11791408061981201, + "rewards/margins": 0.02428712509572506, + "rewards/rejected": -0.14220120012760162, + "step": 162 + }, + { + "epoch": 0.10139968895800933, + "grad_norm": 0.42111936211586, + "learning_rate": 4.5925e-05, + "log_odds_chosen": 0.09586916118860245, + "log_odds_ratio": -0.6500096321105957, + "logits/chosen": 0.3419564366340637, + "logits/rejected": 0.21585996448993683, + "logps/chosen": -1.4152202606201172, + "logps/rejected": -1.4942808151245117, + "loss": 3.3547, + "nll_loss": 3.2897045612335205, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14152203500270844, + "rewards/margins": 0.007906057871878147, + "rewards/rejected": -0.1494280993938446, + "step": 163 + }, + { + "epoch": 0.1020217729393468, + "grad_norm": 0.651195228099823, + "learning_rate": 4.5900000000000004e-05, + "log_odds_chosen": 0.007867768406867981, + "log_odds_ratio": -0.8247443437576294, + "logits/chosen": 0.15761840343475342, + "logits/rejected": 0.04952532798051834, + "logps/chosen": -1.4861036539077759, + "logps/rejected": -1.4350836277008057, + "loss": 2.6309, + "nll_loss": 2.548461437225342, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.14861036837100983, + "rewards/margins": -0.005102001130580902, + "rewards/rejected": -0.14350835978984833, + "step": 164 + }, + { + "epoch": 0.1026438569206843, + "grad_norm": 0.47373661398887634, + "learning_rate": 4.5875000000000004e-05, + "log_odds_chosen": 0.15654607117176056, + "log_odds_ratio": -0.67220139503479, + "logits/chosen": 0.42479315400123596, + "logits/rejected": 0.2729908227920532, + "logps/chosen": -1.3848153352737427, + "logps/rejected": -1.4821150302886963, + "loss": 3.1219, + "nll_loss": 3.054720878601074, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1384815275669098, + "rewards/margins": 0.009729975834488869, + "rewards/rejected": -0.1482115089893341, + "step": 165 + }, + { + "epoch": 0.10326594090202178, + "grad_norm": 0.39424991607666016, + "learning_rate": 4.585e-05, + "log_odds_chosen": 0.8976173400878906, + "log_odds_ratio": -0.5067555904388428, + "logits/chosen": 0.12819889187812805, + "logits/rejected": -0.08246054500341415, + "logps/chosen": -0.9714441299438477, + "logps/rejected": -1.3971306085586548, + "loss": 3.0076, + "nll_loss": 2.9568872451782227, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09714441001415253, + "rewards/margins": 0.04256865009665489, + "rewards/rejected": -0.13971306383609772, + "step": 166 + }, + { + "epoch": 0.10388802488335926, + "grad_norm": 0.4359923303127289, + "learning_rate": 4.5825e-05, + "log_odds_chosen": 0.13466989994049072, + "log_odds_ratio": -0.7068646550178528, + "logits/chosen": 0.29034674167633057, + "logits/rejected": -0.1330055147409439, + "logps/chosen": -1.3547630310058594, + "logps/rejected": -1.426397442817688, + "loss": 3.2544, + "nll_loss": 3.1836836338043213, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13547630608081818, + "rewards/margins": 0.007163448259234428, + "rewards/rejected": -0.14263975620269775, + "step": 167 + }, + { + "epoch": 0.10451010886469674, + "grad_norm": 0.47172248363494873, + "learning_rate": 4.58e-05, + "log_odds_chosen": 0.8191059827804565, + "log_odds_ratio": -0.5032205581665039, + "logits/chosen": 0.17938898503780365, + "logits/rejected": 0.05944395065307617, + "logps/chosen": -1.067983627319336, + "logps/rejected": -1.5608168840408325, + "loss": 2.9386, + "nll_loss": 2.888256311416626, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10679835081100464, + "rewards/margins": 0.04928332567214966, + "rewards/rejected": -0.1560816764831543, + "step": 168 + }, + { + "epoch": 0.10513219284603421, + "grad_norm": 0.5837737321853638, + "learning_rate": 4.5775e-05, + "log_odds_chosen": 0.21645092964172363, + "log_odds_ratio": -0.6055108904838562, + "logits/chosen": 0.42070725560188293, + "logits/rejected": 0.0440496951341629, + "logps/chosen": -1.0943289995193481, + "logps/rejected": -1.2313555479049683, + "loss": 3.6424, + "nll_loss": 3.5818536281585693, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1094328984618187, + "rewards/margins": 0.01370265707373619, + "rewards/rejected": -0.12313555181026459, + "step": 169 + }, + { + "epoch": 0.1057542768273717, + "grad_norm": 0.5199751257896423, + "learning_rate": 4.575e-05, + "log_odds_chosen": 0.17023658752441406, + "log_odds_ratio": -0.7053340077400208, + "logits/chosen": 0.16789722442626953, + "logits/rejected": 0.13283823430538177, + "logps/chosen": -1.2800236940383911, + "logps/rejected": -1.3655457496643066, + "loss": 2.931, + "nll_loss": 2.860494613647461, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1280023753643036, + "rewards/margins": 0.008552204817533493, + "rewards/rejected": -0.1365545690059662, + "step": 170 + }, + { + "epoch": 0.10637636080870917, + "grad_norm": 0.6239449381828308, + "learning_rate": 4.5725e-05, + "log_odds_chosen": -0.17279069125652313, + "log_odds_ratio": -0.8778814077377319, + "logits/chosen": 0.43515917658805847, + "logits/rejected": 0.1489274650812149, + "logps/chosen": -1.5174874067306519, + "logps/rejected": -1.3432010412216187, + "loss": 3.8779, + "nll_loss": 3.790090799331665, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.15174873173236847, + "rewards/margins": -0.017428629100322723, + "rewards/rejected": -0.13432011008262634, + "step": 171 + }, + { + "epoch": 0.10699844479004665, + "grad_norm": 0.5014814734458923, + "learning_rate": 4.5700000000000006e-05, + "log_odds_chosen": 0.5290875434875488, + "log_odds_ratio": -0.5955926179885864, + "logits/chosen": 0.05028457194566727, + "logits/rejected": -0.058185793459415436, + "logps/chosen": -1.0905635356903076, + "logps/rejected": -1.361629605293274, + "loss": 2.4669, + "nll_loss": 2.407294273376465, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.10905636847019196, + "rewards/margins": 0.027106598019599915, + "rewards/rejected": -0.13616296648979187, + "step": 172 + }, + { + "epoch": 0.10762052877138413, + "grad_norm": 0.45314520597457886, + "learning_rate": 4.5675e-05, + "log_odds_chosen": 0.11316041648387909, + "log_odds_ratio": -0.6723555326461792, + "logits/chosen": 0.1640755981206894, + "logits/rejected": -0.034357067197561264, + "logps/chosen": -1.294669508934021, + "logps/rejected": -1.3866937160491943, + "loss": 2.9407, + "nll_loss": 2.8734371662139893, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1294669657945633, + "rewards/margins": 0.009202406741678715, + "rewards/rejected": -0.13866937160491943, + "step": 173 + }, + { + "epoch": 0.10824261275272162, + "grad_norm": 0.44885244965553284, + "learning_rate": 4.5650000000000005e-05, + "log_odds_chosen": 0.3160461485385895, + "log_odds_ratio": -0.5713240504264832, + "logits/chosen": 0.23067131638526917, + "logits/rejected": 0.04281052201986313, + "logps/chosen": -1.31607186794281, + "logps/rejected": -1.5356450080871582, + "loss": 3.2553, + "nll_loss": 3.1981256008148193, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13160717487335205, + "rewards/margins": 0.021957319229841232, + "rewards/rejected": -0.15356451272964478, + "step": 174 + }, + { + "epoch": 0.1088646967340591, + "grad_norm": 0.49723100662231445, + "learning_rate": 4.5625e-05, + "log_odds_chosen": 0.9637750387191772, + "log_odds_ratio": -0.4667288661003113, + "logits/chosen": 0.13240934908390045, + "logits/rejected": -0.04236384481191635, + "logps/chosen": -1.2129688262939453, + "logps/rejected": -1.8437656164169312, + "loss": 3.156, + "nll_loss": 3.109340190887451, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12129689753055573, + "rewards/margins": 0.06307967007160187, + "rewards/rejected": -0.1843765676021576, + "step": 175 + }, + { + "epoch": 0.10948678071539658, + "grad_norm": 0.5128828883171082, + "learning_rate": 4.5600000000000004e-05, + "log_odds_chosen": 0.3789896070957184, + "log_odds_ratio": -0.5478941798210144, + "logits/chosen": 0.15509214997291565, + "logits/rejected": 0.07832697033882141, + "logps/chosen": -1.4777848720550537, + "logps/rejected": -1.7933158874511719, + "loss": 2.8832, + "nll_loss": 2.8284482955932617, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1477784812450409, + "rewards/margins": 0.031553104519844055, + "rewards/rejected": -0.17933160066604614, + "step": 176 + }, + { + "epoch": 0.11010886469673406, + "grad_norm": 0.41308772563934326, + "learning_rate": 4.5575e-05, + "log_odds_chosen": -0.020543716847896576, + "log_odds_ratio": -0.8699101209640503, + "logits/chosen": 0.18716076016426086, + "logits/rejected": -0.06990818679332733, + "logps/chosen": -1.4498722553253174, + "logps/rejected": -1.3525209426879883, + "loss": 3.5567, + "nll_loss": 3.469757556915283, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.14498722553253174, + "rewards/margins": -0.009735142812132835, + "rewards/rejected": -0.13525208830833435, + "step": 177 + }, + { + "epoch": 0.11073094867807154, + "grad_norm": 0.48727282881736755, + "learning_rate": 4.555e-05, + "log_odds_chosen": 0.26913127303123474, + "log_odds_ratio": -0.6672435998916626, + "logits/chosen": 0.163034588098526, + "logits/rejected": -0.04085027799010277, + "logps/chosen": -1.3829305171966553, + "logps/rejected": -1.5941439867019653, + "loss": 3.2551, + "nll_loss": 3.1884212493896484, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13829305768013, + "rewards/margins": 0.021121343597769737, + "rewards/rejected": -0.1594144105911255, + "step": 178 + }, + { + "epoch": 0.11135303265940902, + "grad_norm": 0.3962324857711792, + "learning_rate": 4.5525e-05, + "log_odds_chosen": 0.2522580623626709, + "log_odds_ratio": -0.6461670994758606, + "logits/chosen": 0.006648369133472443, + "logits/rejected": -0.11574047803878784, + "logps/chosen": -1.311452865600586, + "logps/rejected": -1.4216885566711426, + "loss": 2.9589, + "nll_loss": 2.894331693649292, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13114528357982635, + "rewards/margins": 0.011023584753274918, + "rewards/rejected": -0.14216886460781097, + "step": 179 + }, + { + "epoch": 0.1119751166407465, + "grad_norm": 0.4192509353160858, + "learning_rate": 4.55e-05, + "log_odds_chosen": 0.3250114917755127, + "log_odds_ratio": -0.6104202270507812, + "logits/chosen": 0.20732258260250092, + "logits/rejected": 0.005627447739243507, + "logps/chosen": -1.2976369857788086, + "logps/rejected": -1.5418976545333862, + "loss": 3.5311, + "nll_loss": 3.4701075553894043, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12976369261741638, + "rewards/margins": 0.02442607283592224, + "rewards/rejected": -0.15418976545333862, + "step": 180 + }, + { + "epoch": 0.11259720062208398, + "grad_norm": 0.3853883445262909, + "learning_rate": 4.5475e-05, + "log_odds_chosen": 0.11205227673053741, + "log_odds_ratio": -0.65590500831604, + "logits/chosen": 0.15651991963386536, + "logits/rejected": -0.04156707227230072, + "logps/chosen": -1.3088514804840088, + "logps/rejected": -1.4056215286254883, + "loss": 3.1677, + "nll_loss": 3.102067470550537, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13088513910770416, + "rewards/margins": 0.009677015244960785, + "rewards/rejected": -0.14056214690208435, + "step": 181 + }, + { + "epoch": 0.11321928460342146, + "grad_norm": 0.560540497303009, + "learning_rate": 4.545000000000001e-05, + "log_odds_chosen": 0.7333866953849792, + "log_odds_ratio": -0.4243454039096832, + "logits/chosen": 0.22672435641288757, + "logits/rejected": -0.04371669888496399, + "logps/chosen": -0.94305020570755, + "logps/rejected": -1.453209638595581, + "loss": 2.9935, + "nll_loss": 2.951089859008789, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09430501610040665, + "rewards/margins": 0.0510159507393837, + "rewards/rejected": -0.14532096683979034, + "step": 182 + }, + { + "epoch": 0.11384136858475895, + "grad_norm": 0.3617045283317566, + "learning_rate": 4.5425e-05, + "log_odds_chosen": 1.3186687231063843, + "log_odds_ratio": -0.3125315308570862, + "logits/chosen": 0.09166554361581802, + "logits/rejected": 0.04617012292146683, + "logps/chosen": -0.8101480007171631, + "logps/rejected": -1.4856460094451904, + "loss": 3.1663, + "nll_loss": 3.1350221633911133, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08101479709148407, + "rewards/margins": 0.06754979491233826, + "rewards/rejected": -0.14856459200382233, + "step": 183 + }, + { + "epoch": 0.11446345256609643, + "grad_norm": 0.43170198798179626, + "learning_rate": 4.5400000000000006e-05, + "log_odds_chosen": 0.6821365356445312, + "log_odds_ratio": -0.5054466724395752, + "logits/chosen": 0.12610091269016266, + "logits/rejected": 0.06542984396219254, + "logps/chosen": -1.140608310699463, + "logps/rejected": -1.5901718139648438, + "loss": 3.0324, + "nll_loss": 2.9818594455718994, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11406083405017853, + "rewards/margins": 0.04495634511113167, + "rewards/rejected": -0.1590171754360199, + "step": 184 + }, + { + "epoch": 0.1150855365474339, + "grad_norm": 0.3195982277393341, + "learning_rate": 4.5375e-05, + "log_odds_chosen": 0.5379209518432617, + "log_odds_ratio": -0.49095743894577026, + "logits/chosen": 0.18578393757343292, + "logits/rejected": 0.09170369803905487, + "logps/chosen": -1.0872983932495117, + "logps/rejected": -1.476131558418274, + "loss": 3.5126, + "nll_loss": 3.4635496139526367, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10872984677553177, + "rewards/margins": 0.03888332098722458, + "rewards/rejected": -0.14761316776275635, + "step": 185 + }, + { + "epoch": 0.11570762052877138, + "grad_norm": 0.38618817925453186, + "learning_rate": 4.5350000000000005e-05, + "log_odds_chosen": 0.27880579233169556, + "log_odds_ratio": -0.5841034054756165, + "logits/chosen": 0.22152912616729736, + "logits/rejected": 0.037873830646276474, + "logps/chosen": -1.1515580415725708, + "logps/rejected": -1.3237394094467163, + "loss": 2.9728, + "nll_loss": 2.914379835128784, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11515580117702484, + "rewards/margins": 0.01721813902258873, + "rewards/rejected": -0.13237394392490387, + "step": 186 + }, + { + "epoch": 0.11632970451010886, + "grad_norm": 0.4079546332359314, + "learning_rate": 4.5325000000000004e-05, + "log_odds_chosen": -0.15632662177085876, + "log_odds_ratio": -0.7896630764007568, + "logits/chosen": 0.14888915419578552, + "logits/rejected": 0.015266455709934235, + "logps/chosen": -1.393578290939331, + "logps/rejected": -1.2698719501495361, + "loss": 2.9801, + "nll_loss": 2.9011240005493164, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1393578201532364, + "rewards/margins": -0.012370622716844082, + "rewards/rejected": -0.12698720395565033, + "step": 187 + }, + { + "epoch": 0.11695178849144634, + "grad_norm": 0.41905397176742554, + "learning_rate": 4.53e-05, + "log_odds_chosen": -0.014257103204727173, + "log_odds_ratio": -0.7734930515289307, + "logits/chosen": 0.22645491361618042, + "logits/rejected": 0.10181404650211334, + "logps/chosen": -1.4420051574707031, + "logps/rejected": -1.3750616312026978, + "loss": 3.305, + "nll_loss": 3.2276558876037598, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14420051872730255, + "rewards/margins": -0.006694357842206955, + "rewards/rejected": -0.1375061571598053, + "step": 188 + }, + { + "epoch": 0.11757387247278382, + "grad_norm": 0.4148879647254944, + "learning_rate": 4.5275e-05, + "log_odds_chosen": 0.3161711096763611, + "log_odds_ratio": -0.5657863616943359, + "logits/chosen": 0.09662674367427826, + "logits/rejected": -0.03456714004278183, + "logps/chosen": -1.122739553451538, + "logps/rejected": -1.358430027961731, + "loss": 2.703, + "nll_loss": 2.6464426517486572, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11227396130561829, + "rewards/margins": 0.023569051176309586, + "rewards/rejected": -0.13584300875663757, + "step": 189 + }, + { + "epoch": 0.1181959564541213, + "grad_norm": 1.207576870918274, + "learning_rate": 4.525e-05, + "log_odds_chosen": -0.09153705835342407, + "log_odds_ratio": -0.7909804582595825, + "logits/chosen": 0.0819367915391922, + "logits/rejected": 0.03701075166463852, + "logps/chosen": -1.6013344526290894, + "logps/rejected": -1.5169143676757812, + "loss": 3.0113, + "nll_loss": 2.932208776473999, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1601334512233734, + "rewards/margins": -0.008442009799182415, + "rewards/rejected": -0.15169143676757812, + "step": 190 + }, + { + "epoch": 0.1188180404354588, + "grad_norm": 0.42340028285980225, + "learning_rate": 4.5225e-05, + "log_odds_chosen": 0.7335370779037476, + "log_odds_ratio": -0.4513997435569763, + "logits/chosen": 0.15320263803005219, + "logits/rejected": -0.06381958723068237, + "logps/chosen": -0.8546745181083679, + "logps/rejected": -1.3147962093353271, + "loss": 3.2561, + "nll_loss": 3.210986375808716, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08546745777130127, + "rewards/margins": 0.046012166887521744, + "rewards/rejected": -0.13147962093353271, + "step": 191 + }, + { + "epoch": 0.11944012441679627, + "grad_norm": 0.4323185980319977, + "learning_rate": 4.52e-05, + "log_odds_chosen": 0.43490371108055115, + "log_odds_ratio": -0.5183134078979492, + "logits/chosen": 0.3056216835975647, + "logits/rejected": 0.032028887420892715, + "logps/chosen": -1.260435938835144, + "logps/rejected": -1.577862024307251, + "loss": 3.3018, + "nll_loss": 3.249936819076538, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12604360282421112, + "rewards/margins": 0.031742602586746216, + "rewards/rejected": -0.15778620541095734, + "step": 192 + }, + { + "epoch": 0.12006220839813375, + "grad_norm": 0.3796927332878113, + "learning_rate": 4.5175e-05, + "log_odds_chosen": 0.5728176236152649, + "log_odds_ratio": -0.4924409091472626, + "logits/chosen": 0.09786956757307053, + "logits/rejected": -0.08450242131948471, + "logps/chosen": -0.922787070274353, + "logps/rejected": -1.3142082691192627, + "loss": 2.9078, + "nll_loss": 2.858553886413574, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09227870404720306, + "rewards/margins": 0.039142120629549026, + "rewards/rejected": -0.1314208209514618, + "step": 193 + }, + { + "epoch": 0.12068429237947123, + "grad_norm": 0.7153462171554565, + "learning_rate": 4.5150000000000006e-05, + "log_odds_chosen": 0.5174391865730286, + "log_odds_ratio": -0.5253512263298035, + "logits/chosen": 0.3215792179107666, + "logits/rejected": 0.12055753916501999, + "logps/chosen": -1.3720208406448364, + "logps/rejected": -1.7549231052398682, + "loss": 3.0374, + "nll_loss": 2.98486328125, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13720208406448364, + "rewards/margins": 0.03829022869467735, + "rewards/rejected": -0.1754923164844513, + "step": 194 + }, + { + "epoch": 0.12130637636080871, + "grad_norm": 0.4331601560115814, + "learning_rate": 4.5125e-05, + "log_odds_chosen": 0.7227539420127869, + "log_odds_ratio": -0.4814952313899994, + "logits/chosen": 0.2730720341205597, + "logits/rejected": 0.1367463618516922, + "logps/chosen": -1.1998332738876343, + "logps/rejected": -1.7204265594482422, + "loss": 3.2678, + "nll_loss": 3.21962833404541, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11998332291841507, + "rewards/margins": 0.05205933377146721, + "rewards/rejected": -0.17204266786575317, + "step": 195 + }, + { + "epoch": 0.12192846034214619, + "grad_norm": 0.4868512749671936, + "learning_rate": 4.5100000000000005e-05, + "log_odds_chosen": -0.11273391544818878, + "log_odds_ratio": -0.7624410390853882, + "logits/chosen": 0.162788987159729, + "logits/rejected": 0.038346655666828156, + "logps/chosen": -1.2809481620788574, + "logps/rejected": -1.187037706375122, + "loss": 2.6403, + "nll_loss": 2.5640499591827393, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.12809482216835022, + "rewards/margins": -0.009391050785779953, + "rewards/rejected": -0.11870376765727997, + "step": 196 + }, + { + "epoch": 0.12255054432348367, + "grad_norm": 0.4865812659263611, + "learning_rate": 4.5075e-05, + "log_odds_chosen": -0.08828195929527283, + "log_odds_ratio": -0.826860249042511, + "logits/chosen": 0.2504577338695526, + "logits/rejected": 0.20617207884788513, + "logps/chosen": -1.288416862487793, + "logps/rejected": -1.2700859308242798, + "loss": 3.1873, + "nll_loss": 3.1046361923217773, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12884169816970825, + "rewards/margins": -0.001833103597164154, + "rewards/rejected": -0.1270085871219635, + "step": 197 + }, + { + "epoch": 0.12317262830482115, + "grad_norm": 0.605975866317749, + "learning_rate": 4.5050000000000004e-05, + "log_odds_chosen": 0.6784669756889343, + "log_odds_ratio": -0.4767940044403076, + "logits/chosen": 0.038463614881038666, + "logits/rejected": 0.1368597447872162, + "logps/chosen": -1.371410608291626, + "logps/rejected": -1.9200092554092407, + "loss": 2.0605, + "nll_loss": 2.012791633605957, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13714107871055603, + "rewards/margins": 0.054859861731529236, + "rewards/rejected": -0.19200092554092407, + "step": 198 + }, + { + "epoch": 0.12379471228615863, + "grad_norm": 0.3905656635761261, + "learning_rate": 4.5025000000000003e-05, + "log_odds_chosen": 0.27600833773612976, + "log_odds_ratio": -0.5758750438690186, + "logits/chosen": 0.24896469712257385, + "logits/rejected": 0.1932651847600937, + "logps/chosen": -1.2160751819610596, + "logps/rejected": -1.3880665302276611, + "loss": 3.6016, + "nll_loss": 3.5440330505371094, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12160752713680267, + "rewards/margins": 0.01719912886619568, + "rewards/rejected": -0.13880664110183716, + "step": 199 + }, + { + "epoch": 0.12441679626749612, + "grad_norm": 0.9062671065330505, + "learning_rate": 4.5e-05, + "log_odds_chosen": 0.029399242252111435, + "log_odds_ratio": -0.7444265484809875, + "logits/chosen": 0.16195963323116302, + "logits/rejected": -0.002658102661371231, + "logps/chosen": -1.62650465965271, + "logps/rejected": -1.6063220500946045, + "loss": 3.017, + "nll_loss": 2.9425477981567383, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.162650465965271, + "rewards/margins": -0.0020182570442557335, + "rewards/rejected": -0.16063222289085388, + "step": 200 + }, + { + "epoch": 0.12503888024883358, + "grad_norm": 0.7090647220611572, + "learning_rate": 4.4975e-05, + "log_odds_chosen": -0.061332136392593384, + "log_odds_ratio": -0.7987865209579468, + "logits/chosen": 0.1813337355852127, + "logits/rejected": -0.05886400490999222, + "logps/chosen": -1.4757061004638672, + "logps/rejected": -1.383807897567749, + "loss": 3.2882, + "nll_loss": 3.2082958221435547, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14757061004638672, + "rewards/margins": -0.009189811535179615, + "rewards/rejected": -0.13838079571723938, + "step": 201 + }, + { + "epoch": 0.12566096423017106, + "grad_norm": 0.37017297744750977, + "learning_rate": 4.495e-05, + "log_odds_chosen": 0.6404499411582947, + "log_odds_ratio": -0.5468106269836426, + "logits/chosen": 0.2760981321334839, + "logits/rejected": 0.07182542979717255, + "logps/chosen": -1.1483453512191772, + "logps/rejected": -1.6027138233184814, + "loss": 3.5294, + "nll_loss": 3.47471284866333, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11483453214168549, + "rewards/margins": 0.04543685540556908, + "rewards/rejected": -0.16027137637138367, + "step": 202 + }, + { + "epoch": 0.12628304821150854, + "grad_norm": 0.3965308368206024, + "learning_rate": 4.4925e-05, + "log_odds_chosen": -0.044021353125572205, + "log_odds_ratio": -0.7529421448707581, + "logits/chosen": 0.12207438051700592, + "logits/rejected": 0.07484577596187592, + "logps/chosen": -1.0992047786712646, + "logps/rejected": -1.0575580596923828, + "loss": 2.8192, + "nll_loss": 2.7438852787017822, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.10992048680782318, + "rewards/margins": -0.004164676181972027, + "rewards/rejected": -0.10575580596923828, + "step": 203 + }, + { + "epoch": 0.12690513219284602, + "grad_norm": 0.3560815155506134, + "learning_rate": 4.49e-05, + "log_odds_chosen": 0.6957288384437561, + "log_odds_ratio": -0.4497440457344055, + "logits/chosen": 0.4506039619445801, + "logits/rejected": 0.015391908586025238, + "logps/chosen": -1.1910122632980347, + "logps/rejected": -1.6560842990875244, + "loss": 4.0035, + "nll_loss": 3.9584884643554688, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11910124123096466, + "rewards/margins": 0.04650719463825226, + "rewards/rejected": -0.16560842096805573, + "step": 204 + }, + { + "epoch": 0.12752721617418353, + "grad_norm": 0.6403685212135315, + "learning_rate": 4.4875e-05, + "log_odds_chosen": 0.2122831642627716, + "log_odds_ratio": -0.8761559724807739, + "logits/chosen": 0.014004663564264774, + "logits/rejected": -0.14548785984516144, + "logps/chosen": -1.689424753189087, + "logps/rejected": -1.6934268474578857, + "loss": 2.4198, + "nll_loss": 2.332214117050171, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16894248127937317, + "rewards/margins": 0.00040020793676376343, + "rewards/rejected": -0.16934269666671753, + "step": 205 + }, + { + "epoch": 0.128149300155521, + "grad_norm": 0.4000180661678314, + "learning_rate": 4.4850000000000006e-05, + "log_odds_chosen": 1.1184585094451904, + "log_odds_ratio": -0.37039902806282043, + "logits/chosen": 0.2634533941745758, + "logits/rejected": 0.16103744506835938, + "logps/chosen": -1.027080774307251, + "logps/rejected": -1.6481044292449951, + "loss": 3.1462, + "nll_loss": 3.1091270446777344, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10270807147026062, + "rewards/margins": 0.06210235506296158, + "rewards/rejected": -0.1648104339838028, + "step": 206 + }, + { + "epoch": 0.12877138413685849, + "grad_norm": 0.5040024518966675, + "learning_rate": 4.4825e-05, + "log_odds_chosen": 0.2705846130847931, + "log_odds_ratio": -0.59270179271698, + "logits/chosen": 0.15955251455307007, + "logits/rejected": -0.05338404327630997, + "logps/chosen": -1.2470922470092773, + "logps/rejected": -1.4498063325881958, + "loss": 2.6152, + "nll_loss": 2.555975914001465, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12470922619104385, + "rewards/margins": 0.020271409302949905, + "rewards/rejected": -0.14498063921928406, + "step": 207 + }, + { + "epoch": 0.12939346811819596, + "grad_norm": 0.594912052154541, + "learning_rate": 4.4800000000000005e-05, + "log_odds_chosen": 0.7347403764724731, + "log_odds_ratio": -0.44468581676483154, + "logits/chosen": 0.14959490299224854, + "logits/rejected": -0.027468346059322357, + "logps/chosen": -1.372584581375122, + "logps/rejected": -1.9416903257369995, + "loss": 2.7127, + "nll_loss": 2.6681857109069824, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13725847005844116, + "rewards/margins": 0.056910570710897446, + "rewards/rejected": -0.1941690444946289, + "step": 208 + }, + { + "epoch": 0.13001555209953344, + "grad_norm": 0.5162251591682434, + "learning_rate": 4.4775e-05, + "log_odds_chosen": 0.7629825472831726, + "log_odds_ratio": -0.48512592911720276, + "logits/chosen": 0.2162114381790161, + "logits/rejected": 0.039904482662677765, + "logps/chosen": -1.1675233840942383, + "logps/rejected": -1.7713046073913574, + "loss": 2.8182, + "nll_loss": 2.769730806350708, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11675234138965607, + "rewards/margins": 0.06037811189889908, + "rewards/rejected": -0.17713046073913574, + "step": 209 + }, + { + "epoch": 0.13063763608087092, + "grad_norm": 0.4357577860355377, + "learning_rate": 4.4750000000000004e-05, + "log_odds_chosen": 0.16853998601436615, + "log_odds_ratio": -0.6546946167945862, + "logits/chosen": 0.28680700063705444, + "logits/rejected": 0.13408944010734558, + "logps/chosen": -1.4316234588623047, + "logps/rejected": -1.5849097967147827, + "loss": 3.1143, + "nll_loss": 3.0488011837005615, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.143162339925766, + "rewards/margins": 0.015328639186918736, + "rewards/rejected": -0.15849095582962036, + "step": 210 + }, + { + "epoch": 0.1312597200622084, + "grad_norm": 0.48053115606307983, + "learning_rate": 4.4725e-05, + "log_odds_chosen": 0.6203852891921997, + "log_odds_ratio": -0.47376835346221924, + "logits/chosen": 0.30184412002563477, + "logits/rejected": 0.14279696345329285, + "logps/chosen": -1.20229971408844, + "logps/rejected": -1.6811708211898804, + "loss": 3.1291, + "nll_loss": 3.0817553997039795, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12022997438907623, + "rewards/margins": 0.04788711294531822, + "rewards/rejected": -0.16811709105968475, + "step": 211 + }, + { + "epoch": 0.13188180404354588, + "grad_norm": 0.3863763213157654, + "learning_rate": 4.47e-05, + "log_odds_chosen": 0.23602965474128723, + "log_odds_ratio": -0.6337170004844666, + "logits/chosen": 0.14647379517555237, + "logits/rejected": 0.13201558589935303, + "logps/chosen": -1.5172404050827026, + "logps/rejected": -1.723448634147644, + "loss": 2.9906, + "nll_loss": 2.9272663593292236, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15172404050827026, + "rewards/margins": 0.02062082849442959, + "rewards/rejected": -0.1723448783159256, + "step": 212 + }, + { + "epoch": 0.13250388802488336, + "grad_norm": 0.5771374106407166, + "learning_rate": 4.4675e-05, + "log_odds_chosen": 0.6309947371482849, + "log_odds_ratio": -0.5399520397186279, + "logits/chosen": 0.12432387471199036, + "logits/rejected": 0.20462198555469513, + "logps/chosen": -1.5274455547332764, + "logps/rejected": -2.06624436378479, + "loss": 2.5767, + "nll_loss": 2.5227041244506836, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15274454653263092, + "rewards/margins": 0.053879883140325546, + "rewards/rejected": -0.20662443339824677, + "step": 213 + }, + { + "epoch": 0.13312597200622084, + "grad_norm": 0.41249603033065796, + "learning_rate": 4.465e-05, + "log_odds_chosen": -0.003160417079925537, + "log_odds_ratio": -0.777141273021698, + "logits/chosen": 0.14705267548561096, + "logits/rejected": -0.06464418023824692, + "logps/chosen": -1.5321584939956665, + "logps/rejected": -1.5345534086227417, + "loss": 3.1204, + "nll_loss": 3.042703866958618, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.15321585536003113, + "rewards/margins": 0.00023948121815919876, + "rewards/rejected": -0.15345533192157745, + "step": 214 + }, + { + "epoch": 0.13374805598755832, + "grad_norm": 0.4820147752761841, + "learning_rate": 4.4625e-05, + "log_odds_chosen": 0.2911366820335388, + "log_odds_ratio": -0.5818198323249817, + "logits/chosen": 0.2889520227909088, + "logits/rejected": 0.08205986022949219, + "logps/chosen": -1.4115300178527832, + "logps/rejected": -1.6441384553909302, + "loss": 2.9628, + "nll_loss": 2.904639959335327, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1411530077457428, + "rewards/margins": 0.02326083928346634, + "rewards/rejected": -0.16441385447978973, + "step": 215 + }, + { + "epoch": 0.1343701399688958, + "grad_norm": 0.41194218397140503, + "learning_rate": 4.46e-05, + "log_odds_chosen": 0.3945164382457733, + "log_odds_ratio": -0.6375629305839539, + "logits/chosen": 0.421595960855484, + "logits/rejected": 0.30017945170402527, + "logps/chosen": -1.3539952039718628, + "logps/rejected": -1.5963020324707031, + "loss": 3.5731, + "nll_loss": 3.509347915649414, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13539952039718628, + "rewards/margins": 0.024230685085058212, + "rewards/rejected": -0.1596302092075348, + "step": 216 + }, + { + "epoch": 0.13499222395023328, + "grad_norm": 0.503413200378418, + "learning_rate": 4.4575e-05, + "log_odds_chosen": 0.5858572721481323, + "log_odds_ratio": -0.5386984944343567, + "logits/chosen": 0.19849129021167755, + "logits/rejected": 0.03486599028110504, + "logps/chosen": -1.2306039333343506, + "logps/rejected": -1.630487322807312, + "loss": 3.1597, + "nll_loss": 3.105804443359375, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12306039035320282, + "rewards/margins": 0.03998832777142525, + "rewards/rejected": -0.16304872930049896, + "step": 217 + }, + { + "epoch": 0.13561430793157075, + "grad_norm": 0.5384315848350525, + "learning_rate": 4.4550000000000005e-05, + "log_odds_chosen": 1.3037192821502686, + "log_odds_ratio": -0.36916741728782654, + "logits/chosen": 0.14214058220386505, + "logits/rejected": 0.01107453927397728, + "logps/chosen": -0.9095080494880676, + "logps/rejected": -1.8086912631988525, + "loss": 2.5382, + "nll_loss": 2.5013184547424316, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09095080196857452, + "rewards/margins": 0.08991833031177521, + "rewards/rejected": -0.18086913228034973, + "step": 218 + }, + { + "epoch": 0.13623639191290823, + "grad_norm": 0.46685314178466797, + "learning_rate": 4.4525e-05, + "log_odds_chosen": 0.5023396611213684, + "log_odds_ratio": -0.5046614408493042, + "logits/chosen": 0.2844086289405823, + "logits/rejected": 0.1384527087211609, + "logps/chosen": -1.252253532409668, + "logps/rejected": -1.6022236347198486, + "loss": 2.6639, + "nll_loss": 2.613450527191162, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12522533535957336, + "rewards/margins": 0.034997016191482544, + "rewards/rejected": -0.1602223664522171, + "step": 219 + }, + { + "epoch": 0.1368584758942457, + "grad_norm": 0.37732696533203125, + "learning_rate": 4.4500000000000004e-05, + "log_odds_chosen": 0.475115031003952, + "log_odds_ratio": -0.5874269008636475, + "logits/chosen": 0.18952935934066772, + "logits/rejected": -0.05404677242040634, + "logps/chosen": -1.0435460805892944, + "logps/rejected": -1.356130599975586, + "loss": 3.2866, + "nll_loss": 3.227907419204712, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1043546050786972, + "rewards/margins": 0.03125845640897751, + "rewards/rejected": -0.13561305403709412, + "step": 220 + }, + { + "epoch": 0.1374805598755832, + "grad_norm": 0.39852970838546753, + "learning_rate": 4.4475e-05, + "log_odds_chosen": 0.40951022505760193, + "log_odds_ratio": -0.6001676917076111, + "logits/chosen": 0.1417674422264099, + "logits/rejected": -0.023162055760622025, + "logps/chosen": -1.1433157920837402, + "logps/rejected": -1.3727480173110962, + "loss": 2.8523, + "nll_loss": 2.7923316955566406, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11433158069849014, + "rewards/margins": 0.022943219169974327, + "rewards/rejected": -0.13727480173110962, + "step": 221 + }, + { + "epoch": 0.13810264385692067, + "grad_norm": 0.4464816153049469, + "learning_rate": 4.445e-05, + "log_odds_chosen": 0.983132541179657, + "log_odds_ratio": -0.3711632490158081, + "logits/chosen": 0.22909504175186157, + "logits/rejected": -0.061455175280570984, + "logps/chosen": -1.0717201232910156, + "logps/rejected": -1.7862846851348877, + "loss": 3.3108, + "nll_loss": 3.2737152576446533, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10717201232910156, + "rewards/margins": 0.07145645469427109, + "rewards/rejected": -0.17862847447395325, + "step": 222 + }, + { + "epoch": 0.13872472783825818, + "grad_norm": 0.5403879880905151, + "learning_rate": 4.4425e-05, + "log_odds_chosen": 0.5938290357589722, + "log_odds_ratio": -0.4913523495197296, + "logits/chosen": 0.09688084572553635, + "logits/rejected": -0.05809571593999863, + "logps/chosen": -1.2423458099365234, + "logps/rejected": -1.6755822896957397, + "loss": 2.842, + "nll_loss": 2.7928619384765625, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12423457205295563, + "rewards/margins": 0.04332365095615387, + "rewards/rejected": -0.1675582379102707, + "step": 223 + }, + { + "epoch": 0.13934681181959566, + "grad_norm": 0.5137689113616943, + "learning_rate": 4.44e-05, + "log_odds_chosen": 0.9574803113937378, + "log_odds_ratio": -0.6414791345596313, + "logits/chosen": 0.41440945863723755, + "logits/rejected": 0.10038256645202637, + "logps/chosen": -1.3024613857269287, + "logps/rejected": -1.8441886901855469, + "loss": 4.0097, + "nll_loss": 3.94557523727417, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1302461475133896, + "rewards/margins": 0.05417274311184883, + "rewards/rejected": -0.18441888689994812, + "step": 224 + }, + { + "epoch": 0.13996889580093314, + "grad_norm": 0.34440910816192627, + "learning_rate": 4.4375e-05, + "log_odds_chosen": 0.7401940822601318, + "log_odds_ratio": -0.5135765671730042, + "logits/chosen": 0.34442248940467834, + "logits/rejected": -0.011198656633496284, + "logps/chosen": -1.2954169511795044, + "logps/rejected": -1.7026612758636475, + "loss": 3.9705, + "nll_loss": 3.9191346168518066, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12954169511795044, + "rewards/margins": 0.040724433958530426, + "rewards/rejected": -0.17026613652706146, + "step": 225 + }, + { + "epoch": 0.14059097978227061, + "grad_norm": 0.4693191349506378, + "learning_rate": 4.435e-05, + "log_odds_chosen": 0.46523332595825195, + "log_odds_ratio": -0.5510789752006531, + "logits/chosen": 0.09288059175014496, + "logits/rejected": -0.01915971003472805, + "logps/chosen": -1.0227751731872559, + "logps/rejected": -1.2773354053497314, + "loss": 2.7217, + "nll_loss": 2.6666271686553955, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10227751731872559, + "rewards/margins": 0.0254560187458992, + "rewards/rejected": -0.1277335286140442, + "step": 226 + }, + { + "epoch": 0.1412130637636081, + "grad_norm": 0.3790217936038971, + "learning_rate": 4.4325e-05, + "log_odds_chosen": 0.7664673924446106, + "log_odds_ratio": -0.4955465793609619, + "logits/chosen": 0.2821044921875, + "logits/rejected": 0.1695287972688675, + "logps/chosen": -1.0519059896469116, + "logps/rejected": -1.5946249961853027, + "loss": 3.3048, + "nll_loss": 3.2552666664123535, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10519059747457504, + "rewards/margins": 0.054271895438432693, + "rewards/rejected": -0.15946249663829803, + "step": 227 + }, + { + "epoch": 0.14183514774494557, + "grad_norm": 0.43475788831710815, + "learning_rate": 4.43e-05, + "log_odds_chosen": 1.3139894008636475, + "log_odds_ratio": -0.3370991349220276, + "logits/chosen": 0.12189202010631561, + "logits/rejected": -0.0020540207624435425, + "logps/chosen": -1.0322346687316895, + "logps/rejected": -2.0027456283569336, + "loss": 3.0615, + "nll_loss": 3.0277514457702637, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10322347283363342, + "rewards/margins": 0.09705111384391785, + "rewards/rejected": -0.20027457177639008, + "step": 228 + }, + { + "epoch": 0.14245723172628305, + "grad_norm": 0.4523204565048218, + "learning_rate": 4.4275e-05, + "log_odds_chosen": 0.5393589735031128, + "log_odds_ratio": -0.5129015445709229, + "logits/chosen": 0.3066309988498688, + "logits/rejected": 0.01438068225979805, + "logps/chosen": -1.1355011463165283, + "logps/rejected": -1.4902321100234985, + "loss": 3.3797, + "nll_loss": 3.328399181365967, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1135501116514206, + "rewards/margins": 0.035473089665174484, + "rewards/rejected": -0.14902320504188538, + "step": 229 + }, + { + "epoch": 0.14307931570762053, + "grad_norm": 0.45688357949256897, + "learning_rate": 4.4250000000000005e-05, + "log_odds_chosen": 1.0163792371749878, + "log_odds_ratio": -0.42969611287117004, + "logits/chosen": 0.2794167995452881, + "logits/rejected": 0.0069199977442622185, + "logps/chosen": -1.1130561828613281, + "logps/rejected": -1.8443264961242676, + "loss": 3.1188, + "nll_loss": 3.0758509635925293, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11130562424659729, + "rewards/margins": 0.07312701642513275, + "rewards/rejected": -0.18443265557289124, + "step": 230 + }, + { + "epoch": 0.143701399688958, + "grad_norm": 0.46562471985816956, + "learning_rate": 4.4225e-05, + "log_odds_chosen": 0.051547348499298096, + "log_odds_ratio": -0.6991626024246216, + "logits/chosen": 0.3503286838531494, + "logits/rejected": 0.16541802883148193, + "logps/chosen": -1.2530078887939453, + "logps/rejected": -1.2871830463409424, + "loss": 3.1336, + "nll_loss": 3.0636978149414062, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.12530077993869781, + "rewards/margins": 0.003417517989873886, + "rewards/rejected": -0.128718301653862, + "step": 231 + }, + { + "epoch": 0.1443234836702955, + "grad_norm": 0.5583508610725403, + "learning_rate": 4.4200000000000004e-05, + "log_odds_chosen": 0.4175459146499634, + "log_odds_ratio": -0.5737831592559814, + "logits/chosen": 0.4354173243045807, + "logits/rejected": 0.12491598725318909, + "logps/chosen": -1.5473692417144775, + "logps/rejected": -1.9231562614440918, + "loss": 3.3368, + "nll_loss": 3.279399871826172, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15473692119121552, + "rewards/margins": 0.03757869452238083, + "rewards/rejected": -0.19231562316417694, + "step": 232 + }, + { + "epoch": 0.14494556765163297, + "grad_norm": 0.5220412611961365, + "learning_rate": 4.4174999999999996e-05, + "log_odds_chosen": 0.49191123247146606, + "log_odds_ratio": -0.6103449463844299, + "logits/chosen": 0.17081604897975922, + "logits/rejected": 0.05159229040145874, + "logps/chosen": -1.402343988418579, + "logps/rejected": -1.7350009679794312, + "loss": 3.3047, + "nll_loss": 3.2436368465423584, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14023439586162567, + "rewards/margins": 0.03326570987701416, + "rewards/rejected": -0.17350010573863983, + "step": 233 + }, + { + "epoch": 0.14556765163297045, + "grad_norm": 0.7865017652511597, + "learning_rate": 4.415e-05, + "log_odds_chosen": -0.3338541090488434, + "log_odds_ratio": -1.0038307905197144, + "logits/chosen": 0.20039094984531403, + "logits/rejected": 0.13230621814727783, + "logps/chosen": -1.9984074831008911, + "logps/rejected": -1.6774605512619019, + "loss": 2.6104, + "nll_loss": 2.5099916458129883, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1998407542705536, + "rewards/margins": -0.03209469094872475, + "rewards/rejected": -0.16774605214595795, + "step": 234 + }, + { + "epoch": 0.14618973561430793, + "grad_norm": 0.5441064834594727, + "learning_rate": 4.4125e-05, + "log_odds_chosen": 0.19742441177368164, + "log_odds_ratio": -0.6543236970901489, + "logits/chosen": 0.2758321762084961, + "logits/rejected": 0.2791656255722046, + "logps/chosen": -1.3114479780197144, + "logps/rejected": -1.4452178478240967, + "loss": 2.6863, + "nll_loss": 2.620828866958618, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.13114479184150696, + "rewards/margins": 0.013376971706748009, + "rewards/rejected": -0.1445217728614807, + "step": 235 + }, + { + "epoch": 0.1468118195956454, + "grad_norm": 0.4621172845363617, + "learning_rate": 4.41e-05, + "log_odds_chosen": 0.35999900102615356, + "log_odds_ratio": -0.6286050081253052, + "logits/chosen": 0.3370401859283447, + "logits/rejected": 0.1895410120487213, + "logps/chosen": -1.0965385437011719, + "logps/rejected": -1.2593486309051514, + "loss": 3.3825, + "nll_loss": 3.3196167945861816, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10965386033058167, + "rewards/margins": 0.016280999407172203, + "rewards/rejected": -0.12593486905097961, + "step": 236 + }, + { + "epoch": 0.14743390357698288, + "grad_norm": 0.4633600115776062, + "learning_rate": 4.4075e-05, + "log_odds_chosen": 0.15592439472675323, + "log_odds_ratio": -0.6589779853820801, + "logits/chosen": 0.24592751264572144, + "logits/rejected": 0.05784950405359268, + "logps/chosen": -1.350083827972412, + "logps/rejected": -1.4974408149719238, + "loss": 3.17, + "nll_loss": 3.1041297912597656, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13500836491584778, + "rewards/margins": 0.014735697768628597, + "rewards/rejected": -0.14974406361579895, + "step": 237 + }, + { + "epoch": 0.14805598755832036, + "grad_norm": 0.5822882652282715, + "learning_rate": 4.405e-05, + "log_odds_chosen": 0.5478257536888123, + "log_odds_ratio": -0.6282358169555664, + "logits/chosen": 0.28536713123321533, + "logits/rejected": 0.10193793475627899, + "logps/chosen": -1.4900511503219604, + "logps/rejected": -1.923351526260376, + "loss": 3.1027, + "nll_loss": 3.039842128753662, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14900511503219604, + "rewards/margins": 0.043330058455467224, + "rewards/rejected": -0.19233515858650208, + "step": 238 + }, + { + "epoch": 0.14867807153965784, + "grad_norm": 0.4006459712982178, + "learning_rate": 4.4025e-05, + "log_odds_chosen": 0.30393242835998535, + "log_odds_ratio": -0.5825948715209961, + "logits/chosen": 0.30995607376098633, + "logits/rejected": 0.09291841834783554, + "logps/chosen": -1.4452520608901978, + "logps/rejected": -1.6686973571777344, + "loss": 3.2763, + "nll_loss": 3.218071937561035, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1445252150297165, + "rewards/margins": 0.022344522178173065, + "rewards/rejected": -0.16686972975730896, + "step": 239 + }, + { + "epoch": 0.14930015552099535, + "grad_norm": 0.37317949533462524, + "learning_rate": 4.4000000000000006e-05, + "log_odds_chosen": 1.5197196006774902, + "log_odds_ratio": -0.4212367534637451, + "logits/chosen": 0.3777592182159424, + "logits/rejected": 0.2529478073120117, + "logps/chosen": -0.9898640513420105, + "logps/rejected": -2.145714282989502, + "loss": 3.4547, + "nll_loss": 3.4126036167144775, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09898640215396881, + "rewards/margins": 0.11558501422405243, + "rewards/rejected": -0.21457141637802124, + "step": 240 + }, + { + "epoch": 0.14992223950233283, + "grad_norm": 0.5220414996147156, + "learning_rate": 4.3975e-05, + "log_odds_chosen": 0.9724363684654236, + "log_odds_ratio": -0.39547187089920044, + "logits/chosen": 0.18093883991241455, + "logits/rejected": 0.09247509390115738, + "logps/chosen": -1.3090280294418335, + "logps/rejected": -1.9929441213607788, + "loss": 2.5084, + "nll_loss": 2.4688820838928223, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13090281188488007, + "rewards/margins": 0.06839160621166229, + "rewards/rejected": -0.19929441809654236, + "step": 241 + }, + { + "epoch": 0.1505443234836703, + "grad_norm": 0.41178998351097107, + "learning_rate": 4.3950000000000004e-05, + "log_odds_chosen": 0.03777070343494415, + "log_odds_ratio": -0.7433863282203674, + "logits/chosen": 0.18827003240585327, + "logits/rejected": 0.17734594643115997, + "logps/chosen": -1.3193129301071167, + "logps/rejected": -1.3846216201782227, + "loss": 2.8552, + "nll_loss": 2.7808141708374023, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13193130493164062, + "rewards/margins": 0.006530864164233208, + "rewards/rejected": -0.1384621560573578, + "step": 242 + }, + { + "epoch": 0.15116640746500778, + "grad_norm": 0.41735777258872986, + "learning_rate": 4.3925e-05, + "log_odds_chosen": 0.5820766687393188, + "log_odds_ratio": -0.4971098303794861, + "logits/chosen": 0.3844646215438843, + "logits/rejected": 0.1357293277978897, + "logps/chosen": -1.285773515701294, + "logps/rejected": -1.6937378644943237, + "loss": 3.6206, + "nll_loss": 3.5709385871887207, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1285773366689682, + "rewards/margins": 0.0407964326441288, + "rewards/rejected": -0.1693737804889679, + "step": 243 + }, + { + "epoch": 0.15178849144634526, + "grad_norm": 0.4252638816833496, + "learning_rate": 4.39e-05, + "log_odds_chosen": 0.9820137023925781, + "log_odds_ratio": -0.5139923095703125, + "logits/chosen": 0.09503781795501709, + "logits/rejected": 0.09257042407989502, + "logps/chosen": -1.1998246908187866, + "logps/rejected": -1.9771127700805664, + "loss": 2.4225, + "nll_loss": 2.371094226837158, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11998246610164642, + "rewards/margins": 0.07772879302501678, + "rewards/rejected": -0.1977112740278244, + "step": 244 + }, + { + "epoch": 0.15241057542768274, + "grad_norm": 0.35691797733306885, + "learning_rate": 4.3875e-05, + "log_odds_chosen": 0.15277954936027527, + "log_odds_ratio": -0.6356155872344971, + "logits/chosen": 0.1442718803882599, + "logits/rejected": 0.08599300682544708, + "logps/chosen": -1.3047629594802856, + "logps/rejected": -1.41819167137146, + "loss": 2.9185, + "nll_loss": 2.8548996448516846, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13047629594802856, + "rewards/margins": 0.011342879384756088, + "rewards/rejected": -0.14181917905807495, + "step": 245 + }, + { + "epoch": 0.15303265940902022, + "grad_norm": 0.3984750509262085, + "learning_rate": 4.385e-05, + "log_odds_chosen": 1.2786288261413574, + "log_odds_ratio": -0.36163750290870667, + "logits/chosen": 0.1432109922170639, + "logits/rejected": 0.06168469786643982, + "logps/chosen": -0.9494646191596985, + "logps/rejected": -1.8683538436889648, + "loss": 3.0516, + "nll_loss": 3.0154216289520264, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0949464663863182, + "rewards/margins": 0.0918889045715332, + "rewards/rejected": -0.1868353635072708, + "step": 246 + }, + { + "epoch": 0.1536547433903577, + "grad_norm": 0.35005396604537964, + "learning_rate": 4.3825e-05, + "log_odds_chosen": 0.36924123764038086, + "log_odds_ratio": -0.6463004946708679, + "logits/chosen": 0.08100152760744095, + "logits/rejected": -0.09407303482294083, + "logps/chosen": -1.0215667486190796, + "logps/rejected": -1.2288739681243896, + "loss": 2.9297, + "nll_loss": 2.865115165710449, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10215667635202408, + "rewards/margins": 0.020730726420879364, + "rewards/rejected": -0.12288740277290344, + "step": 247 + }, + { + "epoch": 0.15427682737169518, + "grad_norm": 0.5391296744346619, + "learning_rate": 4.38e-05, + "log_odds_chosen": 0.5213131904602051, + "log_odds_ratio": -0.5025854110717773, + "logits/chosen": 0.06299649924039841, + "logits/rejected": -0.09286680817604065, + "logps/chosen": -1.111344337463379, + "logps/rejected": -1.5201202630996704, + "loss": 3.0356, + "nll_loss": 2.9853503704071045, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11113443225622177, + "rewards/margins": 0.04087759926915169, + "rewards/rejected": -0.15201203525066376, + "step": 248 + }, + { + "epoch": 0.15489891135303266, + "grad_norm": 0.454863578081131, + "learning_rate": 4.3775e-05, + "log_odds_chosen": 1.0419408082962036, + "log_odds_ratio": -0.47779956459999084, + "logits/chosen": 0.06458957493305206, + "logits/rejected": -0.05411160737276077, + "logps/chosen": -1.1926398277282715, + "logps/rejected": -1.8617669343948364, + "loss": 3.2576, + "nll_loss": 3.2098097801208496, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11926397681236267, + "rewards/margins": 0.0669127106666565, + "rewards/rejected": -0.18617668747901917, + "step": 249 + }, + { + "epoch": 0.15552099533437014, + "grad_norm": 0.5558252334594727, + "learning_rate": 4.375e-05, + "log_odds_chosen": 1.1860463619232178, + "log_odds_ratio": -0.41251182556152344, + "logits/chosen": -0.08973684906959534, + "logits/rejected": -0.16550233960151672, + "logps/chosen": -1.0720642805099487, + "logps/rejected": -1.8928790092468262, + "loss": 2.5241, + "nll_loss": 2.4828810691833496, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10720643401145935, + "rewards/margins": 0.08208145946264267, + "rewards/rejected": -0.18928790092468262, + "step": 250 + }, + { + "epoch": 0.15614307931570762, + "grad_norm": 0.46578970551490784, + "learning_rate": 4.3725000000000006e-05, + "log_odds_chosen": 0.8737031817436218, + "log_odds_ratio": -0.45753180980682373, + "logits/chosen": 0.10488224774599075, + "logits/rejected": 0.002302175387740135, + "logps/chosen": -1.141048550605774, + "logps/rejected": -1.7917754650115967, + "loss": 2.7639, + "nll_loss": 2.718146562576294, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11410486698150635, + "rewards/margins": 0.0650726854801178, + "rewards/rejected": -0.17917755246162415, + "step": 251 + }, + { + "epoch": 0.1567651632970451, + "grad_norm": 0.5264093279838562, + "learning_rate": 4.3700000000000005e-05, + "log_odds_chosen": 1.2083812952041626, + "log_odds_ratio": -0.46954599022865295, + "logits/chosen": 0.10020960867404938, + "logits/rejected": 0.05089284107089043, + "logps/chosen": -1.2586607933044434, + "logps/rejected": -2.2547106742858887, + "loss": 2.6152, + "nll_loss": 2.568207025527954, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12586607038974762, + "rewards/margins": 0.09960497915744781, + "rewards/rejected": -0.22547104954719543, + "step": 252 + }, + { + "epoch": 0.15738724727838257, + "grad_norm": 0.39940425753593445, + "learning_rate": 4.3675000000000005e-05, + "log_odds_chosen": 0.8237320780754089, + "log_odds_ratio": -0.42615777254104614, + "logits/chosen": 0.12673884630203247, + "logits/rejected": 0.04520142823457718, + "logps/chosen": -1.2019363641738892, + "logps/rejected": -1.8468737602233887, + "loss": 3.0343, + "nll_loss": 2.991687536239624, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12019363045692444, + "rewards/margins": 0.06449373066425323, + "rewards/rejected": -0.18468736112117767, + "step": 253 + }, + { + "epoch": 0.15800933125972005, + "grad_norm": 0.36626678705215454, + "learning_rate": 4.3650000000000004e-05, + "log_odds_chosen": 0.9984500408172607, + "log_odds_ratio": -0.3665887117385864, + "logits/chosen": 0.18472406268119812, + "logits/rejected": 0.037876978516578674, + "logps/chosen": -1.0484715700149536, + "logps/rejected": -1.8369355201721191, + "loss": 3.5121, + "nll_loss": 3.4753966331481934, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10484716296195984, + "rewards/margins": 0.07884638756513596, + "rewards/rejected": -0.1836935579776764, + "step": 254 + }, + { + "epoch": 0.15863141524105753, + "grad_norm": 0.4521494507789612, + "learning_rate": 4.3625e-05, + "log_odds_chosen": 1.2302688360214233, + "log_odds_ratio": -0.3208959698677063, + "logits/chosen": 0.1981630027294159, + "logits/rejected": -0.019413426518440247, + "logps/chosen": -0.9359973669052124, + "logps/rejected": -1.7934815883636475, + "loss": 3.3395, + "nll_loss": 3.3074443340301514, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09359973669052124, + "rewards/margins": 0.08574841916561127, + "rewards/rejected": -0.1793481707572937, + "step": 255 + }, + { + "epoch": 0.159253499222395, + "grad_norm": 0.4002397060394287, + "learning_rate": 4.36e-05, + "log_odds_chosen": 0.2759625315666199, + "log_odds_ratio": -0.5995438098907471, + "logits/chosen": 0.0834539383649826, + "logits/rejected": -0.03273439407348633, + "logps/chosen": -1.3060052394866943, + "logps/rejected": -1.5258723497390747, + "loss": 3.0473, + "nll_loss": 2.9873416423797607, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13060052692890167, + "rewards/margins": 0.021986715495586395, + "rewards/rejected": -0.15258723497390747, + "step": 256 + }, + { + "epoch": 0.1598755832037325, + "grad_norm": 0.6363208293914795, + "learning_rate": 4.3575e-05, + "log_odds_chosen": 0.5167827606201172, + "log_odds_ratio": -0.5308623909950256, + "logits/chosen": 0.23719561100006104, + "logits/rejected": 0.009174295701086521, + "logps/chosen": -1.3490158319473267, + "logps/rejected": -1.7873324155807495, + "loss": 3.2075, + "nll_loss": 3.1543946266174316, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13490158319473267, + "rewards/margins": 0.04383165389299393, + "rewards/rejected": -0.1787332445383072, + "step": 257 + }, + { + "epoch": 0.16049766718507, + "grad_norm": 0.4826381504535675, + "learning_rate": 4.355e-05, + "log_odds_chosen": 0.10458363592624664, + "log_odds_ratio": -0.709639310836792, + "logits/chosen": 0.22416506707668304, + "logits/rejected": 0.1090393140912056, + "logps/chosen": -1.410514235496521, + "logps/rejected": -1.4754109382629395, + "loss": 3.4102, + "nll_loss": 3.339193820953369, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.14105141162872314, + "rewards/margins": 0.006489668972790241, + "rewards/rejected": -0.1475410908460617, + "step": 258 + }, + { + "epoch": 0.16111975116640748, + "grad_norm": 0.41199758648872375, + "learning_rate": 4.352500000000001e-05, + "log_odds_chosen": 1.1602838039398193, + "log_odds_ratio": -0.3441213369369507, + "logits/chosen": 0.12007400393486023, + "logits/rejected": 0.005203016102313995, + "logps/chosen": -0.8913068175315857, + "logps/rejected": -1.666398286819458, + "loss": 3.228, + "nll_loss": 3.1936206817626953, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08913067728281021, + "rewards/margins": 0.07750913500785828, + "rewards/rejected": -0.16663981974124908, + "step": 259 + }, + { + "epoch": 0.16174183514774496, + "grad_norm": 0.5916374325752258, + "learning_rate": 4.35e-05, + "log_odds_chosen": 0.4282585680484772, + "log_odds_ratio": -0.5398960709571838, + "logits/chosen": 0.22822090983390808, + "logits/rejected": 0.1819199174642563, + "logps/chosen": -1.363776445388794, + "logps/rejected": -1.7082788944244385, + "loss": 2.7722, + "nll_loss": 2.7182321548461914, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13637763261795044, + "rewards/margins": 0.03445024788379669, + "rewards/rejected": -0.17082789540290833, + "step": 260 + }, + { + "epoch": 0.16236391912908243, + "grad_norm": 0.5163818001747131, + "learning_rate": 4.3475000000000006e-05, + "log_odds_chosen": 0.7817611694335938, + "log_odds_ratio": -0.419194757938385, + "logits/chosen": 0.11984409391880035, + "logits/rejected": -0.05257415026426315, + "logps/chosen": -1.1561753749847412, + "logps/rejected": -1.754962682723999, + "loss": 2.795, + "nll_loss": 2.7531259059906006, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1156175285577774, + "rewards/margins": 0.05987873673439026, + "rewards/rejected": -0.17549628019332886, + "step": 261 + }, + { + "epoch": 0.1629860031104199, + "grad_norm": 0.3384513854980469, + "learning_rate": 4.345e-05, + "log_odds_chosen": 0.48024481534957886, + "log_odds_ratio": -0.5129582285881042, + "logits/chosen": 0.38080763816833496, + "logits/rejected": 0.12893861532211304, + "logps/chosen": -1.1857705116271973, + "logps/rejected": -1.510347604751587, + "loss": 3.62, + "nll_loss": 3.5687360763549805, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11857704818248749, + "rewards/margins": 0.032457705587148666, + "rewards/rejected": -0.15103475749492645, + "step": 262 + }, + { + "epoch": 0.1636080870917574, + "grad_norm": 0.3694182336330414, + "learning_rate": 4.3425000000000005e-05, + "log_odds_chosen": 1.5465469360351562, + "log_odds_ratio": -0.2594394385814667, + "logits/chosen": 0.19702796638011932, + "logits/rejected": 0.1825985312461853, + "logps/chosen": -1.0576568841934204, + "logps/rejected": -2.280040740966797, + "loss": 2.7614, + "nll_loss": 2.735440492630005, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1057656854391098, + "rewards/margins": 0.12223838269710541, + "rewards/rejected": -0.2280040681362152, + "step": 263 + }, + { + "epoch": 0.16423017107309487, + "grad_norm": 0.4737184941768646, + "learning_rate": 4.3400000000000005e-05, + "log_odds_chosen": 1.012281060218811, + "log_odds_ratio": -0.45542672276496887, + "logits/chosen": 0.18417447805404663, + "logits/rejected": 0.2393220067024231, + "logps/chosen": -1.3665863275527954, + "logps/rejected": -2.137876272201538, + "loss": 3.0061, + "nll_loss": 2.9605910778045654, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13665863871574402, + "rewards/margins": 0.07712897658348083, + "rewards/rejected": -0.21378763020038605, + "step": 264 + }, + { + "epoch": 0.16485225505443235, + "grad_norm": 0.5989867448806763, + "learning_rate": 4.3375000000000004e-05, + "log_odds_chosen": 0.47888749837875366, + "log_odds_ratio": -0.5475578904151917, + "logits/chosen": 0.34955573081970215, + "logits/rejected": -0.01235947385430336, + "logps/chosen": -1.1995731592178345, + "logps/rejected": -1.5407181978225708, + "loss": 3.5055, + "nll_loss": 3.4507837295532227, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1199573203921318, + "rewards/margins": 0.034114498645067215, + "rewards/rejected": -0.15407180786132812, + "step": 265 + }, + { + "epoch": 0.16547433903576983, + "grad_norm": 0.45694050192832947, + "learning_rate": 4.335e-05, + "log_odds_chosen": 1.0779105424880981, + "log_odds_ratio": -0.4133705198764801, + "logits/chosen": 0.31415730714797974, + "logits/rejected": 0.17435882985591888, + "logps/chosen": -1.2905559539794922, + "logps/rejected": -2.1517629623413086, + "loss": 3.4923, + "nll_loss": 3.4509427547454834, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12905558943748474, + "rewards/margins": 0.08612069487571716, + "rewards/rejected": -0.2151763141155243, + "step": 266 + }, + { + "epoch": 0.1660964230171073, + "grad_norm": 0.5878811478614807, + "learning_rate": 4.3325e-05, + "log_odds_chosen": 0.61550372838974, + "log_odds_ratio": -0.5280717015266418, + "logits/chosen": 0.13362732529640198, + "logits/rejected": -0.045731253921985626, + "logps/chosen": -1.3563334941864014, + "logps/rejected": -1.7919440269470215, + "loss": 2.8887, + "nll_loss": 2.8359363079071045, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13563336431980133, + "rewards/margins": 0.04356103017926216, + "rewards/rejected": -0.1791943907737732, + "step": 267 + }, + { + "epoch": 0.1667185069984448, + "grad_norm": 0.32133936882019043, + "learning_rate": 4.33e-05, + "log_odds_chosen": 0.21047142148017883, + "log_odds_ratio": -0.7247399687767029, + "logits/chosen": 0.40043243765830994, + "logits/rejected": 0.05317821353673935, + "logps/chosen": -1.4496357440948486, + "logps/rejected": -1.6929826736450195, + "loss": 3.9373, + "nll_loss": 3.8648571968078613, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1449635773897171, + "rewards/margins": 0.02433469332754612, + "rewards/rejected": -0.16929826140403748, + "step": 268 + }, + { + "epoch": 0.16734059097978227, + "grad_norm": 0.39708441495895386, + "learning_rate": 4.3275e-05, + "log_odds_chosen": 0.2173278033733368, + "log_odds_ratio": -0.659453272819519, + "logits/chosen": 0.16504698991775513, + "logits/rejected": 0.015571564435958862, + "logps/chosen": -1.2936235666275024, + "logps/rejected": -1.4728864431381226, + "loss": 3.2912, + "nll_loss": 3.225303888320923, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.12936235964298248, + "rewards/margins": 0.0179262924939394, + "rewards/rejected": -0.14728865027427673, + "step": 269 + }, + { + "epoch": 0.16796267496111975, + "grad_norm": 0.4461729824542999, + "learning_rate": 4.325e-05, + "log_odds_chosen": 0.7989851236343384, + "log_odds_ratio": -0.5427728295326233, + "logits/chosen": -0.0695042535662651, + "logits/rejected": -0.15135838091373444, + "logps/chosen": -1.2412259578704834, + "logps/rejected": -1.7757339477539062, + "loss": 2.8043, + "nll_loss": 2.7499935626983643, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12412260472774506, + "rewards/margins": 0.053450800478458405, + "rewards/rejected": -0.17757341265678406, + "step": 270 + }, + { + "epoch": 0.16858475894245722, + "grad_norm": 0.5164592266082764, + "learning_rate": 4.322500000000001e-05, + "log_odds_chosen": 0.5904808640480042, + "log_odds_ratio": -0.5027241110801697, + "logits/chosen": 0.006964411586523056, + "logits/rejected": -0.11874233931303024, + "logps/chosen": -1.0042153596878052, + "logps/rejected": -1.401916265487671, + "loss": 2.8845, + "nll_loss": 2.834200143814087, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10042153298854828, + "rewards/margins": 0.039770081639289856, + "rewards/rejected": -0.14019162952899933, + "step": 271 + }, + { + "epoch": 0.1692068429237947, + "grad_norm": 0.5214595198631287, + "learning_rate": 4.32e-05, + "log_odds_chosen": 0.24742233753204346, + "log_odds_ratio": -0.7152416110038757, + "logits/chosen": 0.1348581463098526, + "logits/rejected": -0.040653035044670105, + "logps/chosen": -1.2521984577178955, + "logps/rejected": -1.3138203620910645, + "loss": 3.3, + "nll_loss": 3.2285189628601074, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12521985173225403, + "rewards/margins": 0.006162186153233051, + "rewards/rejected": -0.1313820332288742, + "step": 272 + }, + { + "epoch": 0.16982892690513218, + "grad_norm": 0.40921929478645325, + "learning_rate": 4.3175000000000006e-05, + "log_odds_chosen": 0.8957703113555908, + "log_odds_ratio": -0.4180378317832947, + "logits/chosen": 0.04939873516559601, + "logits/rejected": -0.06932801753282547, + "logps/chosen": -0.9343599677085876, + "logps/rejected": -1.509351134300232, + "loss": 3.069, + "nll_loss": 3.027172088623047, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09343599528074265, + "rewards/margins": 0.05749912187457085, + "rewards/rejected": -0.1509351134300232, + "step": 273 + }, + { + "epoch": 0.17045101088646966, + "grad_norm": 0.5043439865112305, + "learning_rate": 4.315e-05, + "log_odds_chosen": 1.2644999027252197, + "log_odds_ratio": -0.43298041820526123, + "logits/chosen": 0.1321154087781906, + "logits/rejected": -0.09128782153129578, + "logps/chosen": -1.0918232202529907, + "logps/rejected": -2.1638400554656982, + "loss": 2.9592, + "nll_loss": 2.9159512519836426, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10918232053518295, + "rewards/margins": 0.10720168799161911, + "rewards/rejected": -0.21638402342796326, + "step": 274 + }, + { + "epoch": 0.17107309486780714, + "grad_norm": 0.4372323751449585, + "learning_rate": 4.3125000000000005e-05, + "log_odds_chosen": 0.33947059512138367, + "log_odds_ratio": -0.6284760236740112, + "logits/chosen": 0.08530203998088837, + "logits/rejected": -0.054735083132982254, + "logps/chosen": -1.3148372173309326, + "logps/rejected": -1.509586215019226, + "loss": 3.0704, + "nll_loss": 3.007549285888672, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13148371875286102, + "rewards/margins": 0.019474901258945465, + "rewards/rejected": -0.15095862746238708, + "step": 275 + }, + { + "epoch": 0.17169517884914465, + "grad_norm": 0.3891263008117676, + "learning_rate": 4.3100000000000004e-05, + "log_odds_chosen": 0.5235820412635803, + "log_odds_ratio": -0.6117573976516724, + "logits/chosen": 0.34458497166633606, + "logits/rejected": 0.05262750759720802, + "logps/chosen": -1.4382394552230835, + "logps/rejected": -1.8694814443588257, + "loss": 3.8207, + "nll_loss": 3.75954270362854, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14382395148277283, + "rewards/margins": 0.04312421381473541, + "rewards/rejected": -0.18694815039634705, + "step": 276 + }, + { + "epoch": 0.17231726283048213, + "grad_norm": 0.4415547251701355, + "learning_rate": 4.3075000000000003e-05, + "log_odds_chosen": 0.05041904002428055, + "log_odds_ratio": -0.6868739128112793, + "logits/chosen": 0.006680905818939209, + "logits/rejected": -0.09860651195049286, + "logps/chosen": -1.2028601169586182, + "logps/rejected": -1.2418127059936523, + "loss": 2.6857, + "nll_loss": 2.6169979572296143, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1202860176563263, + "rewards/margins": 0.0038952501490712166, + "rewards/rejected": -0.12418127059936523, + "step": 277 + }, + { + "epoch": 0.1729393468118196, + "grad_norm": 0.44455549120903015, + "learning_rate": 4.305e-05, + "log_odds_chosen": 1.729311466217041, + "log_odds_ratio": -0.3406631648540497, + "logits/chosen": 0.18643704056739807, + "logits/rejected": 0.041028942912817, + "logps/chosen": -0.9607067108154297, + "logps/rejected": -2.246669292449951, + "loss": 3.3204, + "nll_loss": 3.286289930343628, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09607066959142685, + "rewards/margins": 0.1285962462425232, + "rewards/rejected": -0.22466690838336945, + "step": 278 + }, + { + "epoch": 0.17356143079315708, + "grad_norm": 0.5485425591468811, + "learning_rate": 4.3025e-05, + "log_odds_chosen": 1.234410285949707, + "log_odds_ratio": -0.316910058259964, + "logits/chosen": 0.17609338462352753, + "logits/rejected": -0.03866102546453476, + "logps/chosen": -1.0069231986999512, + "logps/rejected": -1.8689861297607422, + "loss": 3.0301, + "nll_loss": 2.998425006866455, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10069232434034348, + "rewards/margins": 0.08620627969503403, + "rewards/rejected": -0.1868986040353775, + "step": 279 + }, + { + "epoch": 0.17418351477449456, + "grad_norm": 0.5843217372894287, + "learning_rate": 4.3e-05, + "log_odds_chosen": 0.3664306402206421, + "log_odds_ratio": -0.5743635892868042, + "logits/chosen": 0.08559656143188477, + "logits/rejected": -0.036249175667762756, + "logps/chosen": -1.137782335281372, + "logps/rejected": -1.3918867111206055, + "loss": 2.5858, + "nll_loss": 2.528327226638794, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11377823352813721, + "rewards/margins": 0.02541043609380722, + "rewards/rejected": -0.13918867707252502, + "step": 280 + }, + { + "epoch": 0.17480559875583204, + "grad_norm": 0.4670501947402954, + "learning_rate": 4.2975e-05, + "log_odds_chosen": 0.6214780211448669, + "log_odds_ratio": -0.6075859069824219, + "logits/chosen": 0.2213689535856247, + "logits/rejected": 0.08010444790124893, + "logps/chosen": -1.2412688732147217, + "logps/rejected": -1.5694103240966797, + "loss": 3.1539, + "nll_loss": 3.0931215286254883, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12412689626216888, + "rewards/margins": 0.032814137637615204, + "rewards/rejected": -0.1569410264492035, + "step": 281 + }, + { + "epoch": 0.17542768273716952, + "grad_norm": 0.38018524646759033, + "learning_rate": 4.295e-05, + "log_odds_chosen": 0.8761900663375854, + "log_odds_ratio": -0.5851420164108276, + "logits/chosen": 0.04957732930779457, + "logits/rejected": 0.10427582263946533, + "logps/chosen": -1.2226829528808594, + "logps/rejected": -2.00304913520813, + "loss": 2.5916, + "nll_loss": 2.5330915451049805, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.12226830422878265, + "rewards/margins": 0.0780366063117981, + "rewards/rejected": -0.20030491054058075, + "step": 282 + }, + { + "epoch": 0.176049766718507, + "grad_norm": 0.4785327613353729, + "learning_rate": 4.2925000000000007e-05, + "log_odds_chosen": 0.7665344476699829, + "log_odds_ratio": -0.5781853795051575, + "logits/chosen": 0.4440539479255676, + "logits/rejected": 0.12010644376277924, + "logps/chosen": -1.2878354787826538, + "logps/rejected": -1.7937930822372437, + "loss": 3.7711, + "nll_loss": 3.713322401046753, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12878355383872986, + "rewards/margins": 0.050595760345458984, + "rewards/rejected": -0.17937931418418884, + "step": 283 + }, + { + "epoch": 0.17667185069984448, + "grad_norm": 0.4155225157737732, + "learning_rate": 4.29e-05, + "log_odds_chosen": 1.0096347332000732, + "log_odds_ratio": -0.5321923494338989, + "logits/chosen": 0.3086691200733185, + "logits/rejected": 0.2555179297924042, + "logps/chosen": -1.3326821327209473, + "logps/rejected": -2.250950813293457, + "loss": 3.1272, + "nll_loss": 3.0739986896514893, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13326820731163025, + "rewards/margins": 0.09182686358690262, + "rewards/rejected": -0.22509507834911346, + "step": 284 + }, + { + "epoch": 0.17729393468118196, + "grad_norm": 0.4170939028263092, + "learning_rate": 4.2875000000000005e-05, + "log_odds_chosen": 0.6225306391716003, + "log_odds_ratio": -0.5137684345245361, + "logits/chosen": 0.4927368760108948, + "logits/rejected": 0.17728674411773682, + "logps/chosen": -1.3216047286987305, + "logps/rejected": -1.7400200366973877, + "loss": 3.8377, + "nll_loss": 3.786278247833252, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.132160484790802, + "rewards/margins": 0.0418415293097496, + "rewards/rejected": -0.1740020215511322, + "step": 285 + }, + { + "epoch": 0.17791601866251944, + "grad_norm": 0.5519422888755798, + "learning_rate": 4.285e-05, + "log_odds_chosen": 1.4737498760223389, + "log_odds_ratio": -0.38779395818710327, + "logits/chosen": 0.3228255808353424, + "logits/rejected": 0.10631482303142548, + "logps/chosen": -1.236034631729126, + "logps/rejected": -2.550682544708252, + "loss": 2.8773, + "nll_loss": 2.8385581970214844, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1236034631729126, + "rewards/margins": 0.13146479427814484, + "rewards/rejected": -0.25506827235221863, + "step": 286 + }, + { + "epoch": 0.17853810264385692, + "grad_norm": 0.3894568979740143, + "learning_rate": 4.2825000000000004e-05, + "log_odds_chosen": 1.6931242942810059, + "log_odds_ratio": -0.1958678513765335, + "logits/chosen": 0.2403871715068817, + "logits/rejected": 0.07207652926445007, + "logps/chosen": -1.2004201412200928, + "logps/rejected": -2.5714733600616455, + "loss": 3.0662, + "nll_loss": 3.0466134548187256, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12004202604293823, + "rewards/margins": 0.1371053159236908, + "rewards/rejected": -0.25714734196662903, + "step": 287 + }, + { + "epoch": 0.1791601866251944, + "grad_norm": 0.49171674251556396, + "learning_rate": 4.2800000000000004e-05, + "log_odds_chosen": 0.856521487236023, + "log_odds_ratio": -0.4331633448600769, + "logits/chosen": 0.26269370317459106, + "logits/rejected": 0.2361827790737152, + "logps/chosen": -1.312029242515564, + "logps/rejected": -1.9908480644226074, + "loss": 3.1928, + "nll_loss": 3.1494884490966797, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13120292127132416, + "rewards/margins": 0.06788188219070435, + "rewards/rejected": -0.1990848183631897, + "step": 288 + }, + { + "epoch": 0.17978227060653187, + "grad_norm": 0.4975660443305969, + "learning_rate": 4.2775e-05, + "log_odds_chosen": 1.012892484664917, + "log_odds_ratio": -0.3519851267337799, + "logits/chosen": 0.21247489750385284, + "logits/rejected": 0.05814167857170105, + "logps/chosen": -1.152166485786438, + "logps/rejected": -1.9508147239685059, + "loss": 2.5502, + "nll_loss": 2.514974355697632, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11521665006875992, + "rewards/margins": 0.07986482977867126, + "rewards/rejected": -0.19508148729801178, + "step": 289 + }, + { + "epoch": 0.18040435458786935, + "grad_norm": 0.5224384665489197, + "learning_rate": 4.275e-05, + "log_odds_chosen": 1.7257745265960693, + "log_odds_ratio": -0.24555160105228424, + "logits/chosen": 0.47818949818611145, + "logits/rejected": 0.06968870759010315, + "logps/chosen": -0.8608075976371765, + "logps/rejected": -2.147359848022461, + "loss": 3.4919, + "nll_loss": 3.467371940612793, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08608075976371765, + "rewards/margins": 0.12865521013736725, + "rewards/rejected": -0.2147359699010849, + "step": 290 + }, + { + "epoch": 0.18102643856920683, + "grad_norm": 0.4815826416015625, + "learning_rate": 4.2725e-05, + "log_odds_chosen": 1.6820149421691895, + "log_odds_ratio": -0.23076926171779633, + "logits/chosen": 0.33108270168304443, + "logits/rejected": 0.07832175493240356, + "logps/chosen": -0.956098198890686, + "logps/rejected": -2.2002487182617188, + "loss": 2.7417, + "nll_loss": 2.718578338623047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09560982137918472, + "rewards/margins": 0.12441505491733551, + "rewards/rejected": -0.22002488374710083, + "step": 291 + }, + { + "epoch": 0.1816485225505443, + "grad_norm": 0.37672296166419983, + "learning_rate": 4.27e-05, + "log_odds_chosen": 0.5850558876991272, + "log_odds_ratio": -0.4817024767398834, + "logits/chosen": 0.3864336907863617, + "logits/rejected": 0.1432633250951767, + "logps/chosen": -1.0847342014312744, + "logps/rejected": -1.523407220840454, + "loss": 3.3112, + "nll_loss": 3.2630155086517334, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10847342759370804, + "rewards/margins": 0.04386730492115021, + "rewards/rejected": -0.15234072506427765, + "step": 292 + }, + { + "epoch": 0.1822706065318818, + "grad_norm": 0.5808604955673218, + "learning_rate": 4.2675e-05, + "log_odds_chosen": 0.610927939414978, + "log_odds_ratio": -0.4852539002895355, + "logits/chosen": 0.35360458493232727, + "logits/rejected": 0.008299056440591812, + "logps/chosen": -1.268179178237915, + "logps/rejected": -1.7629632949829102, + "loss": 2.9865, + "nll_loss": 2.9379255771636963, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12681792676448822, + "rewards/margins": 0.04947839677333832, + "rewards/rejected": -0.17629633843898773, + "step": 293 + }, + { + "epoch": 0.1828926905132193, + "grad_norm": 0.412265807390213, + "learning_rate": 4.265e-05, + "log_odds_chosen": 1.2932488918304443, + "log_odds_ratio": -0.3016420006752014, + "logits/chosen": 0.3499048352241516, + "logits/rejected": 0.12447762489318848, + "logps/chosen": -1.0451427698135376, + "logps/rejected": -1.9540801048278809, + "loss": 3.1025, + "nll_loss": 3.0723202228546143, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10451428592205048, + "rewards/margins": 0.09089374542236328, + "rewards/rejected": -0.19540801644325256, + "step": 294 + }, + { + "epoch": 0.18351477449455678, + "grad_norm": 0.44288045167922974, + "learning_rate": 4.2625000000000006e-05, + "log_odds_chosen": 0.4354422688484192, + "log_odds_ratio": -0.5339393615722656, + "logits/chosen": 0.2939504086971283, + "logits/rejected": 0.13727524876594543, + "logps/chosen": -1.2091028690338135, + "logps/rejected": -1.5410089492797852, + "loss": 2.9659, + "nll_loss": 2.9124677181243896, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12091030180454254, + "rewards/margins": 0.03319060429930687, + "rewards/rejected": -0.15410089492797852, + "step": 295 + }, + { + "epoch": 0.18413685847589426, + "grad_norm": 0.4726203680038452, + "learning_rate": 4.26e-05, + "log_odds_chosen": 0.4269523620605469, + "log_odds_ratio": -0.6357343196868896, + "logits/chosen": 0.42432665824890137, + "logits/rejected": 0.09816896915435791, + "logps/chosen": -1.2562463283538818, + "logps/rejected": -1.4864271879196167, + "loss": 3.435, + "nll_loss": 3.371392250061035, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1256246566772461, + "rewards/margins": 0.023018071427941322, + "rewards/rejected": -0.14864271879196167, + "step": 296 + }, + { + "epoch": 0.18475894245723173, + "grad_norm": 0.4871671199798584, + "learning_rate": 4.2575000000000005e-05, + "log_odds_chosen": 0.5284110307693481, + "log_odds_ratio": -0.6341941356658936, + "logits/chosen": 0.3682095408439636, + "logits/rejected": 0.31838396191596985, + "logps/chosen": -1.3797330856323242, + "logps/rejected": -1.8745726346969604, + "loss": 3.0913, + "nll_loss": 3.0278983116149902, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.13797330856323242, + "rewards/margins": 0.04948396235704422, + "rewards/rejected": -0.18745726346969604, + "step": 297 + }, + { + "epoch": 0.1853810264385692, + "grad_norm": 0.33851662278175354, + "learning_rate": 4.2550000000000004e-05, + "log_odds_chosen": 0.6119697093963623, + "log_odds_ratio": -0.6034431457519531, + "logits/chosen": 0.28094005584716797, + "logits/rejected": 0.2045813798904419, + "logps/chosen": -1.17061448097229, + "logps/rejected": -1.5927424430847168, + "loss": 3.4087, + "nll_loss": 3.3483834266662598, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11706145107746124, + "rewards/margins": 0.042212799191474915, + "rewards/rejected": -0.15927425026893616, + "step": 298 + }, + { + "epoch": 0.1860031104199067, + "grad_norm": 0.49449941515922546, + "learning_rate": 4.2525000000000004e-05, + "log_odds_chosen": 0.40445905923843384, + "log_odds_ratio": -0.5552644729614258, + "logits/chosen": 0.2672854959964752, + "logits/rejected": 0.16854549944400787, + "logps/chosen": -1.561603307723999, + "logps/rejected": -1.825971007347107, + "loss": 2.7683, + "nll_loss": 2.7127437591552734, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15616033971309662, + "rewards/margins": 0.026436757296323776, + "rewards/rejected": -0.1825971007347107, + "step": 299 + }, + { + "epoch": 0.18662519440124417, + "grad_norm": 0.48564156889915466, + "learning_rate": 4.25e-05, + "log_odds_chosen": 1.5461804866790771, + "log_odds_ratio": -0.26383474469184875, + "logits/chosen": 0.36446020007133484, + "logits/rejected": 0.2191128432750702, + "logps/chosen": -1.0896272659301758, + "logps/rejected": -2.29062557220459, + "loss": 3.5646, + "nll_loss": 3.5382304191589355, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10896272957324982, + "rewards/margins": 0.12009983509778976, + "rewards/rejected": -0.22906255722045898, + "step": 300 + }, + { + "epoch": 0.18724727838258165, + "grad_norm": 0.39352017641067505, + "learning_rate": 4.2475e-05, + "log_odds_chosen": 1.5315042734146118, + "log_odds_ratio": -0.41960352659225464, + "logits/chosen": 0.3345024585723877, + "logits/rejected": 0.1347406506538391, + "logps/chosen": -1.1930088996887207, + "logps/rejected": -2.30621337890625, + "loss": 3.5042, + "nll_loss": 3.4622511863708496, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11930090188980103, + "rewards/margins": 0.11132044345140457, + "rewards/rejected": -0.230621337890625, + "step": 301 + }, + { + "epoch": 0.18786936236391913, + "grad_norm": 0.3949545621871948, + "learning_rate": 4.245e-05, + "log_odds_chosen": 0.7881875038146973, + "log_odds_ratio": -0.4549405574798584, + "logits/chosen": 0.1703367680311203, + "logits/rejected": 0.10550281405448914, + "logps/chosen": -1.1531308889389038, + "logps/rejected": -1.7436622381210327, + "loss": 3.0154, + "nll_loss": 2.9698915481567383, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1153130903840065, + "rewards/margins": 0.05905313044786453, + "rewards/rejected": -0.17436623573303223, + "step": 302 + }, + { + "epoch": 0.1884914463452566, + "grad_norm": 0.3909085690975189, + "learning_rate": 4.2425e-05, + "log_odds_chosen": 1.3018293380737305, + "log_odds_ratio": -0.33854174613952637, + "logits/chosen": 0.32386866211891174, + "logits/rejected": 0.33090081810951233, + "logps/chosen": -1.2735427618026733, + "logps/rejected": -2.3735671043395996, + "loss": 2.9068, + "nll_loss": 2.872992753982544, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12735427916049957, + "rewards/margins": 0.11000245064496994, + "rewards/rejected": -0.23735672235488892, + "step": 303 + }, + { + "epoch": 0.1891135303265941, + "grad_norm": 0.678255558013916, + "learning_rate": 4.24e-05, + "log_odds_chosen": 0.5951331853866577, + "log_odds_ratio": -0.5492010116577148, + "logits/chosen": 0.18354575335979462, + "logits/rejected": -0.09810749441385269, + "logps/chosen": -1.291944980621338, + "logps/rejected": -1.642785668373108, + "loss": 3.1579, + "nll_loss": 3.102977752685547, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1291944980621338, + "rewards/margins": 0.0350840725004673, + "rewards/rejected": -0.1642785668373108, + "step": 304 + }, + { + "epoch": 0.18973561430793157, + "grad_norm": 0.5240991115570068, + "learning_rate": 4.237500000000001e-05, + "log_odds_chosen": 3.6760783195495605, + "log_odds_ratio": -0.180177241563797, + "logits/chosen": 0.14156953990459442, + "logits/rejected": -0.06919633597135544, + "logps/chosen": -0.7791962623596191, + "logps/rejected": -3.654845714569092, + "loss": 3.0695, + "nll_loss": 3.051457643508911, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07791963219642639, + "rewards/margins": 0.2875649333000183, + "rewards/rejected": -0.3654845356941223, + "step": 305 + }, + { + "epoch": 0.19035769828926905, + "grad_norm": 0.4615384042263031, + "learning_rate": 4.235e-05, + "log_odds_chosen": 0.5181177258491516, + "log_odds_ratio": -0.5444080829620361, + "logits/chosen": 0.20214281976222992, + "logits/rejected": 0.17525801062583923, + "logps/chosen": -1.248335838317871, + "logps/rejected": -1.6521800756454468, + "loss": 2.82, + "nll_loss": 2.7655587196350098, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12483358383178711, + "rewards/margins": 0.0403844378888607, + "rewards/rejected": -0.16521801054477692, + "step": 306 + }, + { + "epoch": 0.19097978227060652, + "grad_norm": 0.4177151620388031, + "learning_rate": 4.2325000000000006e-05, + "log_odds_chosen": 0.533401608467102, + "log_odds_ratio": -0.520737886428833, + "logits/chosen": 0.1740420013666153, + "logits/rejected": 0.11574655026197433, + "logps/chosen": -1.2388856410980225, + "logps/rejected": -1.618835687637329, + "loss": 2.9597, + "nll_loss": 2.9076099395751953, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12388856709003448, + "rewards/margins": 0.03799499571323395, + "rewards/rejected": -0.16188356280326843, + "step": 307 + }, + { + "epoch": 0.191601866251944, + "grad_norm": 0.4279910624027252, + "learning_rate": 4.23e-05, + "log_odds_chosen": 1.0015525817871094, + "log_odds_ratio": -0.4986322224140167, + "logits/chosen": 0.4348335564136505, + "logits/rejected": 0.1550769805908203, + "logps/chosen": -1.2172647714614868, + "logps/rejected": -2.026916742324829, + "loss": 3.3799, + "nll_loss": 3.330019950866699, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12172648310661316, + "rewards/margins": 0.08096518367528915, + "rewards/rejected": -0.20269165933132172, + "step": 308 + }, + { + "epoch": 0.19222395023328148, + "grad_norm": 0.47602641582489014, + "learning_rate": 4.2275000000000004e-05, + "log_odds_chosen": 1.2695807218551636, + "log_odds_ratio": -0.34691569209098816, + "logits/chosen": 0.2914532721042633, + "logits/rejected": 0.17980057001113892, + "logps/chosen": -0.9040358662605286, + "logps/rejected": -1.8016421794891357, + "loss": 3.2324, + "nll_loss": 3.1976821422576904, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09040358662605286, + "rewards/margins": 0.08976064622402191, + "rewards/rejected": -0.18016423285007477, + "step": 309 + }, + { + "epoch": 0.19284603421461896, + "grad_norm": 0.4949467182159424, + "learning_rate": 4.2250000000000004e-05, + "log_odds_chosen": 0.9988858103752136, + "log_odds_ratio": -0.4942994713783264, + "logits/chosen": 0.2518168091773987, + "logits/rejected": 0.07765643298625946, + "logps/chosen": -1.2327001094818115, + "logps/rejected": -2.0293068885803223, + "loss": 2.9551, + "nll_loss": 2.905709743499756, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12327000498771667, + "rewards/margins": 0.07966066896915436, + "rewards/rejected": -0.20293068885803223, + "step": 310 + }, + { + "epoch": 0.19346811819595647, + "grad_norm": 0.4487239122390747, + "learning_rate": 4.2225e-05, + "log_odds_chosen": 0.5391438603401184, + "log_odds_ratio": -0.7081095576286316, + "logits/chosen": 0.42160993814468384, + "logits/rejected": 0.17291218042373657, + "logps/chosen": -1.851615309715271, + "logps/rejected": -2.3373863697052, + "loss": 3.9019, + "nll_loss": 3.831056833267212, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1851615458726883, + "rewards/margins": 0.04857710748910904, + "rewards/rejected": -0.23373863101005554, + "step": 311 + }, + { + "epoch": 0.19409020217729395, + "grad_norm": 0.47805315256118774, + "learning_rate": 4.22e-05, + "log_odds_chosen": 1.0429258346557617, + "log_odds_ratio": -0.42988187074661255, + "logits/chosen": 0.12882010638713837, + "logits/rejected": 0.03473813459277153, + "logps/chosen": -1.2987618446350098, + "logps/rejected": -2.2001380920410156, + "loss": 2.8064, + "nll_loss": 2.7633748054504395, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12987619638442993, + "rewards/margins": 0.09013761579990387, + "rewards/rejected": -0.2200138121843338, + "step": 312 + }, + { + "epoch": 0.19471228615863143, + "grad_norm": 0.633873462677002, + "learning_rate": 4.2175e-05, + "log_odds_chosen": 0.6489430665969849, + "log_odds_ratio": -0.5409046411514282, + "logits/chosen": 0.1894546002149582, + "logits/rejected": -0.016907554119825363, + "logps/chosen": -1.4098745584487915, + "logps/rejected": -1.8865466117858887, + "loss": 3.4695, + "nll_loss": 3.4154553413391113, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14098745584487915, + "rewards/margins": 0.047667186707258224, + "rewards/rejected": -0.18865466117858887, + "step": 313 + }, + { + "epoch": 0.1953343701399689, + "grad_norm": 0.4453391432762146, + "learning_rate": 4.215e-05, + "log_odds_chosen": 1.4420627355575562, + "log_odds_ratio": -0.4442042112350464, + "logits/chosen": 0.13987398147583008, + "logits/rejected": 0.13250389695167542, + "logps/chosen": -1.3193628787994385, + "logps/rejected": -2.651116371154785, + "loss": 2.7875, + "nll_loss": 2.743088722229004, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13193629682064056, + "rewards/margins": 0.133175328373909, + "rewards/rejected": -0.26511162519454956, + "step": 314 + }, + { + "epoch": 0.19595645412130638, + "grad_norm": 0.48520854115486145, + "learning_rate": 4.2125e-05, + "log_odds_chosen": 1.0938706398010254, + "log_odds_ratio": -0.3769736588001251, + "logits/chosen": 0.14695967733860016, + "logits/rejected": 0.03137361258268356, + "logps/chosen": -1.062088966369629, + "logps/rejected": -1.953197717666626, + "loss": 2.8743, + "nll_loss": 2.8366215229034424, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10620889812707901, + "rewards/margins": 0.08911087363958359, + "rewards/rejected": -0.1953197717666626, + "step": 315 + }, + { + "epoch": 0.19657853810264386, + "grad_norm": 0.4708939492702484, + "learning_rate": 4.21e-05, + "log_odds_chosen": 0.7692523002624512, + "log_odds_ratio": -0.4789087176322937, + "logits/chosen": 0.13523553311824799, + "logits/rejected": 0.05263520032167435, + "logps/chosen": -1.1547871828079224, + "logps/rejected": -1.744168996810913, + "loss": 3.1827, + "nll_loss": 3.134774684906006, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11547872424125671, + "rewards/margins": 0.05893819034099579, + "rewards/rejected": -0.1744169145822525, + "step": 316 + }, + { + "epoch": 0.19720062208398134, + "grad_norm": 0.5101141929626465, + "learning_rate": 4.2075000000000006e-05, + "log_odds_chosen": 0.9251267910003662, + "log_odds_ratio": -0.420250803232193, + "logits/chosen": 0.1897871047258377, + "logits/rejected": 0.0760931670665741, + "logps/chosen": -1.4300227165222168, + "logps/rejected": -2.243600845336914, + "loss": 3.0869, + "nll_loss": 3.044924259185791, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14300227165222168, + "rewards/margins": 0.08135779947042465, + "rewards/rejected": -0.22436007857322693, + "step": 317 + }, + { + "epoch": 0.19782270606531882, + "grad_norm": 0.4665040373802185, + "learning_rate": 4.205e-05, + "log_odds_chosen": 1.6941817998886108, + "log_odds_ratio": -0.3107852339744568, + "logits/chosen": 0.10233496129512787, + "logits/rejected": -0.04454576224088669, + "logps/chosen": -1.426357388496399, + "logps/rejected": -2.9099607467651367, + "loss": 3.0493, + "nll_loss": 3.018172025680542, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14263573288917542, + "rewards/margins": 0.14836034178733826, + "rewards/rejected": -0.29099607467651367, + "step": 318 + }, + { + "epoch": 0.1984447900466563, + "grad_norm": 0.8431224822998047, + "learning_rate": 4.2025000000000005e-05, + "log_odds_chosen": 0.7668442130088806, + "log_odds_ratio": -0.489085853099823, + "logits/chosen": 0.1494210809469223, + "logits/rejected": 0.06121387332677841, + "logps/chosen": -1.5238502025604248, + "logps/rejected": -2.1793744564056396, + "loss": 3.1432, + "nll_loss": 3.0943074226379395, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15238502621650696, + "rewards/margins": 0.06555242836475372, + "rewards/rejected": -0.2179374396800995, + "step": 319 + }, + { + "epoch": 0.19906687402799378, + "grad_norm": 0.6626170873641968, + "learning_rate": 4.2e-05, + "log_odds_chosen": 1.027713656425476, + "log_odds_ratio": -0.37676990032196045, + "logits/chosen": 0.16855540871620178, + "logits/rejected": -0.08387507498264313, + "logps/chosen": -1.222766637802124, + "logps/rejected": -2.0757288932800293, + "loss": 2.8647, + "nll_loss": 2.827024459838867, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1222766637802124, + "rewards/margins": 0.08529624342918396, + "rewards/rejected": -0.20757290720939636, + "step": 320 + }, + { + "epoch": 0.19968895800933126, + "grad_norm": 0.4691254794597626, + "learning_rate": 4.1975000000000004e-05, + "log_odds_chosen": 1.1109933853149414, + "log_odds_ratio": -0.37901267409324646, + "logits/chosen": 0.14058086276054382, + "logits/rejected": -0.0027409703470766544, + "logps/chosen": -0.9070495367050171, + "logps/rejected": -1.6633799076080322, + "loss": 2.8839, + "nll_loss": 2.846041202545166, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09070496261119843, + "rewards/margins": 0.07563304156064987, + "rewards/rejected": -0.1663379967212677, + "step": 321 + }, + { + "epoch": 0.20031104199066874, + "grad_norm": 0.5388383865356445, + "learning_rate": 4.195e-05, + "log_odds_chosen": 1.4941861629486084, + "log_odds_ratio": -0.2680909335613251, + "logits/chosen": 0.11168316006660461, + "logits/rejected": -0.132093608379364, + "logps/chosen": -0.9982537031173706, + "logps/rejected": -1.9839081764221191, + "loss": 2.9087, + "nll_loss": 2.881884813308716, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09982537478208542, + "rewards/margins": 0.09856545180082321, + "rewards/rejected": -0.19839082658290863, + "step": 322 + }, + { + "epoch": 0.20093312597200622, + "grad_norm": 0.39389094710350037, + "learning_rate": 4.1925e-05, + "log_odds_chosen": 1.2304775714874268, + "log_odds_ratio": -0.4389082193374634, + "logits/chosen": 0.12545819580554962, + "logits/rejected": 0.040004927664995193, + "logps/chosen": -1.345543384552002, + "logps/rejected": -2.4424848556518555, + "loss": 3.1398, + "nll_loss": 3.095932722091675, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13455434143543243, + "rewards/margins": 0.10969413816928864, + "rewards/rejected": -0.24424846470355988, + "step": 323 + }, + { + "epoch": 0.2015552099533437, + "grad_norm": 0.45025521516799927, + "learning_rate": 4.19e-05, + "log_odds_chosen": 0.9204633831977844, + "log_odds_ratio": -0.5545144081115723, + "logits/chosen": 0.04150621220469475, + "logits/rejected": -0.08123153448104858, + "logps/chosen": -1.1536731719970703, + "logps/rejected": -1.8100297451019287, + "loss": 3.0471, + "nll_loss": 2.9916718006134033, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11536730825901031, + "rewards/margins": 0.06563566625118256, + "rewards/rejected": -0.18100297451019287, + "step": 324 + }, + { + "epoch": 0.20217729393468117, + "grad_norm": 0.4303993880748749, + "learning_rate": 4.1875e-05, + "log_odds_chosen": 1.6544878482818604, + "log_odds_ratio": -0.19887760281562805, + "logits/chosen": 0.08420052379369736, + "logits/rejected": 0.0031083375215530396, + "logps/chosen": -1.3029048442840576, + "logps/rejected": -2.700894594192505, + "loss": 3.1539, + "nll_loss": 3.1340394020080566, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13029049336910248, + "rewards/margins": 0.13979899883270264, + "rewards/rejected": -0.2700894773006439, + "step": 325 + }, + { + "epoch": 0.20279937791601865, + "grad_norm": 1.0563331842422485, + "learning_rate": 4.185e-05, + "log_odds_chosen": 0.21350765228271484, + "log_odds_ratio": -0.6822980642318726, + "logits/chosen": -0.024682089686393738, + "logits/rejected": -0.08352095633745193, + "logps/chosen": -1.4727632999420166, + "logps/rejected": -1.5891326665878296, + "loss": 2.8543, + "nll_loss": 2.7860758304595947, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1472763568162918, + "rewards/margins": 0.011636918410658836, + "rewards/rejected": -0.1589132696390152, + "step": 326 + }, + { + "epoch": 0.20342146189735613, + "grad_norm": 0.48655053973197937, + "learning_rate": 4.1825e-05, + "log_odds_chosen": 0.36451324820518494, + "log_odds_ratio": -0.6466756463050842, + "logits/chosen": 0.02400357648730278, + "logits/rejected": -0.04008547589182854, + "logps/chosen": -1.255650520324707, + "logps/rejected": -1.5385801792144775, + "loss": 3.1882, + "nll_loss": 3.1234936714172363, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1255650520324707, + "rewards/margins": 0.02829296514391899, + "rewards/rejected": -0.15385802090168, + "step": 327 + }, + { + "epoch": 0.2040435458786936, + "grad_norm": 0.5478668212890625, + "learning_rate": 4.18e-05, + "log_odds_chosen": 1.6087690591812134, + "log_odds_ratio": -0.31839194893836975, + "logits/chosen": 0.11044462025165558, + "logits/rejected": -0.06441525369882584, + "logps/chosen": -1.167734146118164, + "logps/rejected": -2.54228138923645, + "loss": 2.9786, + "nll_loss": 2.9467685222625732, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11677341163158417, + "rewards/margins": 0.13745473325252533, + "rewards/rejected": -0.2542281448841095, + "step": 328 + }, + { + "epoch": 0.20466562986003112, + "grad_norm": 0.3749350309371948, + "learning_rate": 4.1775000000000006e-05, + "log_odds_chosen": 0.4586121439933777, + "log_odds_ratio": -0.5654648542404175, + "logits/chosen": 0.09604926407337189, + "logits/rejected": -0.018427755683660507, + "logps/chosen": -1.2352399826049805, + "logps/rejected": -1.6107468605041504, + "loss": 3.0382, + "nll_loss": 2.981649875640869, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1235240027308464, + "rewards/margins": 0.03755068778991699, + "rewards/rejected": -0.161074697971344, + "step": 329 + }, + { + "epoch": 0.2052877138413686, + "grad_norm": 0.4368545413017273, + "learning_rate": 4.175e-05, + "log_odds_chosen": 0.8839046955108643, + "log_odds_ratio": -0.5126104354858398, + "logits/chosen": -0.007048317231237888, + "logits/rejected": -0.09785065799951553, + "logps/chosen": -1.0619440078735352, + "logps/rejected": -1.7408676147460938, + "loss": 3.0543, + "nll_loss": 3.0030674934387207, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.106194406747818, + "rewards/margins": 0.06789234280586243, + "rewards/rejected": -0.17408674955368042, + "step": 330 + }, + { + "epoch": 0.20590979782270608, + "grad_norm": 0.5278705954551697, + "learning_rate": 4.1725000000000005e-05, + "log_odds_chosen": 0.30167797207832336, + "log_odds_ratio": -0.6502044796943665, + "logits/chosen": 0.032889269292354584, + "logits/rejected": 0.03355567902326584, + "logps/chosen": -1.4090936183929443, + "logps/rejected": -1.6204454898834229, + "loss": 3.1308, + "nll_loss": 3.0657973289489746, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1409093588590622, + "rewards/margins": 0.021135203540325165, + "rewards/rejected": -0.16204456984996796, + "step": 331 + }, + { + "epoch": 0.20653188180404355, + "grad_norm": 0.4414202868938446, + "learning_rate": 4.17e-05, + "log_odds_chosen": 1.4241623878479004, + "log_odds_ratio": -0.3517926335334778, + "logits/chosen": 0.11898308247327805, + "logits/rejected": -0.013245618902146816, + "logps/chosen": -0.9463114738464355, + "logps/rejected": -2.077383041381836, + "loss": 2.8483, + "nll_loss": 2.8131093978881836, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0946311503648758, + "rewards/margins": 0.11310717463493347, + "rewards/rejected": -0.20773832499980927, + "step": 332 + }, + { + "epoch": 0.20715396578538103, + "grad_norm": 0.38851961493492126, + "learning_rate": 4.1675e-05, + "log_odds_chosen": 0.28314775228500366, + "log_odds_ratio": -0.578311562538147, + "logits/chosen": -0.008570694364607334, + "logits/rejected": -0.05588866025209427, + "logps/chosen": -1.1838418245315552, + "logps/rejected": -1.3598947525024414, + "loss": 2.8154, + "nll_loss": 2.757525682449341, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11838418245315552, + "rewards/margins": 0.017605291679501534, + "rewards/rejected": -0.1359894871711731, + "step": 333 + }, + { + "epoch": 0.2077760497667185, + "grad_norm": 0.6326988935470581, + "learning_rate": 4.165e-05, + "log_odds_chosen": 0.140260249376297, + "log_odds_ratio": -0.677656888961792, + "logits/chosen": 0.025349613279104233, + "logits/rejected": -0.08781476318836212, + "logps/chosen": -1.341933250427246, + "logps/rejected": -1.4497263431549072, + "loss": 2.7444, + "nll_loss": 2.676629066467285, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1341933161020279, + "rewards/margins": 0.010779321193695068, + "rewards/rejected": -0.14497263729572296, + "step": 334 + }, + { + "epoch": 0.208398133748056, + "grad_norm": 0.4192201793193817, + "learning_rate": 4.1625e-05, + "log_odds_chosen": 1.2017258405685425, + "log_odds_ratio": -0.373602032661438, + "logits/chosen": 0.16877499222755432, + "logits/rejected": 0.06103990972042084, + "logps/chosen": -1.1729239225387573, + "logps/rejected": -1.9996774196624756, + "loss": 3.3254, + "nll_loss": 3.2880568504333496, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1172923892736435, + "rewards/margins": 0.08267535269260406, + "rewards/rejected": -0.19996774196624756, + "step": 335 + }, + { + "epoch": 0.20902021772939347, + "grad_norm": 0.5239099264144897, + "learning_rate": 4.16e-05, + "log_odds_chosen": 0.15256771445274353, + "log_odds_ratio": -0.6872279047966003, + "logits/chosen": 0.0012313630431890488, + "logits/rejected": -0.11143770813941956, + "logps/chosen": -1.2717194557189941, + "logps/rejected": -1.4060418605804443, + "loss": 2.7665, + "nll_loss": 2.6978044509887695, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.12717196345329285, + "rewards/margins": 0.013432244770228863, + "rewards/rejected": -0.1406041979789734, + "step": 336 + }, + { + "epoch": 0.20964230171073095, + "grad_norm": 0.46690982580184937, + "learning_rate": 4.1575e-05, + "log_odds_chosen": 1.1506333351135254, + "log_odds_ratio": -0.396980345249176, + "logits/chosen": 0.14569717645645142, + "logits/rejected": -0.02970746159553528, + "logps/chosen": -0.9054533243179321, + "logps/rejected": -1.7300324440002441, + "loss": 3.1932, + "nll_loss": 3.153456926345825, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09054533392190933, + "rewards/margins": 0.08245791494846344, + "rewards/rejected": -0.17300325632095337, + "step": 337 + }, + { + "epoch": 0.21026438569206843, + "grad_norm": 0.4816306233406067, + "learning_rate": 4.155e-05, + "log_odds_chosen": 1.2588977813720703, + "log_odds_ratio": -0.38654831051826477, + "logits/chosen": 0.1284165233373642, + "logits/rejected": 0.061425551772117615, + "logps/chosen": -1.5059263706207275, + "logps/rejected": -2.580245018005371, + "loss": 2.7022, + "nll_loss": 2.6635427474975586, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1505926251411438, + "rewards/margins": 0.10743187367916107, + "rewards/rejected": -0.25802451372146606, + "step": 338 + }, + { + "epoch": 0.2108864696734059, + "grad_norm": 0.5407478213310242, + "learning_rate": 4.1525e-05, + "log_odds_chosen": 1.0435866117477417, + "log_odds_ratio": -0.4741755723953247, + "logits/chosen": 0.07317960262298584, + "logits/rejected": 0.008852528408169746, + "logps/chosen": -1.398842215538025, + "logps/rejected": -2.2561936378479004, + "loss": 3.1174, + "nll_loss": 3.0700323581695557, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13988423347473145, + "rewards/margins": 0.08573515713214874, + "rewards/rejected": -0.225619375705719, + "step": 339 + }, + { + "epoch": 0.2115085536547434, + "grad_norm": 0.5428896546363831, + "learning_rate": 4.15e-05, + "log_odds_chosen": 1.3254553079605103, + "log_odds_ratio": -0.3870584964752197, + "logits/chosen": 0.12351932376623154, + "logits/rejected": -0.04987223446369171, + "logps/chosen": -1.1178048849105835, + "logps/rejected": -2.1846683025360107, + "loss": 2.6345, + "nll_loss": 2.5957448482513428, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11178049445152283, + "rewards/margins": 0.10668633878231049, + "rewards/rejected": -0.2184668481349945, + "step": 340 + }, + { + "epoch": 0.21213063763608087, + "grad_norm": 0.49065813422203064, + "learning_rate": 4.1475000000000005e-05, + "log_odds_chosen": 0.6055381298065186, + "log_odds_ratio": -0.5067149996757507, + "logits/chosen": 0.11017728596925735, + "logits/rejected": -0.005071556195616722, + "logps/chosen": -1.3153109550476074, + "logps/rejected": -1.7827832698822021, + "loss": 3.0023, + "nll_loss": 2.9516162872314453, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13153107464313507, + "rewards/margins": 0.04674723744392395, + "rewards/rejected": -0.17827832698822021, + "step": 341 + }, + { + "epoch": 0.21275272161741834, + "grad_norm": 0.38548681139945984, + "learning_rate": 4.145e-05, + "log_odds_chosen": 1.6406880617141724, + "log_odds_ratio": -0.32489141821861267, + "logits/chosen": 0.20307300984859467, + "logits/rejected": 0.008104506880044937, + "logps/chosen": -0.8666763305664062, + "logps/rejected": -2.039579391479492, + "loss": 3.6414, + "nll_loss": 3.608938455581665, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08666763454675674, + "rewards/margins": 0.11729030311107635, + "rewards/rejected": -0.2039579451084137, + "step": 342 + }, + { + "epoch": 0.21337480559875582, + "grad_norm": 0.44505774974823, + "learning_rate": 4.1425000000000004e-05, + "log_odds_chosen": 0.9692625999450684, + "log_odds_ratio": -0.4369995594024658, + "logits/chosen": 0.17034012079238892, + "logits/rejected": -0.028462085872888565, + "logps/chosen": -1.098499059677124, + "logps/rejected": -1.6745946407318115, + "loss": 3.3937, + "nll_loss": 3.34997296333313, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10984990000724792, + "rewards/margins": 0.05760956183075905, + "rewards/rejected": -0.16745945811271667, + "step": 343 + }, + { + "epoch": 0.2139968895800933, + "grad_norm": 0.4816812574863434, + "learning_rate": 4.14e-05, + "log_odds_chosen": 1.5928500890731812, + "log_odds_ratio": -0.23111592233181, + "logits/chosen": 0.188461571931839, + "logits/rejected": 0.022169630974531174, + "logps/chosen": -0.9164315462112427, + "logps/rejected": -2.1156816482543945, + "loss": 2.7098, + "nll_loss": 2.6867213249206543, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09164315462112427, + "rewards/margins": 0.11992499977350235, + "rewards/rejected": -0.21156814694404602, + "step": 344 + }, + { + "epoch": 0.21461897356143078, + "grad_norm": 0.39990314841270447, + "learning_rate": 4.1375e-05, + "log_odds_chosen": 0.49777552485466003, + "log_odds_ratio": -0.6073991656303406, + "logits/chosen": 0.18223583698272705, + "logits/rejected": 0.04722006618976593, + "logps/chosen": -1.0849920511245728, + "logps/rejected": -1.4609473943710327, + "loss": 3.2362, + "nll_loss": 3.1754815578460693, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.10849921405315399, + "rewards/margins": 0.03759553283452988, + "rewards/rejected": -0.14609473943710327, + "step": 345 + }, + { + "epoch": 0.21524105754276826, + "grad_norm": 0.5857303738594055, + "learning_rate": 4.135e-05, + "log_odds_chosen": 0.844973087310791, + "log_odds_ratio": -0.46381595730781555, + "logits/chosen": 0.2590794265270233, + "logits/rejected": 0.1343073546886444, + "logps/chosen": -1.3666094541549683, + "logps/rejected": -2.0936036109924316, + "loss": 3.0643, + "nll_loss": 3.0179357528686523, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13666093349456787, + "rewards/margins": 0.0726994052529335, + "rewards/rejected": -0.20936036109924316, + "step": 346 + }, + { + "epoch": 0.21586314152410577, + "grad_norm": 0.567475438117981, + "learning_rate": 4.1325e-05, + "log_odds_chosen": 0.8636181354522705, + "log_odds_ratio": -0.40255892276763916, + "logits/chosen": 0.20596131682395935, + "logits/rejected": 0.061537884175777435, + "logps/chosen": -1.3861594200134277, + "logps/rejected": -2.1171822547912598, + "loss": 2.8103, + "nll_loss": 2.770002603530884, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1386159509420395, + "rewards/margins": 0.07310228049755096, + "rewards/rejected": -0.21171823143959045, + "step": 347 + }, + { + "epoch": 0.21648522550544325, + "grad_norm": 0.5971509218215942, + "learning_rate": 4.13e-05, + "log_odds_chosen": 2.9550795555114746, + "log_odds_ratio": -0.1911197304725647, + "logits/chosen": 0.29377758502960205, + "logits/rejected": 0.28102779388427734, + "logps/chosen": -1.3482720851898193, + "logps/rejected": -4.058900833129883, + "loss": 3.0978, + "nll_loss": 3.0786807537078857, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13482721149921417, + "rewards/margins": 0.2710628807544708, + "rewards/rejected": -0.4058900773525238, + "step": 348 + }, + { + "epoch": 0.21710730948678073, + "grad_norm": 0.44494226574897766, + "learning_rate": 4.1275e-05, + "log_odds_chosen": 1.0941739082336426, + "log_odds_ratio": -0.4475710391998291, + "logits/chosen": 0.15722906589508057, + "logits/rejected": 0.02086580917239189, + "logps/chosen": -1.191224455833435, + "logps/rejected": -2.0817713737487793, + "loss": 2.9021, + "nll_loss": 2.8573296070098877, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1191224530339241, + "rewards/margins": 0.08905468881130219, + "rewards/rejected": -0.20817714929580688, + "step": 349 + }, + { + "epoch": 0.2177293934681182, + "grad_norm": 0.5429697632789612, + "learning_rate": 4.125e-05, + "log_odds_chosen": 1.2548682689666748, + "log_odds_ratio": -0.37429577112197876, + "logits/chosen": 0.28128620982170105, + "logits/rejected": 0.08453608304262161, + "logps/chosen": -1.363343596458435, + "logps/rejected": -2.3076062202453613, + "loss": 3.444, + "nll_loss": 3.4065957069396973, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1363343596458435, + "rewards/margins": 0.09442625939846039, + "rewards/rejected": -0.2307606041431427, + "step": 350 + }, + { + "epoch": 0.21835147744945568, + "grad_norm": 0.41189780831336975, + "learning_rate": 4.1225e-05, + "log_odds_chosen": 1.1283422708511353, + "log_odds_ratio": -0.41111886501312256, + "logits/chosen": 0.27242082357406616, + "logits/rejected": 0.169078066945076, + "logps/chosen": -1.1591994762420654, + "logps/rejected": -2.119546413421631, + "loss": 3.0923, + "nll_loss": 3.0512022972106934, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11591994762420654, + "rewards/margins": 0.0960346907377243, + "rewards/rejected": -0.21195463836193085, + "step": 351 + }, + { + "epoch": 0.21897356143079316, + "grad_norm": 0.4804132878780365, + "learning_rate": 4.12e-05, + "log_odds_chosen": 2.3194806575775146, + "log_odds_ratio": -0.2600080370903015, + "logits/chosen": 0.2595370411872864, + "logits/rejected": 0.08692187070846558, + "logps/chosen": -0.8231872320175171, + "logps/rejected": -2.4552807807922363, + "loss": 3.3194, + "nll_loss": 3.293379306793213, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08231872320175171, + "rewards/margins": 0.16320934891700745, + "rewards/rejected": -0.24552807211875916, + "step": 352 + }, + { + "epoch": 0.21959564541213064, + "grad_norm": 0.5321614146232605, + "learning_rate": 4.1175000000000005e-05, + "log_odds_chosen": 0.7400184869766235, + "log_odds_ratio": -0.46396252512931824, + "logits/chosen": 0.18774618208408356, + "logits/rejected": 0.041869793087244034, + "logps/chosen": -1.439223051071167, + "logps/rejected": -2.082319736480713, + "loss": 2.9231, + "nll_loss": 2.8767247200012207, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14392229914665222, + "rewards/margins": 0.06430967152118683, + "rewards/rejected": -0.20823195576667786, + "step": 353 + }, + { + "epoch": 0.22021772939346812, + "grad_norm": 0.48990121483802795, + "learning_rate": 4.115e-05, + "log_odds_chosen": 1.467127799987793, + "log_odds_ratio": -0.3306187391281128, + "logits/chosen": 0.21457459032535553, + "logits/rejected": -0.0008951930794864893, + "logps/chosen": -1.305351734161377, + "logps/rejected": -2.5240726470947266, + "loss": 3.1787, + "nll_loss": 3.1456456184387207, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13053518533706665, + "rewards/margins": 0.12187208235263824, + "rewards/rejected": -0.2524072527885437, + "step": 354 + }, + { + "epoch": 0.2208398133748056, + "grad_norm": 0.4076468348503113, + "learning_rate": 4.1125000000000004e-05, + "log_odds_chosen": 3.02394962310791, + "log_odds_ratio": -0.1947459578514099, + "logits/chosen": 0.15360787510871887, + "logits/rejected": 0.13919305801391602, + "logps/chosen": -0.9943020939826965, + "logps/rejected": -3.5736258029937744, + "loss": 2.964, + "nll_loss": 2.944533348083496, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09943021088838577, + "rewards/margins": 0.2579323649406433, + "rewards/rejected": -0.3573625683784485, + "step": 355 + }, + { + "epoch": 0.22146189735614308, + "grad_norm": 0.4909219443798065, + "learning_rate": 4.11e-05, + "log_odds_chosen": 1.00634765625, + "log_odds_ratio": -0.43545883893966675, + "logits/chosen": 0.14945323765277863, + "logits/rejected": 0.019120551645755768, + "logps/chosen": -1.1865514516830444, + "logps/rejected": -2.035024642944336, + "loss": 3.0395, + "nll_loss": 2.995992660522461, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11865514516830444, + "rewards/margins": 0.08484730869531631, + "rewards/rejected": -0.20350246131420135, + "step": 356 + }, + { + "epoch": 0.22208398133748056, + "grad_norm": 0.7641182541847229, + "learning_rate": 4.1075e-05, + "log_odds_chosen": 1.090247392654419, + "log_odds_ratio": -0.37240341305732727, + "logits/chosen": 0.05090628191828728, + "logits/rejected": -0.03716772794723511, + "logps/chosen": -1.2965291738510132, + "logps/rejected": -2.2305068969726562, + "loss": 2.3492, + "nll_loss": 2.3119568824768066, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12965291738510132, + "rewards/margins": 0.09339778125286102, + "rewards/rejected": -0.22305069863796234, + "step": 357 + }, + { + "epoch": 0.22270606531881804, + "grad_norm": 0.5538558959960938, + "learning_rate": 4.105e-05, + "log_odds_chosen": 0.3712920546531677, + "log_odds_ratio": -0.6206623315811157, + "logits/chosen": 0.120402030646801, + "logits/rejected": -0.043820858001708984, + "logps/chosen": -1.399167537689209, + "logps/rejected": -1.6653398275375366, + "loss": 2.9107, + "nll_loss": 2.848662853240967, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13991674780845642, + "rewards/margins": 0.02661724016070366, + "rewards/rejected": -0.16653397679328918, + "step": 358 + }, + { + "epoch": 0.22332814930015552, + "grad_norm": 0.4002559185028076, + "learning_rate": 4.1025e-05, + "log_odds_chosen": 0.884466290473938, + "log_odds_ratio": -0.485196590423584, + "logits/chosen": 0.16781805455684662, + "logits/rejected": 0.09334772825241089, + "logps/chosen": -1.0677001476287842, + "logps/rejected": -1.5183968544006348, + "loss": 3.1326, + "nll_loss": 3.084113597869873, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10677002370357513, + "rewards/margins": 0.04506966099143028, + "rewards/rejected": -0.15183967351913452, + "step": 359 + }, + { + "epoch": 0.223950233281493, + "grad_norm": 0.45029738545417786, + "learning_rate": 4.1e-05, + "log_odds_chosen": 0.6543417572975159, + "log_odds_ratio": -0.5618710517883301, + "logits/chosen": 0.2441825568675995, + "logits/rejected": 0.06579945981502533, + "logps/chosen": -1.5026423931121826, + "logps/rejected": -2.0812129974365234, + "loss": 3.486, + "nll_loss": 3.4297986030578613, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15026423335075378, + "rewards/margins": 0.05785707011818886, + "rewards/rejected": -0.20812132954597473, + "step": 360 + }, + { + "epoch": 0.22457231726283047, + "grad_norm": 0.40222129225730896, + "learning_rate": 4.0975e-05, + "log_odds_chosen": 0.6278929710388184, + "log_odds_ratio": -0.5374911427497864, + "logits/chosen": 0.1057760939002037, + "logits/rejected": 0.1274702399969101, + "logps/chosen": -1.3516254425048828, + "logps/rejected": -1.9070372581481934, + "loss": 3.0704, + "nll_loss": 3.0166256427764893, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13516254723072052, + "rewards/margins": 0.05554117262363434, + "rewards/rejected": -0.19070371985435486, + "step": 361 + }, + { + "epoch": 0.22519440124416795, + "grad_norm": 0.40902429819107056, + "learning_rate": 4.095e-05, + "log_odds_chosen": 1.3791449069976807, + "log_odds_ratio": -0.3504829406738281, + "logits/chosen": 0.09938657283782959, + "logits/rejected": -0.04903809353709221, + "logps/chosen": -0.8722810745239258, + "logps/rejected": -1.6328226327896118, + "loss": 3.063, + "nll_loss": 3.027944326400757, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08722810447216034, + "rewards/margins": 0.0760541558265686, + "rewards/rejected": -0.16328227519989014, + "step": 362 + }, + { + "epoch": 0.22581648522550543, + "grad_norm": 0.5102895498275757, + "learning_rate": 4.0925000000000005e-05, + "log_odds_chosen": 1.6143276691436768, + "log_odds_ratio": -0.24948588013648987, + "logits/chosen": 0.1433638036251068, + "logits/rejected": 0.04843227565288544, + "logps/chosen": -1.1994624137878418, + "logps/rejected": -2.5450100898742676, + "loss": 2.8281, + "nll_loss": 2.803130626678467, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11994623392820358, + "rewards/margins": 0.13455477356910706, + "rewards/rejected": -0.2545010447502136, + "step": 363 + }, + { + "epoch": 0.2264385692068429, + "grad_norm": 0.44875141978263855, + "learning_rate": 4.09e-05, + "log_odds_chosen": 0.5202021598815918, + "log_odds_ratio": -0.5619540214538574, + "logits/chosen": 0.09863647073507309, + "logits/rejected": -0.04804396256804466, + "logps/chosen": -1.4303956031799316, + "logps/rejected": -1.8165171146392822, + "loss": 2.7145, + "nll_loss": 2.658271312713623, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14303956925868988, + "rewards/margins": 0.038612157106399536, + "rewards/rejected": -0.1816517412662506, + "step": 364 + }, + { + "epoch": 0.22706065318818042, + "grad_norm": 0.4456920325756073, + "learning_rate": 4.0875000000000004e-05, + "log_odds_chosen": 1.450744390487671, + "log_odds_ratio": -0.3320462107658386, + "logits/chosen": 0.09102889895439148, + "logits/rejected": -0.05849744379520416, + "logps/chosen": -1.1014940738677979, + "logps/rejected": -2.2115225791931152, + "loss": 3.3094, + "nll_loss": 3.2761690616607666, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11014941334724426, + "rewards/margins": 0.11100286990404129, + "rewards/rejected": -0.22115227580070496, + "step": 365 + }, + { + "epoch": 0.2276827371695179, + "grad_norm": 0.45694804191589355, + "learning_rate": 4.085e-05, + "log_odds_chosen": 0.8244272470474243, + "log_odds_ratio": -0.4660112261772156, + "logits/chosen": 0.12547065317630768, + "logits/rejected": -0.11931107938289642, + "logps/chosen": -1.0817716121673584, + "logps/rejected": -1.7403573989868164, + "loss": 2.971, + "nll_loss": 2.9244022369384766, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10817715525627136, + "rewards/margins": 0.06585858762264252, + "rewards/rejected": -0.17403574287891388, + "step": 366 + }, + { + "epoch": 0.22830482115085537, + "grad_norm": 0.5407382249832153, + "learning_rate": 4.0825e-05, + "log_odds_chosen": 2.503481864929199, + "log_odds_ratio": -0.13278703391551971, + "logits/chosen": 0.10355640947818756, + "logits/rejected": -0.04844285547733307, + "logps/chosen": -0.958575963973999, + "logps/rejected": -2.8335940837860107, + "loss": 2.8018, + "nll_loss": 2.788564682006836, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09585760533809662, + "rewards/margins": 0.18750178813934326, + "rewards/rejected": -0.2833594083786011, + "step": 367 + }, + { + "epoch": 0.22892690513219285, + "grad_norm": 0.6709921956062317, + "learning_rate": 4.08e-05, + "log_odds_chosen": 0.13700546324253082, + "log_odds_ratio": -0.6914812326431274, + "logits/chosen": 0.29757586121559143, + "logits/rejected": 0.058456920087337494, + "logps/chosen": -1.8119229078292847, + "logps/rejected": -1.8780629634857178, + "loss": 3.6408, + "nll_loss": 3.5716493129730225, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1811922788619995, + "rewards/margins": 0.006614033132791519, + "rewards/rejected": -0.18780632317066193, + "step": 368 + }, + { + "epoch": 0.22954898911353033, + "grad_norm": 0.5146610736846924, + "learning_rate": 4.0775e-05, + "log_odds_chosen": 2.349278688430786, + "log_odds_ratio": -0.31061261892318726, + "logits/chosen": 0.2917703092098236, + "logits/rejected": 0.0938117578625679, + "logps/chosen": -1.0622313022613525, + "logps/rejected": -3.1076037883758545, + "loss": 3.223, + "nll_loss": 3.1919517517089844, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10622313618659973, + "rewards/margins": 0.20453724265098572, + "rewards/rejected": -0.31076037883758545, + "step": 369 + }, + { + "epoch": 0.2301710730948678, + "grad_norm": 0.32432612776756287, + "learning_rate": 4.075e-05, + "log_odds_chosen": 1.531700849533081, + "log_odds_ratio": -0.26523175835609436, + "logits/chosen": 0.28544509410858154, + "logits/rejected": 0.16868844628334045, + "logps/chosen": -1.1210910081863403, + "logps/rejected": -2.238771915435791, + "loss": 3.6429, + "nll_loss": 3.61639666557312, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11210910975933075, + "rewards/margins": 0.11176810413599014, + "rewards/rejected": -0.2238771915435791, + "step": 370 + }, + { + "epoch": 0.2307931570762053, + "grad_norm": 0.4330305755138397, + "learning_rate": 4.0725e-05, + "log_odds_chosen": 1.4429383277893066, + "log_odds_ratio": -0.29110294580459595, + "logits/chosen": 0.26996076107025146, + "logits/rejected": 0.07536883652210236, + "logps/chosen": -1.1473499536514282, + "logps/rejected": -2.227917432785034, + "loss": 3.2222, + "nll_loss": 3.1930618286132812, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11473499983549118, + "rewards/margins": 0.10805673897266388, + "rewards/rejected": -0.22279173135757446, + "step": 371 + }, + { + "epoch": 0.23141524105754277, + "grad_norm": 0.5261003375053406, + "learning_rate": 4.07e-05, + "log_odds_chosen": 2.001164197921753, + "log_odds_ratio": -0.22053340077400208, + "logits/chosen": 0.22258536517620087, + "logits/rejected": -0.05359187722206116, + "logps/chosen": -0.984519362449646, + "logps/rejected": -2.612748146057129, + "loss": 3.3361, + "nll_loss": 3.314016342163086, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09845194220542908, + "rewards/margins": 0.1628229022026062, + "rewards/rejected": -0.2612748146057129, + "step": 372 + }, + { + "epoch": 0.23203732503888025, + "grad_norm": 0.5155344009399414, + "learning_rate": 4.0675e-05, + "log_odds_chosen": 0.6205844879150391, + "log_odds_ratio": -0.536081075668335, + "logits/chosen": 0.16748444736003876, + "logits/rejected": 0.03588568791747093, + "logps/chosen": -0.976515531539917, + "logps/rejected": -1.502734899520874, + "loss": 3.0293, + "nll_loss": 2.9756860733032227, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09765155613422394, + "rewards/margins": 0.052621930837631226, + "rewards/rejected": -0.15027348697185516, + "step": 373 + }, + { + "epoch": 0.23265940902021773, + "grad_norm": 0.46208807826042175, + "learning_rate": 4.065e-05, + "log_odds_chosen": 0.9866193532943726, + "log_odds_ratio": -0.41447627544403076, + "logits/chosen": 0.19046258926391602, + "logits/rejected": 0.1209864392876625, + "logps/chosen": -0.9953656196594238, + "logps/rejected": -1.6837455034255981, + "loss": 3.1588, + "nll_loss": 3.1173107624053955, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09953656792640686, + "rewards/margins": 0.0688379779458046, + "rewards/rejected": -0.16837453842163086, + "step": 374 + }, + { + "epoch": 0.2332814930015552, + "grad_norm": 0.5262720584869385, + "learning_rate": 4.0625000000000005e-05, + "log_odds_chosen": 1.1617014408111572, + "log_odds_ratio": -0.3742392063140869, + "logits/chosen": 0.2684021592140198, + "logits/rejected": 0.0713948905467987, + "logps/chosen": -1.4048610925674438, + "logps/rejected": -2.383072853088379, + "loss": 3.3322, + "nll_loss": 3.29473876953125, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14048609137535095, + "rewards/margins": 0.0978211984038353, + "rewards/rejected": -0.23830729722976685, + "step": 375 + }, + { + "epoch": 0.23390357698289269, + "grad_norm": 0.4657245874404907, + "learning_rate": 4.0600000000000004e-05, + "log_odds_chosen": 1.0480836629867554, + "log_odds_ratio": -0.3715401887893677, + "logits/chosen": 0.17920681834220886, + "logits/rejected": -0.03246612474322319, + "logps/chosen": -1.4259849786758423, + "logps/rejected": -2.313983201980591, + "loss": 3.0117, + "nll_loss": 2.9745419025421143, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14259850978851318, + "rewards/margins": 0.08879982680082321, + "rewards/rejected": -0.2313983291387558, + "step": 376 + }, + { + "epoch": 0.23452566096423016, + "grad_norm": 0.4098886549472809, + "learning_rate": 4.0575000000000004e-05, + "log_odds_chosen": 0.9914601445198059, + "log_odds_ratio": -0.43344154953956604, + "logits/chosen": 0.24701879918575287, + "logits/rejected": 0.019207902252674103, + "logps/chosen": -1.3967047929763794, + "logps/rejected": -2.260988235473633, + "loss": 3.6364, + "nll_loss": 3.5931003093719482, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1396704763174057, + "rewards/margins": 0.08642837405204773, + "rewards/rejected": -0.22609883546829224, + "step": 377 + }, + { + "epoch": 0.23514774494556764, + "grad_norm": 0.47420018911361694, + "learning_rate": 4.055e-05, + "log_odds_chosen": 2.0201539993286133, + "log_odds_ratio": -0.2169860154390335, + "logits/chosen": 0.25134286284446716, + "logits/rejected": 0.0921395868062973, + "logps/chosen": -1.110857605934143, + "logps/rejected": -2.820281744003296, + "loss": 2.9052, + "nll_loss": 2.8835182189941406, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11108576506376266, + "rewards/margins": 0.17094242572784424, + "rewards/rejected": -0.2820281982421875, + "step": 378 + }, + { + "epoch": 0.23576982892690512, + "grad_norm": 0.6942557096481323, + "learning_rate": 4.0525e-05, + "log_odds_chosen": 1.0329240560531616, + "log_odds_ratio": -0.7378867268562317, + "logits/chosen": 0.11842384189367294, + "logits/rejected": 0.05716584250330925, + "logps/chosen": -1.7523798942565918, + "logps/rejected": -2.5439324378967285, + "loss": 2.6703, + "nll_loss": 2.596536159515381, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1752379834651947, + "rewards/margins": 0.07915525138378143, + "rewards/rejected": -0.25439321994781494, + "step": 379 + }, + { + "epoch": 0.2363919129082426, + "grad_norm": 0.49551472067832947, + "learning_rate": 4.05e-05, + "log_odds_chosen": 1.2499074935913086, + "log_odds_ratio": -0.4396931529045105, + "logits/chosen": 0.3040961027145386, + "logits/rejected": 0.13865968585014343, + "logps/chosen": -1.3086928129196167, + "logps/rejected": -2.284111499786377, + "loss": 3.2365, + "nll_loss": 3.192500114440918, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1308692842721939, + "rewards/margins": 0.09754188358783722, + "rewards/rejected": -0.22841116786003113, + "step": 380 + }, + { + "epoch": 0.23701399688958008, + "grad_norm": 0.4651765823364258, + "learning_rate": 4.0475e-05, + "log_odds_chosen": 0.9740829467773438, + "log_odds_ratio": -0.4101206362247467, + "logits/chosen": 0.18327367305755615, + "logits/rejected": -0.007276562973856926, + "logps/chosen": -1.0384430885314941, + "logps/rejected": -1.7929588556289673, + "loss": 2.9892, + "nll_loss": 2.9481401443481445, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10384431481361389, + "rewards/margins": 0.0754515752196312, + "rewards/rejected": -0.17929589748382568, + "step": 381 + }, + { + "epoch": 0.2376360808709176, + "grad_norm": 0.4907761216163635, + "learning_rate": 4.045000000000001e-05, + "log_odds_chosen": 0.9015824794769287, + "log_odds_ratio": -0.4331977963447571, + "logits/chosen": 0.11043369770050049, + "logits/rejected": 0.11448623239994049, + "logps/chosen": -1.1956536769866943, + "logps/rejected": -1.9301912784576416, + "loss": 3.053, + "nll_loss": 3.009702682495117, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11956537514925003, + "rewards/margins": 0.07345376163721085, + "rewards/rejected": -0.19301912188529968, + "step": 382 + }, + { + "epoch": 0.23825816485225507, + "grad_norm": 0.45944109559059143, + "learning_rate": 4.0425e-05, + "log_odds_chosen": 0.9134513735771179, + "log_odds_ratio": -0.4373936057090759, + "logits/chosen": 0.05043463408946991, + "logits/rejected": 0.10286575555801392, + "logps/chosen": -1.2276078462600708, + "logps/rejected": -1.9750827550888062, + "loss": 2.8257, + "nll_loss": 2.781928777694702, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12276078015565872, + "rewards/margins": 0.0747474879026413, + "rewards/rejected": -0.19750827550888062, + "step": 383 + }, + { + "epoch": 0.23888024883359255, + "grad_norm": 0.5201630592346191, + "learning_rate": 4.0400000000000006e-05, + "log_odds_chosen": 2.114027738571167, + "log_odds_ratio": -0.15464414656162262, + "logits/chosen": 0.2518041431903839, + "logits/rejected": 0.06087159365415573, + "logps/chosen": -0.7840594053268433, + "logps/rejected": -2.2629692554473877, + "loss": 3.3323, + "nll_loss": 3.316868782043457, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0784059390425682, + "rewards/margins": 0.14789099991321564, + "rewards/rejected": -0.22629694640636444, + "step": 384 + }, + { + "epoch": 0.23950233281493002, + "grad_norm": 0.44370707869529724, + "learning_rate": 4.0375e-05, + "log_odds_chosen": 1.3410742282867432, + "log_odds_ratio": -0.322849839925766, + "logits/chosen": -0.04521143063902855, + "logits/rejected": -0.05069145932793617, + "logps/chosen": -1.05382239818573, + "logps/rejected": -2.046505928039551, + "loss": 2.6729, + "nll_loss": 2.6405997276306152, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10538224130868912, + "rewards/margins": 0.0992683619260788, + "rewards/rejected": -0.2046506106853485, + "step": 385 + }, + { + "epoch": 0.2401244167962675, + "grad_norm": 0.40675365924835205, + "learning_rate": 4.0350000000000005e-05, + "log_odds_chosen": 1.3835175037384033, + "log_odds_ratio": -0.26051661372184753, + "logits/chosen": 0.23836039006710052, + "logits/rejected": 0.06631730496883392, + "logps/chosen": -1.0143346786499023, + "logps/rejected": -2.0367674827575684, + "loss": 3.5034, + "nll_loss": 3.4773647785186768, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10143347829580307, + "rewards/margins": 0.10224328190088272, + "rewards/rejected": -0.2036767452955246, + "step": 386 + }, + { + "epoch": 0.24074650077760498, + "grad_norm": 0.46325162053108215, + "learning_rate": 4.0325000000000004e-05, + "log_odds_chosen": 0.7983840107917786, + "log_odds_ratio": -0.5282416939735413, + "logits/chosen": 0.2139953374862671, + "logits/rejected": 0.19283059239387512, + "logps/chosen": -1.2492709159851074, + "logps/rejected": -1.9619114398956299, + "loss": 2.8089, + "nll_loss": 2.756054639816284, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1249270886182785, + "rewards/margins": 0.07126405835151672, + "rewards/rejected": -0.19619114696979523, + "step": 387 + }, + { + "epoch": 0.24136858475894246, + "grad_norm": 0.5103418231010437, + "learning_rate": 4.0300000000000004e-05, + "log_odds_chosen": 1.5594842433929443, + "log_odds_ratio": -0.2448807656764984, + "logits/chosen": 0.17046277225017548, + "logits/rejected": 0.07869112491607666, + "logps/chosen": -1.0376362800598145, + "logps/rejected": -2.278496503829956, + "loss": 2.7251, + "nll_loss": 2.7006213665008545, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1037636250257492, + "rewards/margins": 0.12408602982759476, + "rewards/rejected": -0.22784966230392456, + "step": 388 + }, + { + "epoch": 0.24199066874027994, + "grad_norm": 0.3780629634857178, + "learning_rate": 4.0275e-05, + "log_odds_chosen": 1.3771535158157349, + "log_odds_ratio": -0.3319317102432251, + "logits/chosen": 0.32596805691719055, + "logits/rejected": 0.1879710555076599, + "logps/chosen": -1.2158101797103882, + "logps/rejected": -2.255094289779663, + "loss": 3.3506, + "nll_loss": 3.3174211978912354, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12158101797103882, + "rewards/margins": 0.10392840951681137, + "rewards/rejected": -0.22550944983959198, + "step": 389 + }, + { + "epoch": 0.24261275272161742, + "grad_norm": 0.4106887876987457, + "learning_rate": 4.025e-05, + "log_odds_chosen": 0.6944050788879395, + "log_odds_ratio": -0.46452128887176514, + "logits/chosen": 0.10568028688430786, + "logits/rejected": 0.0720917209982872, + "logps/chosen": -1.2601232528686523, + "logps/rejected": -1.7703832387924194, + "loss": 2.8362, + "nll_loss": 2.789783239364624, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12601232528686523, + "rewards/margins": 0.051026009023189545, + "rewards/rejected": -0.17703832685947418, + "step": 390 + }, + { + "epoch": 0.2432348367029549, + "grad_norm": 0.39343398809432983, + "learning_rate": 4.0225e-05, + "log_odds_chosen": 0.9674738645553589, + "log_odds_ratio": -0.5284614562988281, + "logits/chosen": 0.10175783932209015, + "logits/rejected": 0.03822772949934006, + "logps/chosen": -1.2551448345184326, + "logps/rejected": -2.0882956981658936, + "loss": 3.0856, + "nll_loss": 3.0327351093292236, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12551449239253998, + "rewards/margins": 0.08331510424613953, + "rewards/rejected": -0.2088295817375183, + "step": 391 + }, + { + "epoch": 0.24385692068429238, + "grad_norm": 0.4384991228580475, + "learning_rate": 4.02e-05, + "log_odds_chosen": 0.7777218818664551, + "log_odds_ratio": -0.4195319414138794, + "logits/chosen": 0.3055810332298279, + "logits/rejected": 0.05448678508400917, + "logps/chosen": -1.110863447189331, + "logps/rejected": -1.6843622922897339, + "loss": 3.2888, + "nll_loss": 3.2468209266662598, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11108633875846863, + "rewards/margins": 0.05734989792108536, + "rewards/rejected": -0.1684362292289734, + "step": 392 + }, + { + "epoch": 0.24447900466562986, + "grad_norm": 0.5314688682556152, + "learning_rate": 4.0175e-05, + "log_odds_chosen": 0.9773632884025574, + "log_odds_ratio": -0.4269845485687256, + "logits/chosen": 0.17569248378276825, + "logits/rejected": 0.1796281337738037, + "logps/chosen": -1.25931978225708, + "logps/rejected": -2.1036195755004883, + "loss": 2.7412, + "nll_loss": 2.6984646320343018, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.125931978225708, + "rewards/margins": 0.08442999422550201, + "rewards/rejected": -0.21036197245121002, + "step": 393 + }, + { + "epoch": 0.24510108864696734, + "grad_norm": 0.5598838329315186, + "learning_rate": 4.015000000000001e-05, + "log_odds_chosen": 1.459074854850769, + "log_odds_ratio": -0.30608847737312317, + "logits/chosen": 0.1628699004650116, + "logits/rejected": 0.10045505315065384, + "logps/chosen": -1.0321791172027588, + "logps/rejected": -2.095522880554199, + "loss": 2.6775, + "nll_loss": 2.646920680999756, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10321791470050812, + "rewards/margins": 0.10633436590433121, + "rewards/rejected": -0.20955227315425873, + "step": 394 + }, + { + "epoch": 0.24572317262830481, + "grad_norm": 0.31368446350097656, + "learning_rate": 4.0125e-05, + "log_odds_chosen": 1.034085750579834, + "log_odds_ratio": -0.4138377904891968, + "logits/chosen": 0.2146773636341095, + "logits/rejected": 0.06570446491241455, + "logps/chosen": -0.9849491715431213, + "logps/rejected": -1.6467113494873047, + "loss": 3.8255, + "nll_loss": 3.784114360809326, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09849491715431213, + "rewards/margins": 0.06617622822523117, + "rewards/rejected": -0.1646711528301239, + "step": 395 + }, + { + "epoch": 0.2463452566096423, + "grad_norm": 0.5984493494033813, + "learning_rate": 4.0100000000000006e-05, + "log_odds_chosen": 1.0793639421463013, + "log_odds_ratio": -0.47389787435531616, + "logits/chosen": 0.25743645429611206, + "logits/rejected": 0.023621711879968643, + "logps/chosen": -1.2558321952819824, + "logps/rejected": -2.103111505508423, + "loss": 3.2582, + "nll_loss": 3.2107701301574707, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.125583216547966, + "rewards/margins": 0.084727942943573, + "rewards/rejected": -0.210311159491539, + "step": 396 + }, + { + "epoch": 0.24696734059097977, + "grad_norm": 0.37724167108535767, + "learning_rate": 4.0075e-05, + "log_odds_chosen": 0.8990364670753479, + "log_odds_ratio": -0.4033519923686981, + "logits/chosen": 0.26665544509887695, + "logits/rejected": 0.15528859198093414, + "logps/chosen": -1.1553454399108887, + "logps/rejected": -1.854149580001831, + "loss": 3.484, + "nll_loss": 3.443643093109131, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11553453654050827, + "rewards/margins": 0.0698804259300232, + "rewards/rejected": -0.18541496992111206, + "step": 397 + }, + { + "epoch": 0.24758942457231725, + "grad_norm": 0.8459545373916626, + "learning_rate": 4.0050000000000004e-05, + "log_odds_chosen": 1.516906976699829, + "log_odds_ratio": -0.2759634852409363, + "logits/chosen": 0.23380643129348755, + "logits/rejected": 0.14687879383563995, + "logps/chosen": -1.2292195558547974, + "logps/rejected": -2.45884108543396, + "loss": 2.9524, + "nll_loss": 2.924772024154663, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12292195856571198, + "rewards/margins": 0.12296216189861298, + "rewards/rejected": -0.24588412046432495, + "step": 398 + }, + { + "epoch": 0.24821150855365473, + "grad_norm": 0.7587859630584717, + "learning_rate": 4.0025000000000004e-05, + "log_odds_chosen": 1.5884180068969727, + "log_odds_ratio": -0.3095462918281555, + "logits/chosen": 0.18802863359451294, + "logits/rejected": -0.01595812290906906, + "logps/chosen": -1.3920193910598755, + "logps/rejected": -2.773435115814209, + "loss": 3.247, + "nll_loss": 3.2160043716430664, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13920193910598755, + "rewards/margins": 0.13814157247543335, + "rewards/rejected": -0.2773435115814209, + "step": 399 + }, + { + "epoch": 0.24883359253499224, + "grad_norm": 0.7157498598098755, + "learning_rate": 4e-05, + "log_odds_chosen": 0.774174690246582, + "log_odds_ratio": -0.6406145691871643, + "logits/chosen": 0.3137771189212799, + "logits/rejected": 0.028129838407039642, + "logps/chosen": -1.4273900985717773, + "logps/rejected": -2.0852622985839844, + "loss": 4.025, + "nll_loss": 3.9609427452087402, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.14273901283740997, + "rewards/margins": 0.06578721851110458, + "rewards/rejected": -0.20852622389793396, + "step": 400 + }, + { + "epoch": 0.24945567651632972, + "grad_norm": 0.6268599033355713, + "learning_rate": 3.9975e-05, + "log_odds_chosen": 1.2294279336929321, + "log_odds_ratio": -0.4779622554779053, + "logits/chosen": 0.16509678959846497, + "logits/rejected": 0.07056587189435959, + "logps/chosen": -1.2676770687103271, + "logps/rejected": -2.3259618282318115, + "loss": 3.3218, + "nll_loss": 3.274020195007324, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12676770985126495, + "rewards/margins": 0.10582847893238068, + "rewards/rejected": -0.23259618878364563, + "step": 401 + }, + { + "epoch": 0.25007776049766717, + "grad_norm": 0.5052048563957214, + "learning_rate": 3.995e-05, + "log_odds_chosen": 1.3162705898284912, + "log_odds_ratio": -0.32979390025138855, + "logits/chosen": 0.14668475091457367, + "logits/rejected": 0.01875954680144787, + "logps/chosen": -1.279245138168335, + "logps/rejected": -2.367394208908081, + "loss": 3.0466, + "nll_loss": 3.013622522354126, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12792451679706573, + "rewards/margins": 0.10881491005420685, + "rewards/rejected": -0.23673942685127258, + "step": 402 + }, + { + "epoch": 0.2506998444790047, + "grad_norm": 0.586941659450531, + "learning_rate": 3.9925e-05, + "log_odds_chosen": 0.97333163022995, + "log_odds_ratio": -0.4262157678604126, + "logits/chosen": 0.199526846408844, + "logits/rejected": 0.12170709669589996, + "logps/chosen": -1.0942950248718262, + "logps/rejected": -1.8334949016571045, + "loss": 2.9178, + "nll_loss": 2.8751866817474365, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1094295084476471, + "rewards/margins": 0.07391998916864395, + "rewards/rejected": -0.18334949016571045, + "step": 403 + }, + { + "epoch": 0.2513219284603421, + "grad_norm": 0.7138959765434265, + "learning_rate": 3.99e-05, + "log_odds_chosen": 1.5978549718856812, + "log_odds_ratio": -0.6236208081245422, + "logits/chosen": 0.08655344694852829, + "logits/rejected": 0.06523916870355606, + "logps/chosen": -1.0683348178863525, + "logps/rejected": -2.3083322048187256, + "loss": 3.0347, + "nll_loss": 2.972341537475586, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10683348774909973, + "rewards/margins": 0.12399972975254059, + "rewards/rejected": -0.2308332324028015, + "step": 404 + }, + { + "epoch": 0.25194401244167963, + "grad_norm": 0.4803338050842285, + "learning_rate": 3.9875e-05, + "log_odds_chosen": 0.7191067337989807, + "log_odds_ratio": -0.4708095192909241, + "logits/chosen": 0.049791183322668076, + "logits/rejected": -0.03897171467542648, + "logps/chosen": -1.1012663841247559, + "logps/rejected": -1.5942356586456299, + "loss": 3.0085, + "nll_loss": 2.961468458175659, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11012664437294006, + "rewards/margins": 0.049296922981739044, + "rewards/rejected": -0.1594235748052597, + "step": 405 + }, + { + "epoch": 0.2525660964230171, + "grad_norm": 0.5744175314903259, + "learning_rate": 3.9850000000000006e-05, + "log_odds_chosen": 0.09449946880340576, + "log_odds_ratio": -0.6956384778022766, + "logits/chosen": 0.15730737149715424, + "logits/rejected": -0.03350464627146721, + "logps/chosen": -1.461653709411621, + "logps/rejected": -1.5220292806625366, + "loss": 3.3611, + "nll_loss": 3.2915573120117188, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1461653709411621, + "rewards/margins": 0.006037542596459389, + "rewards/rejected": -0.15220291912555695, + "step": 406 + }, + { + "epoch": 0.2531881804043546, + "grad_norm": 0.4293562173843384, + "learning_rate": 3.9825e-05, + "log_odds_chosen": 0.5901978611946106, + "log_odds_ratio": -0.5310933589935303, + "logits/chosen": 0.02693868800997734, + "logits/rejected": -0.0009972341358661652, + "logps/chosen": -1.046227216720581, + "logps/rejected": -1.3914040327072144, + "loss": 2.7241, + "nll_loss": 2.670952320098877, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1046227216720581, + "rewards/margins": 0.03451768681406975, + "rewards/rejected": -0.13914041221141815, + "step": 407 + }, + { + "epoch": 0.25381026438569204, + "grad_norm": 1.1754698753356934, + "learning_rate": 3.9800000000000005e-05, + "log_odds_chosen": 1.0236449241638184, + "log_odds_ratio": -0.46178126335144043, + "logits/chosen": 0.1028258204460144, + "logits/rejected": -0.047515541315078735, + "logps/chosen": -1.7946947813034058, + "logps/rejected": -2.7080202102661133, + "loss": 2.8661, + "nll_loss": 2.8199520111083984, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17946948111057281, + "rewards/margins": 0.09133254736661911, + "rewards/rejected": -0.27080202102661133, + "step": 408 + }, + { + "epoch": 0.25443234836702955, + "grad_norm": 0.4363967180252075, + "learning_rate": 3.9775e-05, + "log_odds_chosen": 1.6175098419189453, + "log_odds_ratio": -0.37422215938568115, + "logits/chosen": 0.22522154450416565, + "logits/rejected": 0.0489632673561573, + "logps/chosen": -0.9020571112632751, + "logps/rejected": -1.936967134475708, + "loss": 3.776, + "nll_loss": 3.7385733127593994, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09020572155714035, + "rewards/margins": 0.10349100083112717, + "rewards/rejected": -0.1936967372894287, + "step": 409 + }, + { + "epoch": 0.25505443234836706, + "grad_norm": 0.594385027885437, + "learning_rate": 3.9750000000000004e-05, + "log_odds_chosen": 1.2434051036834717, + "log_odds_ratio": -0.32507485151290894, + "logits/chosen": 0.14571481943130493, + "logits/rejected": 0.02298334613442421, + "logps/chosen": -1.0427602529525757, + "logps/rejected": -1.886343240737915, + "loss": 3.0004, + "nll_loss": 2.9678754806518555, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10427601635456085, + "rewards/margins": 0.08435830473899841, + "rewards/rejected": -0.18863432109355927, + "step": 410 + }, + { + "epoch": 0.2556765163297045, + "grad_norm": 0.41299790143966675, + "learning_rate": 3.9725e-05, + "log_odds_chosen": 0.741004228591919, + "log_odds_ratio": -0.4465721845626831, + "logits/chosen": 0.18152594566345215, + "logits/rejected": 0.13067328929901123, + "logps/chosen": -1.281395435333252, + "logps/rejected": -1.903831958770752, + "loss": 3.0622, + "nll_loss": 3.0175766944885254, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12813954055309296, + "rewards/margins": 0.06224365532398224, + "rewards/rejected": -0.1903831958770752, + "step": 411 + }, + { + "epoch": 0.256298600311042, + "grad_norm": 0.6275978088378906, + "learning_rate": 3.97e-05, + "log_odds_chosen": 0.0313977487385273, + "log_odds_ratio": -0.7415766716003418, + "logits/chosen": -0.03772881627082825, + "logits/rejected": 0.12028350681066513, + "logps/chosen": -1.4508265256881714, + "logps/rejected": -1.4406688213348389, + "loss": 2.3823, + "nll_loss": 2.30818772315979, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14508265256881714, + "rewards/margins": -0.001015770249068737, + "rewards/rejected": -0.14406687021255493, + "step": 412 + }, + { + "epoch": 0.25692068429237946, + "grad_norm": 0.4781981408596039, + "learning_rate": 3.9675e-05, + "log_odds_chosen": 1.5340921878814697, + "log_odds_ratio": -0.28959184885025024, + "logits/chosen": 0.34700697660446167, + "logits/rejected": 0.2925771176815033, + "logps/chosen": -1.3058507442474365, + "logps/rejected": -2.6288981437683105, + "loss": 3.491, + "nll_loss": 3.4620461463928223, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13058508932590485, + "rewards/margins": 0.13230472803115845, + "rewards/rejected": -0.2628898024559021, + "step": 413 + }, + { + "epoch": 0.25754276827371697, + "grad_norm": 0.494865357875824, + "learning_rate": 3.965e-05, + "log_odds_chosen": 1.5314979553222656, + "log_odds_ratio": -0.25367340445518494, + "logits/chosen": 0.18154457211494446, + "logits/rejected": 0.1370812952518463, + "logps/chosen": -0.9703949689865112, + "logps/rejected": -1.9872558116912842, + "loss": 3.0317, + "nll_loss": 3.006338119506836, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09703950583934784, + "rewards/margins": 0.10168607532978058, + "rewards/rejected": -0.19872558116912842, + "step": 414 + }, + { + "epoch": 0.2581648522550544, + "grad_norm": 0.45149925351142883, + "learning_rate": 3.9625e-05, + "log_odds_chosen": 0.49176663160324097, + "log_odds_ratio": -0.6137273907661438, + "logits/chosen": 0.19695249199867249, + "logits/rejected": 0.18039798736572266, + "logps/chosen": -1.330711841583252, + "logps/rejected": -1.8058357238769531, + "loss": 3.122, + "nll_loss": 3.0606541633605957, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1330711841583252, + "rewards/margins": 0.047512397170066833, + "rewards/rejected": -0.18058358132839203, + "step": 415 + }, + { + "epoch": 0.25878693623639193, + "grad_norm": 0.4677426517009735, + "learning_rate": 3.960000000000001e-05, + "log_odds_chosen": 1.4863536357879639, + "log_odds_ratio": -0.3277415931224823, + "logits/chosen": 0.3289963901042938, + "logits/rejected": 0.295125275850296, + "logps/chosen": -1.3954858779907227, + "logps/rejected": -2.662785530090332, + "loss": 3.3443, + "nll_loss": 3.3114874362945557, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13954858481884003, + "rewards/margins": 0.12672998011112213, + "rewards/rejected": -0.26627856492996216, + "step": 416 + }, + { + "epoch": 0.2594090202177294, + "grad_norm": 0.5604446530342102, + "learning_rate": 3.9575e-05, + "log_odds_chosen": 1.654098629951477, + "log_odds_ratio": -0.34370043873786926, + "logits/chosen": 0.44623494148254395, + "logits/rejected": 0.26756906509399414, + "logps/chosen": -1.5715718269348145, + "logps/rejected": -3.0832409858703613, + "loss": 3.74, + "nll_loss": 3.7056450843811035, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15715718269348145, + "rewards/margins": 0.1511669158935547, + "rewards/rejected": -0.30832409858703613, + "step": 417 + }, + { + "epoch": 0.2600311041990669, + "grad_norm": 0.5016684532165527, + "learning_rate": 3.9550000000000006e-05, + "log_odds_chosen": 1.0671627521514893, + "log_odds_ratio": -0.34129106998443604, + "logits/chosen": 0.09102344512939453, + "logits/rejected": 0.1018139123916626, + "logps/chosen": -1.1463091373443604, + "logps/rejected": -1.9536468982696533, + "loss": 2.8157, + "nll_loss": 2.781564474105835, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11463090777397156, + "rewards/margins": 0.0807337760925293, + "rewards/rejected": -0.19536468386650085, + "step": 418 + }, + { + "epoch": 0.26065318818040434, + "grad_norm": 0.9377952218055725, + "learning_rate": 3.9525e-05, + "log_odds_chosen": 2.2686891555786133, + "log_odds_ratio": -0.14280670881271362, + "logits/chosen": 0.29692742228507996, + "logits/rejected": 0.15223044157028198, + "logps/chosen": -1.185152292251587, + "logps/rejected": -3.158766269683838, + "loss": 2.8944, + "nll_loss": 2.880141258239746, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11851523816585541, + "rewards/margins": 0.19736140966415405, + "rewards/rejected": -0.31587663292884827, + "step": 419 + }, + { + "epoch": 0.26127527216174184, + "grad_norm": 0.5251947045326233, + "learning_rate": 3.9500000000000005e-05, + "log_odds_chosen": 1.5507677793502808, + "log_odds_ratio": -0.3057706654071808, + "logits/chosen": 0.32404136657714844, + "logits/rejected": 0.18444237112998962, + "logps/chosen": -1.1873574256896973, + "logps/rejected": -2.446171522140503, + "loss": 3.1758, + "nll_loss": 3.1452338695526123, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11873574554920197, + "rewards/margins": 0.1258814036846161, + "rewards/rejected": -0.24461714923381805, + "step": 420 + }, + { + "epoch": 0.2618973561430793, + "grad_norm": 0.4308774471282959, + "learning_rate": 3.9475000000000004e-05, + "log_odds_chosen": 0.3947753608226776, + "log_odds_ratio": -0.5612890124320984, + "logits/chosen": 0.13869617879390717, + "logits/rejected": 0.044950664043426514, + "logps/chosen": -1.2930006980895996, + "logps/rejected": -1.6258165836334229, + "loss": 2.9868, + "nll_loss": 2.9306952953338623, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.129300057888031, + "rewards/margins": 0.0332816019654274, + "rewards/rejected": -0.162581667304039, + "step": 421 + }, + { + "epoch": 0.2625194401244168, + "grad_norm": 0.5830161571502686, + "learning_rate": 3.9450000000000003e-05, + "log_odds_chosen": 1.1204309463500977, + "log_odds_ratio": -0.3383479714393616, + "logits/chosen": 0.07506772875785828, + "logits/rejected": -0.03575318679213524, + "logps/chosen": -1.3158361911773682, + "logps/rejected": -2.236865997314453, + "loss": 2.7736, + "nll_loss": 2.7397449016571045, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13158363103866577, + "rewards/margins": 0.09210299700498581, + "rewards/rejected": -0.22368663549423218, + "step": 422 + }, + { + "epoch": 0.26314152410575425, + "grad_norm": 0.44899922609329224, + "learning_rate": 3.9425e-05, + "log_odds_chosen": 0.8703987002372742, + "log_odds_ratio": -0.5454012155532837, + "logits/chosen": 0.2875048816204071, + "logits/rejected": 0.11582295596599579, + "logps/chosen": -1.363781213760376, + "logps/rejected": -2.0890636444091797, + "loss": 3.2192, + "nll_loss": 3.1646461486816406, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13637810945510864, + "rewards/margins": 0.07252823561429977, + "rewards/rejected": -0.208906352519989, + "step": 423 + }, + { + "epoch": 0.26376360808709176, + "grad_norm": 0.5044350028038025, + "learning_rate": 3.94e-05, + "log_odds_chosen": 1.7807879447937012, + "log_odds_ratio": -0.2641463577747345, + "logits/chosen": 0.25100329518318176, + "logits/rejected": 0.0806911289691925, + "logps/chosen": -1.1451863050460815, + "logps/rejected": -2.675082206726074, + "loss": 2.879, + "nll_loss": 2.852630138397217, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11451863497495651, + "rewards/margins": 0.15298959612846375, + "rewards/rejected": -0.26750820875167847, + "step": 424 + }, + { + "epoch": 0.2643856920684292, + "grad_norm": 0.5463127493858337, + "learning_rate": 3.9375e-05, + "log_odds_chosen": 1.7012643814086914, + "log_odds_ratio": -0.3870460093021393, + "logits/chosen": 0.1893111914396286, + "logits/rejected": 0.12804879248142242, + "logps/chosen": -1.3288249969482422, + "logps/rejected": -2.868129253387451, + "loss": 3.3723, + "nll_loss": 3.3335795402526855, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1328825205564499, + "rewards/margins": 0.1539304256439209, + "rewards/rejected": -0.286812961101532, + "step": 425 + }, + { + "epoch": 0.2650077760497667, + "grad_norm": 0.8619997501373291, + "learning_rate": 3.935e-05, + "log_odds_chosen": 0.8874719738960266, + "log_odds_ratio": -0.7657593488693237, + "logits/chosen": 0.08044306933879852, + "logits/rejected": -0.008549835532903671, + "logps/chosen": -1.7457143068313599, + "logps/rejected": -2.4340968132019043, + "loss": 2.9367, + "nll_loss": 2.8601346015930176, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1745714396238327, + "rewards/margins": 0.06883823126554489, + "rewards/rejected": -0.2434096783399582, + "step": 426 + }, + { + "epoch": 0.2656298600311042, + "grad_norm": 0.5804911851882935, + "learning_rate": 3.9325e-05, + "log_odds_chosen": 1.0181046724319458, + "log_odds_ratio": -0.5276904106140137, + "logits/chosen": 0.20519691705703735, + "logits/rejected": 0.11686623841524124, + "logps/chosen": -1.3472039699554443, + "logps/rejected": -2.2560369968414307, + "loss": 3.1032, + "nll_loss": 3.050477981567383, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13472039997577667, + "rewards/margins": 0.0908832848072052, + "rewards/rejected": -0.22560366988182068, + "step": 427 + }, + { + "epoch": 0.2662519440124417, + "grad_norm": 0.4489701986312866, + "learning_rate": 3.9300000000000007e-05, + "log_odds_chosen": 1.4261120557785034, + "log_odds_ratio": -0.3344850540161133, + "logits/chosen": 0.06815585494041443, + "logits/rejected": 0.005228975787758827, + "logps/chosen": -1.0918066501617432, + "logps/rejected": -2.263209342956543, + "loss": 2.801, + "nll_loss": 2.767580509185791, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10918065905570984, + "rewards/margins": 0.1171402856707573, + "rewards/rejected": -0.22632096707820892, + "step": 428 + }, + { + "epoch": 0.2668740279937792, + "grad_norm": 0.6455628871917725, + "learning_rate": 3.9275e-05, + "log_odds_chosen": 1.4339535236358643, + "log_odds_ratio": -0.32922857999801636, + "logits/chosen": 0.24753251671791077, + "logits/rejected": 0.17592373490333557, + "logps/chosen": -1.4441932439804077, + "logps/rejected": -2.687839984893799, + "loss": 3.094, + "nll_loss": 3.0610883235931396, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14441931247711182, + "rewards/margins": 0.1243646889925003, + "rewards/rejected": -0.2687840163707733, + "step": 429 + }, + { + "epoch": 0.26749611197511663, + "grad_norm": 0.29282695055007935, + "learning_rate": 3.9250000000000005e-05, + "log_odds_chosen": 1.340848445892334, + "log_odds_ratio": -0.34622523188591003, + "logits/chosen": 0.22931736707687378, + "logits/rejected": 0.14595317840576172, + "logps/chosen": -1.1169357299804688, + "logps/rejected": -2.1826369762420654, + "loss": 3.4491, + "nll_loss": 3.4144582748413086, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11169356852769852, + "rewards/margins": 0.10657013207674026, + "rewards/rejected": -0.21826371550559998, + "step": 430 + }, + { + "epoch": 0.26811819595645414, + "grad_norm": 0.3759579062461853, + "learning_rate": 3.9225e-05, + "log_odds_chosen": 0.9186004996299744, + "log_odds_ratio": -0.4763484001159668, + "logits/chosen": 0.2613193690776825, + "logits/rejected": 0.11609724164009094, + "logps/chosen": -0.910784125328064, + "logps/rejected": -1.6030433177947998, + "loss": 3.5052, + "nll_loss": 3.4575998783111572, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09107840061187744, + "rewards/margins": 0.06922592967748642, + "rewards/rejected": -0.16030432283878326, + "step": 431 + }, + { + "epoch": 0.2687402799377916, + "grad_norm": 0.4951987564563751, + "learning_rate": 3.9200000000000004e-05, + "log_odds_chosen": 1.3083531856536865, + "log_odds_ratio": -0.3793916702270508, + "logits/chosen": 0.3077886700630188, + "logits/rejected": 0.05748777836561203, + "logps/chosen": -1.2742183208465576, + "logps/rejected": -2.2997703552246094, + "loss": 3.7485, + "nll_loss": 3.710557460784912, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12742182612419128, + "rewards/margins": 0.10255520790815353, + "rewards/rejected": -0.22997704148292542, + "step": 432 + }, + { + "epoch": 0.2693623639191291, + "grad_norm": 0.4444817006587982, + "learning_rate": 3.9175000000000004e-05, + "log_odds_chosen": 1.2291901111602783, + "log_odds_ratio": -0.35217738151550293, + "logits/chosen": 0.2221524715423584, + "logits/rejected": 0.05530994012951851, + "logps/chosen": -0.9905591011047363, + "logps/rejected": -1.9143832921981812, + "loss": 3.3073, + "nll_loss": 3.272057056427002, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09905591607093811, + "rewards/margins": 0.09238240122795105, + "rewards/rejected": -0.19143833220005035, + "step": 433 + }, + { + "epoch": 0.26998444790046655, + "grad_norm": 0.6299095749855042, + "learning_rate": 3.915e-05, + "log_odds_chosen": 1.615001916885376, + "log_odds_ratio": -0.3864189386367798, + "logits/chosen": 0.24696266651153564, + "logits/rejected": 0.18707673251628876, + "logps/chosen": -1.9024136066436768, + "logps/rejected": -3.3632092475891113, + "loss": 2.8607, + "nll_loss": 2.822024345397949, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.19024136662483215, + "rewards/margins": 0.14607959985733032, + "rewards/rejected": -0.3363209664821625, + "step": 434 + }, + { + "epoch": 0.27060653188180406, + "grad_norm": 0.5415927767753601, + "learning_rate": 3.9125e-05, + "log_odds_chosen": 4.024477481842041, + "log_odds_ratio": -0.14453338086605072, + "logits/chosen": 0.2853761315345764, + "logits/rejected": 0.15076112747192383, + "logps/chosen": -1.2033590078353882, + "logps/rejected": -4.912008285522461, + "loss": 2.7905, + "nll_loss": 2.776012897491455, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1203359067440033, + "rewards/margins": 0.3708649277687073, + "rewards/rejected": -0.4912008047103882, + "step": 435 + }, + { + "epoch": 0.2712286158631415, + "grad_norm": 0.6048150658607483, + "learning_rate": 3.91e-05, + "log_odds_chosen": 0.31674981117248535, + "log_odds_ratio": -0.6299794912338257, + "logits/chosen": 0.13437308371067047, + "logits/rejected": 0.002337227575480938, + "logps/chosen": -1.596144199371338, + "logps/rejected": -1.849827527999878, + "loss": 2.6519, + "nll_loss": 2.588890790939331, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1596144139766693, + "rewards/margins": 0.025368336588144302, + "rewards/rejected": -0.1849827617406845, + "step": 436 + }, + { + "epoch": 0.271850699844479, + "grad_norm": 0.44196486473083496, + "learning_rate": 3.9075e-05, + "log_odds_chosen": 1.744307041168213, + "log_odds_ratio": -0.26184773445129395, + "logits/chosen": 0.14910796284675598, + "logits/rejected": 0.08620636910200119, + "logps/chosen": -1.1879832744598389, + "logps/rejected": -2.592353343963623, + "loss": 2.8721, + "nll_loss": 2.845905303955078, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11879833042621613, + "rewards/margins": 0.14043700695037842, + "rewards/rejected": -0.25923532247543335, + "step": 437 + }, + { + "epoch": 0.27247278382581647, + "grad_norm": 0.5670360326766968, + "learning_rate": 3.905e-05, + "log_odds_chosen": 2.0719993114471436, + "log_odds_ratio": -0.2230648547410965, + "logits/chosen": 0.24322016537189484, + "logits/rejected": 0.2413720339536667, + "logps/chosen": -1.1990163326263428, + "logps/rejected": -2.9432950019836426, + "loss": 2.7375, + "nll_loss": 2.715202569961548, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1199016347527504, + "rewards/margins": 0.17442789673805237, + "rewards/rejected": -0.29432952404022217, + "step": 438 + }, + { + "epoch": 0.273094867807154, + "grad_norm": 0.4377356767654419, + "learning_rate": 3.9025e-05, + "log_odds_chosen": 2.4671876430511475, + "log_odds_ratio": -0.4254373013973236, + "logits/chosen": 0.3846701383590698, + "logits/rejected": 0.2242540866136551, + "logps/chosen": -0.9989205598831177, + "logps/rejected": -3.226830244064331, + "loss": 3.7535, + "nll_loss": 3.7109501361846924, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09989205747842789, + "rewards/margins": 0.22279097139835358, + "rewards/rejected": -0.3226830065250397, + "step": 439 + }, + { + "epoch": 0.2737169517884914, + "grad_norm": 0.6218655109405518, + "learning_rate": 3.9000000000000006e-05, + "log_odds_chosen": 1.7963087558746338, + "log_odds_ratio": -0.24013981223106384, + "logits/chosen": 0.22267626225948334, + "logits/rejected": 0.15220728516578674, + "logps/chosen": -1.4764683246612549, + "logps/rejected": -3.08296275138855, + "loss": 2.6897, + "nll_loss": 2.665696620941162, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14764682948589325, + "rewards/margins": 0.16064943373203278, + "rewards/rejected": -0.308296263217926, + "step": 440 + }, + { + "epoch": 0.27433903576982893, + "grad_norm": 0.4783235788345337, + "learning_rate": 3.8975e-05, + "log_odds_chosen": 3.722914934158325, + "log_odds_ratio": -0.1403815746307373, + "logits/chosen": 0.3612845838069916, + "logits/rejected": 0.3681058883666992, + "logps/chosen": -1.2698994874954224, + "logps/rejected": -4.6545090675354, + "loss": 3.0846, + "nll_loss": 3.0705621242523193, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1269899606704712, + "rewards/margins": 0.3384609520435333, + "rewards/rejected": -0.46545088291168213, + "step": 441 + }, + { + "epoch": 0.2749611197511664, + "grad_norm": 0.6110908389091492, + "learning_rate": 3.8950000000000005e-05, + "log_odds_chosen": 1.8417094945907593, + "log_odds_ratio": -0.23535192012786865, + "logits/chosen": 0.3509153127670288, + "logits/rejected": 0.10706407576799393, + "logps/chosen": -1.146148920059204, + "logps/rejected": -2.6451005935668945, + "loss": 3.4359, + "nll_loss": 3.4123191833496094, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11461488902568817, + "rewards/margins": 0.14989519119262695, + "rewards/rejected": -0.2645100951194763, + "step": 442 + }, + { + "epoch": 0.2755832037325039, + "grad_norm": 0.378417044878006, + "learning_rate": 3.8925e-05, + "log_odds_chosen": 1.9329280853271484, + "log_odds_ratio": -0.31626445055007935, + "logits/chosen": 0.2477678805589676, + "logits/rejected": 0.19056656956672668, + "logps/chosen": -0.9245951175689697, + "logps/rejected": -2.271580219268799, + "loss": 3.5382, + "nll_loss": 3.5066051483154297, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09245951473712921, + "rewards/margins": 0.1346985101699829, + "rewards/rejected": -0.22715803980827332, + "step": 443 + }, + { + "epoch": 0.27620528771384134, + "grad_norm": 0.45432165265083313, + "learning_rate": 3.8900000000000004e-05, + "log_odds_chosen": 2.492096185684204, + "log_odds_ratio": -0.20271611213684082, + "logits/chosen": 0.1072557121515274, + "logits/rejected": 0.1309928297996521, + "logps/chosen": -1.2175402641296387, + "logps/rejected": -3.431077003479004, + "loss": 2.5011, + "nll_loss": 2.480823040008545, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12175402045249939, + "rewards/margins": 0.2213537096977234, + "rewards/rejected": -0.3431077003479004, + "step": 444 + }, + { + "epoch": 0.27682737169517885, + "grad_norm": 0.4896884262561798, + "learning_rate": 3.8875e-05, + "log_odds_chosen": 2.018673896789551, + "log_odds_ratio": -0.23892150819301605, + "logits/chosen": 0.18642401695251465, + "logits/rejected": 0.1272982805967331, + "logps/chosen": -1.0007119178771973, + "logps/rejected": -2.652472496032715, + "loss": 2.8004, + "nll_loss": 2.776474714279175, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10007119923830032, + "rewards/margins": 0.16517606377601624, + "rewards/rejected": -0.26524725556373596, + "step": 445 + }, + { + "epoch": 0.27744945567651635, + "grad_norm": 0.4476563036441803, + "learning_rate": 3.885e-05, + "log_odds_chosen": 1.5900170803070068, + "log_odds_ratio": -0.3700055480003357, + "logits/chosen": 0.41444650292396545, + "logits/rejected": 0.26796096563339233, + "logps/chosen": -1.4082850217819214, + "logps/rejected": -2.710878372192383, + "loss": 3.8047, + "nll_loss": 3.7677254676818848, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14082849025726318, + "rewards/margins": 0.13025934994220734, + "rewards/rejected": -0.2710878551006317, + "step": 446 + }, + { + "epoch": 0.2780715396578538, + "grad_norm": 0.44968464970588684, + "learning_rate": 3.8825e-05, + "log_odds_chosen": 1.3255045413970947, + "log_odds_ratio": -0.2901913523674011, + "logits/chosen": 0.2749924957752228, + "logits/rejected": 0.2749479413032532, + "logps/chosen": -1.219002604484558, + "logps/rejected": -2.211946964263916, + "loss": 3.2752, + "nll_loss": 3.246131420135498, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12190026044845581, + "rewards/margins": 0.09929443150758743, + "rewards/rejected": -0.22119468450546265, + "step": 447 + }, + { + "epoch": 0.2786936236391913, + "grad_norm": 0.6323361992835999, + "learning_rate": 3.88e-05, + "log_odds_chosen": 2.376352071762085, + "log_odds_ratio": -0.16851823031902313, + "logits/chosen": 0.2564919590950012, + "logits/rejected": 0.3483697772026062, + "logps/chosen": -1.1505968570709229, + "logps/rejected": -3.2237629890441895, + "loss": 2.4031, + "nll_loss": 2.386216640472412, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11505968123674393, + "rewards/margins": 0.20731663703918457, + "rewards/rejected": -0.3223763108253479, + "step": 448 + }, + { + "epoch": 0.27931570762052876, + "grad_norm": 0.4380817115306854, + "learning_rate": 3.8775e-05, + "log_odds_chosen": 1.486014723777771, + "log_odds_ratio": -0.3191292881965637, + "logits/chosen": 0.35001346468925476, + "logits/rejected": 0.26604193449020386, + "logps/chosen": -1.0675163269042969, + "logps/rejected": -2.318084478378296, + "loss": 3.3673, + "nll_loss": 3.3354063034057617, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10675162822008133, + "rewards/margins": 0.12505681812763214, + "rewards/rejected": -0.23180843889713287, + "step": 449 + }, + { + "epoch": 0.27993779160186627, + "grad_norm": 0.5374330282211304, + "learning_rate": 3.875e-05, + "log_odds_chosen": 2.0906572341918945, + "log_odds_ratio": -0.2168201506137848, + "logits/chosen": 0.2877808213233948, + "logits/rejected": 0.3449041545391083, + "logps/chosen": -1.3126355409622192, + "logps/rejected": -2.9990274906158447, + "loss": 3.0555, + "nll_loss": 3.033806085586548, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13126355409622192, + "rewards/margins": 0.1686391979455948, + "rewards/rejected": -0.2999027371406555, + "step": 450 + }, + { + "epoch": 0.2805598755832037, + "grad_norm": 0.6015781760215759, + "learning_rate": 3.8725e-05, + "log_odds_chosen": 2.5022025108337402, + "log_odds_ratio": -0.19380271434783936, + "logits/chosen": 0.3818150460720062, + "logits/rejected": 0.35132819414138794, + "logps/chosen": -1.2525804042816162, + "logps/rejected": -3.4683022499084473, + "loss": 3.4237, + "nll_loss": 3.4043030738830566, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12525805830955505, + "rewards/margins": 0.22157219052314758, + "rewards/rejected": -0.34683024883270264, + "step": 451 + }, + { + "epoch": 0.28118195956454123, + "grad_norm": 0.5427994132041931, + "learning_rate": 3.8700000000000006e-05, + "log_odds_chosen": 1.5231120586395264, + "log_odds_ratio": -0.28168755769729614, + "logits/chosen": 0.3691140115261078, + "logits/rejected": 0.37976768612861633, + "logps/chosen": -1.2954881191253662, + "logps/rejected": -2.606727361679077, + "loss": 3.1849, + "nll_loss": 3.1567115783691406, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1295488178730011, + "rewards/margins": 0.13112393021583557, + "rewards/rejected": -0.26067274808883667, + "step": 452 + }, + { + "epoch": 0.2818040435458787, + "grad_norm": 0.6339102983474731, + "learning_rate": 3.8675e-05, + "log_odds_chosen": 4.337133407592773, + "log_odds_ratio": -0.21208305656909943, + "logits/chosen": 0.25384026765823364, + "logits/rejected": 0.20061862468719482, + "logps/chosen": -1.4610943794250488, + "logps/rejected": -5.5489044189453125, + "loss": 3.273, + "nll_loss": 3.251758098602295, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1461094468832016, + "rewards/margins": 0.4087810218334198, + "rewards/rejected": -0.5548904538154602, + "step": 453 + }, + { + "epoch": 0.2824261275272162, + "grad_norm": 1.4807077646255493, + "learning_rate": 3.8650000000000004e-05, + "log_odds_chosen": 2.719972848892212, + "log_odds_ratio": -0.1599472612142563, + "logits/chosen": 0.38763532042503357, + "logits/rejected": 0.3248913884162903, + "logps/chosen": -1.1536898612976074, + "logps/rejected": -3.492626905441284, + "loss": 3.2056, + "nll_loss": 3.1896491050720215, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11536898463964462, + "rewards/margins": 0.2338937222957611, + "rewards/rejected": -0.34926271438598633, + "step": 454 + }, + { + "epoch": 0.28304821150855364, + "grad_norm": 0.4285638928413391, + "learning_rate": 3.8625e-05, + "log_odds_chosen": 3.288182020187378, + "log_odds_ratio": -0.05790887400507927, + "logits/chosen": 0.2987384498119354, + "logits/rejected": 0.29059848189353943, + "logps/chosen": -1.2740700244903564, + "logps/rejected": -4.2349653244018555, + "loss": 3.027, + "nll_loss": 3.0212230682373047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1274069994688034, + "rewards/margins": 0.2960895299911499, + "rewards/rejected": -0.4234965443611145, + "step": 455 + }, + { + "epoch": 0.28367029548989114, + "grad_norm": 0.8652162551879883, + "learning_rate": 3.86e-05, + "log_odds_chosen": 1.5928850173950195, + "log_odds_ratio": -0.311477929353714, + "logits/chosen": 0.17478427290916443, + "logits/rejected": 0.1352236568927765, + "logps/chosen": -1.60139799118042, + "logps/rejected": -2.999675750732422, + "loss": 3.2718, + "nll_loss": 3.2406058311462402, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.160139799118042, + "rewards/margins": 0.13982778787612915, + "rewards/rejected": -0.29996758699417114, + "step": 456 + }, + { + "epoch": 0.2842923794712286, + "grad_norm": 0.3627634644508362, + "learning_rate": 3.8575e-05, + "log_odds_chosen": 2.181387186050415, + "log_odds_ratio": -0.3236231803894043, + "logits/chosen": 0.05983370169997215, + "logits/rejected": 0.02825239673256874, + "logps/chosen": -1.1143271923065186, + "logps/rejected": -2.988661766052246, + "loss": 3.0779, + "nll_loss": 3.0455780029296875, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11143273115158081, + "rewards/margins": 0.18743345141410828, + "rewards/rejected": -0.2988661825656891, + "step": 457 + }, + { + "epoch": 0.2849144634525661, + "grad_norm": 0.5619986057281494, + "learning_rate": 3.855e-05, + "log_odds_chosen": 1.8027219772338867, + "log_odds_ratio": -0.46999043226242065, + "logits/chosen": -0.01704075187444687, + "logits/rejected": 0.08329185098409653, + "logps/chosen": -1.0725547075271606, + "logps/rejected": -2.68211030960083, + "loss": 2.6529, + "nll_loss": 2.6059412956237793, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1072554737329483, + "rewards/margins": 0.16095557808876038, + "rewards/rejected": -0.2682110369205475, + "step": 458 + }, + { + "epoch": 0.28553654743390355, + "grad_norm": 0.4395221471786499, + "learning_rate": 3.8525e-05, + "log_odds_chosen": 3.0232295989990234, + "log_odds_ratio": -0.25974196195602417, + "logits/chosen": 0.24080729484558105, + "logits/rejected": 0.15837368369102478, + "logps/chosen": -0.9511310458183289, + "logps/rejected": -3.5407259464263916, + "loss": 3.3904, + "nll_loss": 3.364377021789551, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.095113106071949, + "rewards/margins": 0.25895950198173523, + "rewards/rejected": -0.35407260060310364, + "step": 459 + }, + { + "epoch": 0.28615863141524106, + "grad_norm": 0.39195650815963745, + "learning_rate": 3.85e-05, + "log_odds_chosen": 3.548543930053711, + "log_odds_ratio": -0.2414306104183197, + "logits/chosen": 0.20818278193473816, + "logits/rejected": 0.2002563178539276, + "logps/chosen": -0.9960837364196777, + "logps/rejected": -4.0006537437438965, + "loss": 3.0262, + "nll_loss": 3.002067804336548, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09960837662220001, + "rewards/margins": 0.3004570007324219, + "rewards/rejected": -0.40006542205810547, + "step": 460 + }, + { + "epoch": 0.2867807153965785, + "grad_norm": 0.6457217931747437, + "learning_rate": 3.8475e-05, + "log_odds_chosen": 2.5402870178222656, + "log_odds_ratio": -0.3850615918636322, + "logits/chosen": 0.11771635711193085, + "logits/rejected": 0.03514918312430382, + "logps/chosen": -1.3628007173538208, + "logps/rejected": -3.764535427093506, + "loss": 2.7094, + "nll_loss": 2.670870304107666, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13628007471561432, + "rewards/margins": 0.24017345905303955, + "rewards/rejected": -0.3764535188674927, + "step": 461 + }, + { + "epoch": 0.287402799377916, + "grad_norm": 0.5981135368347168, + "learning_rate": 3.845e-05, + "log_odds_chosen": 1.2403737306594849, + "log_odds_ratio": -0.6255621314048767, + "logits/chosen": 0.06767088919878006, + "logits/rejected": 0.015279887244105339, + "logps/chosen": -1.2029376029968262, + "logps/rejected": -2.2572832107543945, + "loss": 2.8872, + "nll_loss": 2.8246262073516846, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.12029377371072769, + "rewards/margins": 0.10543454438447952, + "rewards/rejected": -0.22572831809520721, + "step": 462 + }, + { + "epoch": 0.2880248833592535, + "grad_norm": 0.6233921647071838, + "learning_rate": 3.8425e-05, + "log_odds_chosen": 2.528999090194702, + "log_odds_ratio": -0.2973198890686035, + "logits/chosen": 0.1612185388803482, + "logits/rejected": 0.18806105852127075, + "logps/chosen": -1.4305648803710938, + "logps/rejected": -3.712616443634033, + "loss": 2.868, + "nll_loss": 2.838242292404175, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1430564969778061, + "rewards/margins": 0.22820515930652618, + "rewards/rejected": -0.3712616562843323, + "step": 463 + }, + { + "epoch": 0.288646967340591, + "grad_norm": 0.43462324142456055, + "learning_rate": 3.8400000000000005e-05, + "log_odds_chosen": 3.214184522628784, + "log_odds_ratio": -0.06043536216020584, + "logits/chosen": 0.21719320118427277, + "logits/rejected": 0.25715163350105286, + "logps/chosen": -1.121093511581421, + "logps/rejected": -3.9434640407562256, + "loss": 2.9904, + "nll_loss": 2.9843649864196777, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11210934817790985, + "rewards/margins": 0.28223708271980286, + "rewards/rejected": -0.3943464159965515, + "step": 464 + }, + { + "epoch": 0.2892690513219285, + "grad_norm": 0.5641807913780212, + "learning_rate": 3.8375e-05, + "log_odds_chosen": 4.3896636962890625, + "log_odds_ratio": -0.18577708303928375, + "logits/chosen": 0.0453701950609684, + "logits/rejected": 0.07587449252605438, + "logps/chosen": -1.2915798425674438, + "logps/rejected": -5.313374996185303, + "loss": 2.8604, + "nll_loss": 2.841811418533325, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12915797531604767, + "rewards/margins": 0.4021795392036438, + "rewards/rejected": -0.5313374996185303, + "step": 465 + }, + { + "epoch": 0.28989113530326593, + "grad_norm": 0.7072799801826477, + "learning_rate": 3.8350000000000004e-05, + "log_odds_chosen": 3.1329102516174316, + "log_odds_ratio": -0.38895708322525024, + "logits/chosen": 0.36617377400398254, + "logits/rejected": 0.32684844732284546, + "logps/chosen": -1.486217975616455, + "logps/rejected": -4.195459842681885, + "loss": 3.3864, + "nll_loss": 3.347512722015381, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1486217975616455, + "rewards/margins": 0.2709242105484009, + "rewards/rejected": -0.4195460081100464, + "step": 466 + }, + { + "epoch": 0.29051321928460344, + "grad_norm": 0.5267555713653564, + "learning_rate": 3.8324999999999996e-05, + "log_odds_chosen": 1.2240618467330933, + "log_odds_ratio": -0.39744335412979126, + "logits/chosen": 0.22255158424377441, + "logits/rejected": 0.18081636726856232, + "logps/chosen": -1.1293349266052246, + "logps/rejected": -2.1319031715393066, + "loss": 2.8698, + "nll_loss": 2.830047130584717, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11293349415063858, + "rewards/margins": 0.10025681555271149, + "rewards/rejected": -0.21319028735160828, + "step": 467 + }, + { + "epoch": 0.2911353032659409, + "grad_norm": 0.6344406604766846, + "learning_rate": 3.83e-05, + "log_odds_chosen": 6.454854488372803, + "log_odds_ratio": -0.14626577496528625, + "logits/chosen": 0.26416561007499695, + "logits/rejected": 0.411790668964386, + "logps/chosen": -1.2627651691436768, + "logps/rejected": -7.43460750579834, + "loss": 3.1443, + "nll_loss": 3.129704475402832, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12627652287483215, + "rewards/margins": 0.6171842813491821, + "rewards/rejected": -0.7434607148170471, + "step": 468 + }, + { + "epoch": 0.2917573872472784, + "grad_norm": 2.3335068225860596, + "learning_rate": 3.8275e-05, + "log_odds_chosen": 5.498930931091309, + "log_odds_ratio": -0.5096065998077393, + "logits/chosen": 0.3187922239303589, + "logits/rejected": 0.4804098308086395, + "logps/chosen": -1.7013590335845947, + "logps/rejected": -6.9031877517700195, + "loss": 3.3374, + "nll_loss": 3.2864151000976562, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.17013590037822723, + "rewards/margins": 0.5201829075813293, + "rewards/rejected": -0.690318763256073, + "step": 469 + }, + { + "epoch": 0.29237947122861585, + "grad_norm": 0.3446737825870514, + "learning_rate": 3.825e-05, + "log_odds_chosen": 4.659626007080078, + "log_odds_ratio": -0.05383168160915375, + "logits/chosen": 0.3359685242176056, + "logits/rejected": 0.3225572407245636, + "logps/chosen": -1.2879467010498047, + "logps/rejected": -5.530665397644043, + "loss": 4.0427, + "nll_loss": 4.037296295166016, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12879468500614166, + "rewards/margins": 0.4242718815803528, + "rewards/rejected": -0.5530665516853333, + "step": 470 + }, + { + "epoch": 0.29300155520995336, + "grad_norm": 0.6117813587188721, + "learning_rate": 3.8225e-05, + "log_odds_chosen": 3.6450319290161133, + "log_odds_ratio": -0.11542224884033203, + "logits/chosen": 0.22536617517471313, + "logits/rejected": 0.26635074615478516, + "logps/chosen": -0.9902899265289307, + "logps/rejected": -4.163326740264893, + "loss": 3.1959, + "nll_loss": 3.184352397918701, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09902900457382202, + "rewards/margins": 0.31730368733406067, + "rewards/rejected": -0.4163326919078827, + "step": 471 + }, + { + "epoch": 0.2936236391912908, + "grad_norm": 0.6647369861602783, + "learning_rate": 3.82e-05, + "log_odds_chosen": 4.759244441986084, + "log_odds_ratio": -0.22806090116500854, + "logits/chosen": 0.2276982069015503, + "logits/rejected": 0.2902960777282715, + "logps/chosen": -1.0806303024291992, + "logps/rejected": -5.489457607269287, + "loss": 2.7768, + "nll_loss": 2.754019260406494, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10806304216384888, + "rewards/margins": 0.44088274240493774, + "rewards/rejected": -0.5489457845687866, + "step": 472 + }, + { + "epoch": 0.2942457231726283, + "grad_norm": 0.4787210524082184, + "learning_rate": 3.8175e-05, + "log_odds_chosen": 3.1084511280059814, + "log_odds_ratio": -0.15226207673549652, + "logits/chosen": 0.3156070113182068, + "logits/rejected": 0.38743242621421814, + "logps/chosen": -1.3634812831878662, + "logps/rejected": -4.172502040863037, + "loss": 3.6274, + "nll_loss": 3.612142324447632, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13634812831878662, + "rewards/margins": 0.28090211749076843, + "rewards/rejected": -0.41725024580955505, + "step": 473 + }, + { + "epoch": 0.29486780715396577, + "grad_norm": 0.4799170196056366, + "learning_rate": 3.8150000000000006e-05, + "log_odds_chosen": 5.302570343017578, + "log_odds_ratio": -0.17532306909561157, + "logits/chosen": 0.2545361816883087, + "logits/rejected": 0.4782401919364929, + "logps/chosen": -1.1034696102142334, + "logps/rejected": -6.034841537475586, + "loss": 2.6736, + "nll_loss": 2.6560819149017334, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1103469654917717, + "rewards/margins": 0.4931372106075287, + "rewards/rejected": -0.6034841537475586, + "step": 474 + }, + { + "epoch": 0.2954898911353033, + "grad_norm": 0.43535855412483215, + "learning_rate": 3.8125e-05, + "log_odds_chosen": 7.762436389923096, + "log_odds_ratio": -0.13623476028442383, + "logits/chosen": 0.1623404324054718, + "logits/rejected": 0.2579556405544281, + "logps/chosen": -0.7613686323165894, + "logps/rejected": -7.789710521697998, + "loss": 3.0684, + "nll_loss": 3.054818630218506, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.07613686472177505, + "rewards/margins": 0.7028341889381409, + "rewards/rejected": -0.7789710760116577, + "step": 475 + }, + { + "epoch": 0.2961119751166407, + "grad_norm": 1.3790374994277954, + "learning_rate": 3.8100000000000005e-05, + "log_odds_chosen": 3.3232266902923584, + "log_odds_ratio": -0.1542171686887741, + "logits/chosen": 0.13157705962657928, + "logits/rejected": 0.14837421476840973, + "logps/chosen": -1.329514741897583, + "logps/rejected": -4.3654656410217285, + "loss": 2.7209, + "nll_loss": 2.7054383754730225, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13295146822929382, + "rewards/margins": 0.3035951256752014, + "rewards/rejected": -0.43654656410217285, + "step": 476 + }, + { + "epoch": 0.29673405909797823, + "grad_norm": 0.4154740869998932, + "learning_rate": 3.8075e-05, + "log_odds_chosen": 3.3031208515167236, + "log_odds_ratio": -0.19959819316864014, + "logits/chosen": 0.29767587780952454, + "logits/rejected": 0.3195898234844208, + "logps/chosen": -1.2283899784088135, + "logps/rejected": -4.255227088928223, + "loss": 3.426, + "nll_loss": 3.4059970378875732, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12283900380134583, + "rewards/margins": 0.3026837110519409, + "rewards/rejected": -0.42552274465560913, + "step": 477 + }, + { + "epoch": 0.2973561430793157, + "grad_norm": 0.440824955701828, + "learning_rate": 3.805e-05, + "log_odds_chosen": 5.133951187133789, + "log_odds_ratio": -0.1343551129102707, + "logits/chosen": -0.004576465114951134, + "logits/rejected": 0.09814856946468353, + "logps/chosen": -1.0036342144012451, + "logps/rejected": -5.681285381317139, + "loss": 2.9374, + "nll_loss": 2.923962116241455, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10036341845989227, + "rewards/margins": 0.46776506304740906, + "rewards/rejected": -0.5681285262107849, + "step": 478 + }, + { + "epoch": 0.2979782270606532, + "grad_norm": 0.4802106022834778, + "learning_rate": 3.8025e-05, + "log_odds_chosen": 4.080648422241211, + "log_odds_ratio": -0.15446895360946655, + "logits/chosen": 0.006258752197027206, + "logits/rejected": 0.04351950064301491, + "logps/chosen": -1.1790379285812378, + "logps/rejected": -4.813777923583984, + "loss": 2.8196, + "nll_loss": 2.804121732711792, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11790378391742706, + "rewards/margins": 0.3634740114212036, + "rewards/rejected": -0.48137781023979187, + "step": 479 + }, + { + "epoch": 0.2986003110419907, + "grad_norm": 0.4847407341003418, + "learning_rate": 3.8e-05, + "log_odds_chosen": 4.479074478149414, + "log_odds_ratio": -0.12381869554519653, + "logits/chosen": 0.2093774825334549, + "logits/rejected": 0.29991966485977173, + "logps/chosen": -1.0310206413269043, + "logps/rejected": -5.081138610839844, + "loss": 2.6951, + "nll_loss": 2.682741165161133, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10310205817222595, + "rewards/margins": 0.4050118029117584, + "rewards/rejected": -0.5081138610839844, + "step": 480 + }, + { + "epoch": 0.29922239502332815, + "grad_norm": 0.9063189029693604, + "learning_rate": 3.7975e-05, + "log_odds_chosen": 2.8616793155670166, + "log_odds_ratio": -0.12458339333534241, + "logits/chosen": 0.1728355586528778, + "logits/rejected": 0.1352435052394867, + "logps/chosen": -1.2799311876296997, + "logps/rejected": -3.802501678466797, + "loss": 3.3188, + "nll_loss": 3.306375026702881, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1279931366443634, + "rewards/margins": 0.25225701928138733, + "rewards/rejected": -0.38025015592575073, + "step": 481 + }, + { + "epoch": 0.29984447900466565, + "grad_norm": 0.42489972710609436, + "learning_rate": 3.795e-05, + "log_odds_chosen": 5.341192245483398, + "log_odds_ratio": -0.26994985342025757, + "logits/chosen": 0.02017594873905182, + "logits/rejected": 0.0014428049325942993, + "logps/chosen": -1.1768755912780762, + "logps/rejected": -6.195101261138916, + "loss": 2.8912, + "nll_loss": 2.8642163276672363, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11768755316734314, + "rewards/margins": 0.5018225312232971, + "rewards/rejected": -0.6195101737976074, + "step": 482 + }, + { + "epoch": 0.3004665629860031, + "grad_norm": 0.4654333293437958, + "learning_rate": 3.7925e-05, + "log_odds_chosen": 2.1485209465026855, + "log_odds_ratio": -0.2876686751842499, + "logits/chosen": 0.02476568892598152, + "logits/rejected": 0.06350227445363998, + "logps/chosen": -1.2585699558258057, + "logps/rejected": -3.1734275817871094, + "loss": 3.0375, + "nll_loss": 3.008713960647583, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12585701048374176, + "rewards/margins": 0.19148576259613037, + "rewards/rejected": -0.31734275817871094, + "step": 483 + }, + { + "epoch": 0.3010886469673406, + "grad_norm": 1.332303524017334, + "learning_rate": 3.79e-05, + "log_odds_chosen": 1.5843894481658936, + "log_odds_ratio": -0.5189324617385864, + "logits/chosen": 0.19418621063232422, + "logits/rejected": 0.06159728765487671, + "logps/chosen": -1.9245703220367432, + "logps/rejected": -3.3311855792999268, + "loss": 3.4679, + "nll_loss": 3.4160549640655518, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.19245702028274536, + "rewards/margins": 0.14066153764724731, + "rewards/rejected": -0.3331185579299927, + "step": 484 + }, + { + "epoch": 0.30171073094867806, + "grad_norm": 0.6246785521507263, + "learning_rate": 3.7875e-05, + "log_odds_chosen": 4.188319206237793, + "log_odds_ratio": -0.4303605556488037, + "logits/chosen": 0.15692827105522156, + "logits/rejected": 0.29972484707832336, + "logps/chosen": -1.1631245613098145, + "logps/rejected": -5.161314964294434, + "loss": 2.4339, + "nll_loss": 2.390859365463257, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11631244421005249, + "rewards/margins": 0.3998190760612488, + "rewards/rejected": -0.5161314606666565, + "step": 485 + }, + { + "epoch": 0.30233281493001557, + "grad_norm": 0.4516681134700775, + "learning_rate": 3.7850000000000005e-05, + "log_odds_chosen": 3.0724825859069824, + "log_odds_ratio": -0.13238348066806793, + "logits/chosen": 0.14594241976737976, + "logits/rejected": 0.19076189398765564, + "logps/chosen": -0.9410706758499146, + "logps/rejected": -3.3255326747894287, + "loss": 2.9302, + "nll_loss": 2.9170100688934326, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09410707652568817, + "rewards/margins": 0.2384462058544159, + "rewards/rejected": -0.33255326747894287, + "step": 486 + }, + { + "epoch": 0.302954898911353, + "grad_norm": 0.44087883830070496, + "learning_rate": 3.7825e-05, + "log_odds_chosen": 3.9114091396331787, + "log_odds_ratio": -0.2950913608074188, + "logits/chosen": 0.15809951722621918, + "logits/rejected": 0.14425241947174072, + "logps/chosen": -1.040985345840454, + "logps/rejected": -4.549111366271973, + "loss": 3.16, + "nll_loss": 3.1304802894592285, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10409855097532272, + "rewards/margins": 0.3508126437664032, + "rewards/rejected": -0.45491117238998413, + "step": 487 + }, + { + "epoch": 0.30357698289269053, + "grad_norm": 0.556199312210083, + "learning_rate": 3.7800000000000004e-05, + "log_odds_chosen": 5.235114097595215, + "log_odds_ratio": -0.041219230741262436, + "logits/chosen": 0.14572152495384216, + "logits/rejected": 0.3106115758419037, + "logps/chosen": -1.1664040088653564, + "logps/rejected": -5.896844863891602, + "loss": 3.1604, + "nll_loss": 3.1562514305114746, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11664040386676788, + "rewards/margins": 0.47304409742355347, + "rewards/rejected": -0.5896844863891602, + "step": 488 + }, + { + "epoch": 0.304199066874028, + "grad_norm": 0.5189743041992188, + "learning_rate": 3.7775e-05, + "log_odds_chosen": 7.963743209838867, + "log_odds_ratio": -0.1266322284936905, + "logits/chosen": 0.3704788088798523, + "logits/rejected": 0.6580029726028442, + "logps/chosen": -1.3174169063568115, + "logps/rejected": -9.020464897155762, + "loss": 3.4173, + "nll_loss": 3.4046013355255127, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1317417025566101, + "rewards/margins": 0.7703048586845398, + "rewards/rejected": -0.9020465016365051, + "step": 489 + }, + { + "epoch": 0.3048211508553655, + "grad_norm": 0.4847100079059601, + "learning_rate": 3.775e-05, + "log_odds_chosen": 9.21757984161377, + "log_odds_ratio": -0.10597797483205795, + "logits/chosen": 0.16972588002681732, + "logits/rejected": 0.3820986747741699, + "logps/chosen": -1.2503827810287476, + "logps/rejected": -10.164284706115723, + "loss": 3.2205, + "nll_loss": 3.2099387645721436, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.125038281083107, + "rewards/margins": 0.8913902044296265, + "rewards/rejected": -1.0164284706115723, + "step": 490 + }, + { + "epoch": 0.30544323483670294, + "grad_norm": 0.8318912386894226, + "learning_rate": 3.7725e-05, + "log_odds_chosen": 9.271547317504883, + "log_odds_ratio": -0.021590309217572212, + "logits/chosen": 0.3189174234867096, + "logits/rejected": 0.49795106053352356, + "logps/chosen": -1.382540225982666, + "logps/rejected": -10.197100639343262, + "loss": 3.9656, + "nll_loss": 3.963407039642334, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13825401663780212, + "rewards/margins": 0.8814560174942017, + "rewards/rejected": -1.0197100639343262, + "step": 491 + }, + { + "epoch": 0.30606531881804044, + "grad_norm": 0.40964096784591675, + "learning_rate": 3.77e-05, + "log_odds_chosen": 5.522929668426514, + "log_odds_ratio": -0.1387336701154709, + "logits/chosen": 0.29802757501602173, + "logits/rejected": 0.4043997526168823, + "logps/chosen": -1.073734164237976, + "logps/rejected": -6.166069507598877, + "loss": 3.7311, + "nll_loss": 3.7172298431396484, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1073734238743782, + "rewards/margins": 0.5092335343360901, + "rewards/rejected": -0.6166069507598877, + "step": 492 + }, + { + "epoch": 0.3066874027993779, + "grad_norm": 0.5874708294868469, + "learning_rate": 3.7675e-05, + "log_odds_chosen": 12.224292755126953, + "log_odds_ratio": -0.021359411999583244, + "logits/chosen": 0.136999249458313, + "logits/rejected": 0.7200852632522583, + "logps/chosen": -0.9732953906059265, + "logps/rejected": -12.676000595092773, + "loss": 2.3038, + "nll_loss": 2.3016741275787354, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09732954204082489, + "rewards/margins": 1.1702704429626465, + "rewards/rejected": -1.2676000595092773, + "step": 493 + }, + { + "epoch": 0.3073094867807154, + "grad_norm": 0.5743467807769775, + "learning_rate": 3.765e-05, + "log_odds_chosen": 12.042713165283203, + "log_odds_ratio": -0.070807546377182, + "logits/chosen": 0.2361012101173401, + "logits/rejected": 0.562972903251648, + "logps/chosen": -1.5356069803237915, + "logps/rejected": -13.233451843261719, + "loss": 3.359, + "nll_loss": 3.3519067764282227, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15356069803237915, + "rewards/margins": 1.1697845458984375, + "rewards/rejected": -1.3233451843261719, + "step": 494 + }, + { + "epoch": 0.30793157076205285, + "grad_norm": 1.004876732826233, + "learning_rate": 3.7625e-05, + "log_odds_chosen": 6.287815570831299, + "log_odds_ratio": -0.2341613471508026, + "logits/chosen": 0.1172030121088028, + "logits/rejected": 0.23774453997612, + "logps/chosen": -1.0990171432495117, + "logps/rejected": -6.949299335479736, + "loss": 3.1907, + "nll_loss": 3.167259454727173, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10990171879529953, + "rewards/margins": 0.5850282311439514, + "rewards/rejected": -0.6949299573898315, + "step": 495 + }, + { + "epoch": 0.30855365474339036, + "grad_norm": 3.0034093856811523, + "learning_rate": 3.76e-05, + "log_odds_chosen": 7.123260498046875, + "log_odds_ratio": -0.10044410824775696, + "logits/chosen": 0.23248106241226196, + "logits/rejected": 0.4872136116027832, + "logps/chosen": -1.2667349576950073, + "logps/rejected": -8.054340362548828, + "loss": 3.0054, + "nll_loss": 2.9953811168670654, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12667348980903625, + "rewards/margins": 0.6787605285644531, + "rewards/rejected": -0.8054340481758118, + "step": 496 + }, + { + "epoch": 0.3091757387247278, + "grad_norm": 1.1735531091690063, + "learning_rate": 3.7575e-05, + "log_odds_chosen": 8.58930492401123, + "log_odds_ratio": -0.04813535511493683, + "logits/chosen": 0.26603323221206665, + "logits/rejected": 0.5334312915802002, + "logps/chosen": -1.2953003644943237, + "logps/rejected": -9.486214637756348, + "loss": 3.1711, + "nll_loss": 3.166259288787842, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12953002750873566, + "rewards/margins": 0.8190913796424866, + "rewards/rejected": -0.9486214518547058, + "step": 497 + }, + { + "epoch": 0.3097978227060653, + "grad_norm": 0.6103359460830688, + "learning_rate": 3.7550000000000005e-05, + "log_odds_chosen": 2.1337530612945557, + "log_odds_ratio": -0.2864910066127777, + "logits/chosen": 0.15103283524513245, + "logits/rejected": 0.25624915957450867, + "logps/chosen": -1.439260482788086, + "logps/rejected": -3.371507406234741, + "loss": 3.1723, + "nll_loss": 3.143685817718506, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14392603933811188, + "rewards/margins": 0.1932247132062912, + "rewards/rejected": -0.33715078234672546, + "step": 498 + }, + { + "epoch": 0.3104199066874028, + "grad_norm": 0.43494969606399536, + "learning_rate": 3.7525e-05, + "log_odds_chosen": 3.525885820388794, + "log_odds_ratio": -0.3266465663909912, + "logits/chosen": 0.008623511530458927, + "logits/rejected": 0.14156433939933777, + "logps/chosen": -1.0080041885375977, + "logps/rejected": -4.299755096435547, + "loss": 2.4517, + "nll_loss": 2.4190571308135986, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10080042481422424, + "rewards/margins": 0.32917505502700806, + "rewards/rejected": -0.4299754798412323, + "step": 499 + }, + { + "epoch": 0.3110419906687403, + "grad_norm": 0.4379410147666931, + "learning_rate": 3.7500000000000003e-05, + "log_odds_chosen": 1.4496530294418335, + "log_odds_ratio": -0.3601369857788086, + "logits/chosen": 0.027029283344745636, + "logits/rejected": 0.10909566283226013, + "logps/chosen": -1.3700931072235107, + "logps/rejected": -2.6318204402923584, + "loss": 2.8374, + "nll_loss": 2.801370620727539, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13700930774211884, + "rewards/margins": 0.1261727213859558, + "rewards/rejected": -0.26318204402923584, + "step": 500 + }, + { + "epoch": 0.3116640746500778, + "grad_norm": 0.631767988204956, + "learning_rate": 3.7475e-05, + "log_odds_chosen": 2.5015196800231934, + "log_odds_ratio": -0.40100595355033875, + "logits/chosen": -0.09599044919013977, + "logits/rejected": -0.06666186451911926, + "logps/chosen": -0.8881614804267883, + "logps/rejected": -3.0457301139831543, + "loss": 2.8761, + "nll_loss": 2.8360183238983154, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08881615102291107, + "rewards/margins": 0.2157568335533142, + "rewards/rejected": -0.3045729994773865, + "step": 501 + }, + { + "epoch": 0.31228615863141523, + "grad_norm": 0.5571648478507996, + "learning_rate": 3.745e-05, + "log_odds_chosen": 3.5235862731933594, + "log_odds_ratio": -0.2553158402442932, + "logits/chosen": 0.08271875232458115, + "logits/rejected": 0.159298837184906, + "logps/chosen": -1.5579495429992676, + "logps/rejected": -4.873930931091309, + "loss": 3.0358, + "nll_loss": 3.010316848754883, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15579496324062347, + "rewards/margins": 0.331598162651062, + "rewards/rejected": -0.4873931109905243, + "step": 502 + }, + { + "epoch": 0.31290824261275274, + "grad_norm": 0.9667811393737793, + "learning_rate": 3.7425e-05, + "log_odds_chosen": 1.5735365152359009, + "log_odds_ratio": -0.4731370806694031, + "logits/chosen": 0.03942291438579559, + "logits/rejected": -0.06971655786037445, + "logps/chosen": -1.4615811109542847, + "logps/rejected": -2.880722761154175, + "loss": 3.1011, + "nll_loss": 3.0537822246551514, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1461580991744995, + "rewards/margins": 0.14191415905952454, + "rewards/rejected": -0.28807225823402405, + "step": 503 + }, + { + "epoch": 0.3135303265940902, + "grad_norm": 0.496913880109787, + "learning_rate": 3.74e-05, + "log_odds_chosen": 2.018817901611328, + "log_odds_ratio": -0.37110626697540283, + "logits/chosen": -0.06371671706438065, + "logits/rejected": -0.035864636301994324, + "logps/chosen": -0.9149460792541504, + "logps/rejected": -2.4723567962646484, + "loss": 2.7536, + "nll_loss": 2.7165191173553467, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0914946049451828, + "rewards/margins": 0.15574106574058533, + "rewards/rejected": -0.24723568558692932, + "step": 504 + }, + { + "epoch": 0.3141524105754277, + "grad_norm": 0.45206576585769653, + "learning_rate": 3.737500000000001e-05, + "log_odds_chosen": 2.5706026554107666, + "log_odds_ratio": -0.44339919090270996, + "logits/chosen": 0.18639887869358063, + "logits/rejected": 0.09812915325164795, + "logps/chosen": -1.0650825500488281, + "logps/rejected": -3.2684202194213867, + "loss": 3.6104, + "nll_loss": 3.5660290718078613, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10650825500488281, + "rewards/margins": 0.2203337550163269, + "rewards/rejected": -0.3268420100212097, + "step": 505 + }, + { + "epoch": 0.31477449455676515, + "grad_norm": 0.46192917227745056, + "learning_rate": 3.735e-05, + "log_odds_chosen": 1.1359951496124268, + "log_odds_ratio": -0.45393139123916626, + "logits/chosen": 0.20836736261844635, + "logits/rejected": 0.045999638736248016, + "logps/chosen": -1.4677722454071045, + "logps/rejected": -2.4714083671569824, + "loss": 3.5338, + "nll_loss": 3.4884276390075684, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1467772126197815, + "rewards/margins": 0.10036361217498779, + "rewards/rejected": -0.24714083969593048, + "step": 506 + }, + { + "epoch": 0.31539657853810266, + "grad_norm": 0.8858140110969543, + "learning_rate": 3.7325000000000006e-05, + "log_odds_chosen": 0.8142578601837158, + "log_odds_ratio": -0.41193997859954834, + "logits/chosen": 0.23445630073547363, + "logits/rejected": 0.15942780673503876, + "logps/chosen": -1.313204050064087, + "logps/rejected": -1.9680757522583008, + "loss": 3.3672, + "nll_loss": 3.326014995574951, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13132041692733765, + "rewards/margins": 0.06548717617988586, + "rewards/rejected": -0.19680756330490112, + "step": 507 + }, + { + "epoch": 0.3160186625194401, + "grad_norm": 0.513054370880127, + "learning_rate": 3.73e-05, + "log_odds_chosen": 1.9759085178375244, + "log_odds_ratio": -0.40382054448127747, + "logits/chosen": 0.1355867236852646, + "logits/rejected": -0.025263607501983643, + "logps/chosen": -1.2560956478118896, + "logps/rejected": -3.0063164234161377, + "loss": 3.0785, + "nll_loss": 3.038083553314209, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12560957670211792, + "rewards/margins": 0.17502208054065704, + "rewards/rejected": -0.30063164234161377, + "step": 508 + }, + { + "epoch": 0.3166407465007776, + "grad_norm": 0.29589080810546875, + "learning_rate": 3.7275000000000005e-05, + "log_odds_chosen": 3.7314577102661133, + "log_odds_ratio": -0.2652418315410614, + "logits/chosen": 0.1787126213312149, + "logits/rejected": 0.24298204481601715, + "logps/chosen": -1.1753450632095337, + "logps/rejected": -4.593437194824219, + "loss": 3.1982, + "nll_loss": 3.1716480255126953, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11753450334072113, + "rewards/margins": 0.3418092131614685, + "rewards/rejected": -0.45934373140335083, + "step": 509 + }, + { + "epoch": 0.31726283048211507, + "grad_norm": 0.37716519832611084, + "learning_rate": 3.7250000000000004e-05, + "log_odds_chosen": 6.13275146484375, + "log_odds_ratio": -0.0573071613907814, + "logits/chosen": 0.1888727992773056, + "logits/rejected": 0.2561497092247009, + "logps/chosen": -1.123389720916748, + "logps/rejected": -6.81281852722168, + "loss": 3.15, + "nll_loss": 3.1442646980285645, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11233897507190704, + "rewards/margins": 0.5689429044723511, + "rewards/rejected": -0.6812818050384521, + "step": 510 + }, + { + "epoch": 0.3178849144634526, + "grad_norm": 0.48151102662086487, + "learning_rate": 3.7225000000000004e-05, + "log_odds_chosen": 2.6327872276306152, + "log_odds_ratio": -0.2002970576286316, + "logits/chosen": 0.3196089267730713, + "logits/rejected": 0.27417632937431335, + "logps/chosen": -1.2908976078033447, + "logps/rejected": -3.636533260345459, + "loss": 3.504, + "nll_loss": 3.4839515686035156, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12908975780010223, + "rewards/margins": 0.23456355929374695, + "rewards/rejected": -0.3636533319950104, + "step": 511 + }, + { + "epoch": 0.31850699844479, + "grad_norm": 0.43662747740745544, + "learning_rate": 3.72e-05, + "log_odds_chosen": 4.564843654632568, + "log_odds_ratio": -0.06708259880542755, + "logits/chosen": 0.24633480608463287, + "logits/rejected": 0.48307889699935913, + "logps/chosen": -0.9144176840782166, + "logps/rejected": -4.8161234855651855, + "loss": 3.12, + "nll_loss": 3.1133110523223877, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09144177287817001, + "rewards/margins": 0.39017054438591003, + "rewards/rejected": -0.48161232471466064, + "step": 512 + }, + { + "epoch": 0.31912908242612753, + "grad_norm": 0.5609356164932251, + "learning_rate": 3.7175e-05, + "log_odds_chosen": 5.504610538482666, + "log_odds_ratio": -0.08810669928789139, + "logits/chosen": 0.2721938490867615, + "logits/rejected": 0.42912524938583374, + "logps/chosen": -1.2787365913391113, + "logps/rejected": -6.235076904296875, + "loss": 3.0593, + "nll_loss": 3.050450086593628, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12787367403507233, + "rewards/margins": 0.49563395977020264, + "rewards/rejected": -0.6235076189041138, + "step": 513 + }, + { + "epoch": 0.319751166407465, + "grad_norm": 0.3104366958141327, + "learning_rate": 3.715e-05, + "log_odds_chosen": 6.418957710266113, + "log_odds_ratio": -0.17403581738471985, + "logits/chosen": 0.2422790229320526, + "logits/rejected": 0.45724615454673767, + "logps/chosen": -1.1649143695831299, + "logps/rejected": -7.193992614746094, + "loss": 3.2466, + "nll_loss": 3.2292304039001465, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11649143695831299, + "rewards/margins": 0.6029078364372253, + "rewards/rejected": -0.7193993330001831, + "step": 514 + }, + { + "epoch": 0.3203732503888025, + "grad_norm": 0.4039505422115326, + "learning_rate": 3.7125e-05, + "log_odds_chosen": 4.793672561645508, + "log_odds_ratio": -0.17864283919334412, + "logits/chosen": 0.31905102729797363, + "logits/rejected": 0.5531411170959473, + "logps/chosen": -0.8685755133628845, + "logps/rejected": -5.093353271484375, + "loss": 3.4313, + "nll_loss": 3.4134421348571777, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08685755729675293, + "rewards/margins": 0.4224777817726135, + "rewards/rejected": -0.5093352794647217, + "step": 515 + }, + { + "epoch": 0.32099533437014, + "grad_norm": 0.6209097504615784, + "learning_rate": 3.71e-05, + "log_odds_chosen": 13.188308715820312, + "log_odds_ratio": -0.03541084751486778, + "logits/chosen": 0.2741844952106476, + "logits/rejected": 0.9028793573379517, + "logps/chosen": -1.2709870338439941, + "logps/rejected": -13.850850105285645, + "loss": 2.5904, + "nll_loss": 2.5868520736694336, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1270987093448639, + "rewards/margins": 1.257986307144165, + "rewards/rejected": -1.385085105895996, + "step": 516 + }, + { + "epoch": 0.32161741835147745, + "grad_norm": 0.4564976990222931, + "learning_rate": 3.707500000000001e-05, + "log_odds_chosen": 8.428442001342773, + "log_odds_ratio": -0.1521356999874115, + "logits/chosen": 0.31411972641944885, + "logits/rejected": 0.6657370328903198, + "logps/chosen": -1.1624163389205933, + "logps/rejected": -9.256325721740723, + "loss": 3.2951, + "nll_loss": 3.279919385910034, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11624162644147873, + "rewards/margins": 0.8093909025192261, + "rewards/rejected": -0.9256325960159302, + "step": 517 + }, + { + "epoch": 0.32223950233281495, + "grad_norm": 0.4547824263572693, + "learning_rate": 3.705e-05, + "log_odds_chosen": 8.413576126098633, + "log_odds_ratio": -0.18886855244636536, + "logits/chosen": 0.28079456090927124, + "logits/rejected": 0.5538430213928223, + "logps/chosen": -1.1201964616775513, + "logps/rejected": -9.210822105407715, + "loss": 3.1612, + "nll_loss": 3.1422665119171143, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11201964318752289, + "rewards/margins": 0.8090626001358032, + "rewards/rejected": -0.9210821986198425, + "step": 518 + }, + { + "epoch": 0.3228615863141524, + "grad_norm": 0.47383055090904236, + "learning_rate": 3.7025000000000005e-05, + "log_odds_chosen": 7.101673126220703, + "log_odds_ratio": -0.2151382565498352, + "logits/chosen": 0.19054706394672394, + "logits/rejected": 0.4452645480632782, + "logps/chosen": -0.9745645523071289, + "logps/rejected": -7.625181674957275, + "loss": 2.8218, + "nll_loss": 2.8003106117248535, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09745645523071289, + "rewards/margins": 0.6650617718696594, + "rewards/rejected": -0.7625181674957275, + "step": 519 + }, + { + "epoch": 0.3234836702954899, + "grad_norm": 0.6366752982139587, + "learning_rate": 3.7e-05, + "log_odds_chosen": 8.329351425170898, + "log_odds_ratio": -0.3641209304332733, + "logits/chosen": 0.14251962304115295, + "logits/rejected": 0.6523263454437256, + "logps/chosen": -1.3124771118164062, + "logps/rejected": -9.418330192565918, + "loss": 2.3793, + "nll_loss": 2.3428869247436523, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13124771416187286, + "rewards/margins": 0.8105852603912354, + "rewards/rejected": -0.9418330192565918, + "step": 520 + }, + { + "epoch": 0.32410575427682736, + "grad_norm": 0.4376268684864044, + "learning_rate": 3.6975000000000004e-05, + "log_odds_chosen": 10.756841659545898, + "log_odds_ratio": -0.2801904082298279, + "logits/chosen": 0.31616631150245667, + "logits/rejected": 0.5765305757522583, + "logps/chosen": -1.2397606372833252, + "logps/rejected": -11.706258773803711, + "loss": 3.3466, + "nll_loss": 3.318598508834839, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12397606670856476, + "rewards/margins": 1.046649694442749, + "rewards/rejected": -1.1706258058547974, + "step": 521 + }, + { + "epoch": 0.32472783825816487, + "grad_norm": 0.4194689393043518, + "learning_rate": 3.6950000000000004e-05, + "log_odds_chosen": 9.908319473266602, + "log_odds_ratio": -0.1611042022705078, + "logits/chosen": 0.32544493675231934, + "logits/rejected": 0.6462383270263672, + "logps/chosen": -1.2010586261749268, + "logps/rejected": -10.787912368774414, + "loss": 3.4332, + "nll_loss": 3.4171175956726074, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12010586261749268, + "rewards/margins": 0.9586854577064514, + "rewards/rejected": -1.0787913799285889, + "step": 522 + }, + { + "epoch": 0.3253499222395023, + "grad_norm": 1.505491018295288, + "learning_rate": 3.6925e-05, + "log_odds_chosen": 14.340056419372559, + "log_odds_ratio": -0.0028261165134608746, + "logits/chosen": 0.2382410168647766, + "logits/rejected": 0.7966728806495667, + "logps/chosen": -1.5478522777557373, + "logps/rejected": -15.318090438842773, + "loss": 2.7164, + "nll_loss": 2.716071128845215, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15478524565696716, + "rewards/margins": 1.3770238161087036, + "rewards/rejected": -1.5318089723587036, + "step": 523 + }, + { + "epoch": 0.3259720062208398, + "grad_norm": 0.4445713460445404, + "learning_rate": 3.69e-05, + "log_odds_chosen": 10.326703071594238, + "log_odds_ratio": -0.0006719048251397908, + "logits/chosen": 0.380736768245697, + "logits/rejected": 1.0418426990509033, + "logps/chosen": -1.3673847913742065, + "logps/rejected": -11.366878509521484, + "loss": 3.4216, + "nll_loss": 3.4214975833892822, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13673847913742065, + "rewards/margins": 0.9999493956565857, + "rewards/rejected": -1.1366878747940063, + "step": 524 + }, + { + "epoch": 0.3265940902021773, + "grad_norm": 0.7201062440872192, + "learning_rate": 3.6875e-05, + "log_odds_chosen": 10.034008979797363, + "log_odds_ratio": -0.09337079524993896, + "logits/chosen": 0.24129562079906464, + "logits/rejected": 0.7426089644432068, + "logps/chosen": -1.3333206176757812, + "logps/rejected": -11.076189041137695, + "loss": 2.9608, + "nll_loss": 2.951493263244629, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13333207368850708, + "rewards/margins": 0.9742867946624756, + "rewards/rejected": -1.1076189279556274, + "step": 525 + }, + { + "epoch": 0.3272161741835148, + "grad_norm": 0.3761167526245117, + "learning_rate": 3.685e-05, + "log_odds_chosen": 3.660090923309326, + "log_odds_ratio": -0.5031037330627441, + "logits/chosen": 0.13932842016220093, + "logits/rejected": 0.2806503176689148, + "logps/chosen": -1.4467041492462158, + "logps/rejected": -5.056726455688477, + "loss": 3.1734, + "nll_loss": 3.1230688095092773, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14467042684555054, + "rewards/margins": 0.36100226640701294, + "rewards/rejected": -0.5056726932525635, + "step": 526 + }, + { + "epoch": 0.32783825816485224, + "grad_norm": 0.48558077216148376, + "learning_rate": 3.6825e-05, + "log_odds_chosen": 7.3520402908325195, + "log_odds_ratio": -0.22055096924304962, + "logits/chosen": 0.23364008963108063, + "logits/rejected": 0.43131324648857117, + "logps/chosen": -0.8987447023391724, + "logps/rejected": -7.75795316696167, + "loss": 3.0376, + "nll_loss": 3.0155234336853027, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08987447619438171, + "rewards/margins": 0.6859208345413208, + "rewards/rejected": -0.7757953405380249, + "step": 527 + }, + { + "epoch": 0.32846034214618974, + "grad_norm": 0.5533658266067505, + "learning_rate": 3.68e-05, + "log_odds_chosen": 7.092010498046875, + "log_odds_ratio": -0.21083347499370575, + "logits/chosen": 0.16821296513080597, + "logits/rejected": 0.5857659578323364, + "logps/chosen": -1.1237003803253174, + "logps/rejected": -7.805783748626709, + "loss": 2.9438, + "nll_loss": 2.9227511882781982, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11237004399299622, + "rewards/margins": 0.6682083606719971, + "rewards/rejected": -0.7805783748626709, + "step": 528 + }, + { + "epoch": 0.3290824261275272, + "grad_norm": 0.3960796594619751, + "learning_rate": 3.6775000000000006e-05, + "log_odds_chosen": 5.841064453125, + "log_odds_ratio": -0.2500973641872406, + "logits/chosen": 0.27127107977867126, + "logits/rejected": 0.43577903509140015, + "logps/chosen": -1.4152770042419434, + "logps/rejected": -7.040769100189209, + "loss": 3.6124, + "nll_loss": 3.587347984313965, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1415277123451233, + "rewards/margins": 0.5625491738319397, + "rewards/rejected": -0.704076886177063, + "step": 529 + }, + { + "epoch": 0.3297045101088647, + "grad_norm": 0.5108814835548401, + "learning_rate": 3.675e-05, + "log_odds_chosen": 4.806069850921631, + "log_odds_ratio": -0.23009052872657776, + "logits/chosen": -0.039525844156742096, + "logits/rejected": 0.47278881072998047, + "logps/chosen": -1.1424496173858643, + "logps/rejected": -5.656998157501221, + "loss": 2.4652, + "nll_loss": 2.442180633544922, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1142449676990509, + "rewards/margins": 0.4514548182487488, + "rewards/rejected": -0.5656998157501221, + "step": 530 + }, + { + "epoch": 0.33032659409020215, + "grad_norm": 0.4421898424625397, + "learning_rate": 3.6725000000000005e-05, + "log_odds_chosen": 4.877275466918945, + "log_odds_ratio": -0.16743728518486023, + "logits/chosen": 0.05251733213663101, + "logits/rejected": 0.2228899598121643, + "logps/chosen": -0.9268969297409058, + "logps/rejected": -5.308384418487549, + "loss": 2.8371, + "nll_loss": 2.820338010787964, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09268969297409058, + "rewards/margins": 0.4381487965583801, + "rewards/rejected": -0.5308384895324707, + "step": 531 + }, + { + "epoch": 0.33094867807153966, + "grad_norm": 0.33455726504325867, + "learning_rate": 3.6700000000000004e-05, + "log_odds_chosen": 3.417280912399292, + "log_odds_ratio": -0.38553696870803833, + "logits/chosen": 0.15889662504196167, + "logits/rejected": 0.3938848376274109, + "logps/chosen": -1.285894751548767, + "logps/rejected": -4.560215950012207, + "loss": 3.2018, + "nll_loss": 3.1632089614868164, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1285894811153412, + "rewards/margins": 0.3274320960044861, + "rewards/rejected": -0.45602160692214966, + "step": 532 + }, + { + "epoch": 0.33157076205287717, + "grad_norm": 0.3787088096141815, + "learning_rate": 3.6675000000000004e-05, + "log_odds_chosen": 9.068405151367188, + "log_odds_ratio": -0.08513309061527252, + "logits/chosen": 0.15244531631469727, + "logits/rejected": 0.667584240436554, + "logps/chosen": -1.1898918151855469, + "logps/rejected": -9.916119575500488, + "loss": 2.9166, + "nll_loss": 2.9080939292907715, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11898918449878693, + "rewards/margins": 0.8726227879524231, + "rewards/rejected": -0.9916119575500488, + "step": 533 + }, + { + "epoch": 0.3321928460342146, + "grad_norm": 0.39991286396980286, + "learning_rate": 3.665e-05, + "log_odds_chosen": 12.67526912689209, + "log_odds_ratio": -0.1024663895368576, + "logits/chosen": 0.21998846530914307, + "logits/rejected": 0.7742512226104736, + "logps/chosen": -1.0572272539138794, + "logps/rejected": -13.273682594299316, + "loss": 2.9368, + "nll_loss": 2.9265105724334717, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10572272539138794, + "rewards/margins": 1.2216455936431885, + "rewards/rejected": -1.3273682594299316, + "step": 534 + }, + { + "epoch": 0.3328149300155521, + "grad_norm": 0.49979427456855774, + "learning_rate": 3.6625e-05, + "log_odds_chosen": 1.2147586345672607, + "log_odds_ratio": -0.6691287755966187, + "logits/chosen": 0.12728288769721985, + "logits/rejected": 0.22928330302238464, + "logps/chosen": -1.2779240608215332, + "logps/rejected": -2.464627265930176, + "loss": 2.5579, + "nll_loss": 2.491028308868408, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.12779241800308228, + "rewards/margins": 0.11867032200098038, + "rewards/rejected": -0.24646273255348206, + "step": 535 + }, + { + "epoch": 0.3334370139968896, + "grad_norm": 0.5618945360183716, + "learning_rate": 3.66e-05, + "log_odds_chosen": 5.239035129547119, + "log_odds_ratio": -0.4188866913318634, + "logits/chosen": 0.32197320461273193, + "logits/rejected": 0.5903685092926025, + "logps/chosen": -1.3105459213256836, + "logps/rejected": -6.369636535644531, + "loss": 3.088, + "nll_loss": 3.0460805892944336, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1310545951128006, + "rewards/margins": 0.5059090852737427, + "rewards/rejected": -0.6369636654853821, + "step": 536 + }, + { + "epoch": 0.3340590979782271, + "grad_norm": 0.5546212792396545, + "learning_rate": 3.6575e-05, + "log_odds_chosen": 8.182401657104492, + "log_odds_ratio": -0.22162377834320068, + "logits/chosen": 0.09892675280570984, + "logits/rejected": 0.21920329332351685, + "logps/chosen": -1.0027060508728027, + "logps/rejected": -8.776458740234375, + "loss": 2.8454, + "nll_loss": 2.823225975036621, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1002705991268158, + "rewards/margins": 0.777375340461731, + "rewards/rejected": -0.8776458501815796, + "step": 537 + }, + { + "epoch": 0.33468118195956453, + "grad_norm": 0.47566530108451843, + "learning_rate": 3.655e-05, + "log_odds_chosen": 5.837797164916992, + "log_odds_ratio": -0.18034599721431732, + "logits/chosen": 0.30575647950172424, + "logits/rejected": 0.5681968331336975, + "logps/chosen": -1.1159521341323853, + "logps/rejected": -6.579317092895508, + "loss": 3.0867, + "nll_loss": 3.068636178970337, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11159522086381912, + "rewards/margins": 0.5463365316390991, + "rewards/rejected": -0.6579316854476929, + "step": 538 + }, + { + "epoch": 0.33530326594090204, + "grad_norm": 0.5338764786720276, + "learning_rate": 3.652500000000001e-05, + "log_odds_chosen": 9.230720520019531, + "log_odds_ratio": -0.18835598230361938, + "logits/chosen": 0.4432418942451477, + "logits/rejected": 1.0067334175109863, + "logps/chosen": -0.791349470615387, + "logps/rejected": -9.255400657653809, + "loss": 3.4137, + "nll_loss": 3.3948874473571777, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07913494855165482, + "rewards/margins": 0.8464051485061646, + "rewards/rejected": -0.9255399703979492, + "step": 539 + }, + { + "epoch": 0.3359253499222395, + "grad_norm": 0.4006078839302063, + "learning_rate": 3.65e-05, + "log_odds_chosen": 7.939189434051514, + "log_odds_ratio": -0.17464585602283478, + "logits/chosen": 0.3552432358264923, + "logits/rejected": 0.7513606548309326, + "logps/chosen": -1.0403473377227783, + "logps/rejected": -8.616355895996094, + "loss": 3.3614, + "nll_loss": 3.3439626693725586, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10403473675251007, + "rewards/margins": 0.7576008439064026, + "rewards/rejected": -0.8616355061531067, + "step": 540 + }, + { + "epoch": 0.336547433903577, + "grad_norm": 0.652712345123291, + "learning_rate": 3.6475000000000006e-05, + "log_odds_chosen": 7.361583709716797, + "log_odds_ratio": -0.44913148880004883, + "logits/chosen": 0.5314218997955322, + "logits/rejected": 0.9203664660453796, + "logps/chosen": -1.7339417934417725, + "logps/rejected": -8.831292152404785, + "loss": 3.4607, + "nll_loss": 3.4157681465148926, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17339418828487396, + "rewards/margins": 0.7097350358963013, + "rewards/rejected": -0.8831292390823364, + "step": 541 + }, + { + "epoch": 0.33716951788491445, + "grad_norm": 0.5391427278518677, + "learning_rate": 3.645e-05, + "log_odds_chosen": 5.927590370178223, + "log_odds_ratio": -0.17163582146167755, + "logits/chosen": 0.45557430386543274, + "logits/rejected": 0.758292019367218, + "logps/chosen": -1.0969210863113403, + "logps/rejected": -6.686923980712891, + "loss": 3.0634, + "nll_loss": 3.0462679862976074, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10969211906194687, + "rewards/margins": 0.5590003728866577, + "rewards/rejected": -0.6686924695968628, + "step": 542 + }, + { + "epoch": 0.33779160186625196, + "grad_norm": 0.5085594058036804, + "learning_rate": 3.6425000000000004e-05, + "log_odds_chosen": 6.805022239685059, + "log_odds_ratio": -0.04457619786262512, + "logits/chosen": 0.2579970359802246, + "logits/rejected": 0.8251427412033081, + "logps/chosen": -1.0637521743774414, + "logps/rejected": -7.367990016937256, + "loss": 2.7691, + "nll_loss": 2.7646713256835938, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10637522488832474, + "rewards/margins": 0.6304237246513367, + "rewards/rejected": -0.7367990016937256, + "step": 543 + }, + { + "epoch": 0.3384136858475894, + "grad_norm": 1.546451449394226, + "learning_rate": 3.6400000000000004e-05, + "log_odds_chosen": 7.0844831466674805, + "log_odds_ratio": -0.15514059364795685, + "logits/chosen": 0.32124999165534973, + "logits/rejected": 0.5545608997344971, + "logps/chosen": -1.4081883430480957, + "logps/rejected": -8.002361297607422, + "loss": 3.4905, + "nll_loss": 3.475017547607422, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14081883430480957, + "rewards/margins": 0.6594172716140747, + "rewards/rejected": -0.8002361059188843, + "step": 544 + }, + { + "epoch": 0.3390357698289269, + "grad_norm": 0.9646157622337341, + "learning_rate": 3.6375e-05, + "log_odds_chosen": 5.977232933044434, + "log_odds_ratio": -0.31709274649620056, + "logits/chosen": 0.2529332637786865, + "logits/rejected": 0.3822195529937744, + "logps/chosen": -1.1004133224487305, + "logps/rejected": -6.800418376922607, + "loss": 3.3593, + "nll_loss": 3.3275961875915527, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11004132777452469, + "rewards/margins": 0.5700005292892456, + "rewards/rejected": -0.6800418496131897, + "step": 545 + }, + { + "epoch": 0.33965785381026437, + "grad_norm": 0.4293280839920044, + "learning_rate": 3.635e-05, + "log_odds_chosen": 8.562162399291992, + "log_odds_ratio": -0.13144966959953308, + "logits/chosen": 0.12223690003156662, + "logits/rejected": 0.3512270450592041, + "logps/chosen": -0.9846524596214294, + "logps/rejected": -9.026788711547852, + "loss": 2.9709, + "nll_loss": 2.957728385925293, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09846524894237518, + "rewards/margins": 0.8042137026786804, + "rewards/rejected": -0.9026789665222168, + "step": 546 + }, + { + "epoch": 0.34027993779160187, + "grad_norm": 0.5592858195304871, + "learning_rate": 3.6325e-05, + "log_odds_chosen": 7.228453159332275, + "log_odds_ratio": -0.07215896248817444, + "logits/chosen": 0.19593659043312073, + "logits/rejected": 0.7204120755195618, + "logps/chosen": -1.225344181060791, + "logps/rejected": -8.123269081115723, + "loss": 3.223, + "nll_loss": 3.2157747745513916, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12253442406654358, + "rewards/margins": 0.6897925138473511, + "rewards/rejected": -0.8123269081115723, + "step": 547 + }, + { + "epoch": 0.3409020217729393, + "grad_norm": 0.3459852635860443, + "learning_rate": 3.63e-05, + "log_odds_chosen": 8.127766609191895, + "log_odds_ratio": -0.10982684791088104, + "logits/chosen": 0.18933087587356567, + "logits/rejected": 0.6136522889137268, + "logps/chosen": -1.2939542531967163, + "logps/rejected": -9.101163864135742, + "loss": 3.2767, + "nll_loss": 3.265726089477539, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12939542531967163, + "rewards/margins": 0.7807210683822632, + "rewards/rejected": -0.91011643409729, + "step": 548 + }, + { + "epoch": 0.34152410575427683, + "grad_norm": 0.5195790529251099, + "learning_rate": 3.6275e-05, + "log_odds_chosen": 10.92127799987793, + "log_odds_ratio": -0.03485168516635895, + "logits/chosen": 0.1153639703989029, + "logits/rejected": 1.0010063648223877, + "logps/chosen": -0.8691779375076294, + "logps/rejected": -11.154041290283203, + "loss": 2.4175, + "nll_loss": 2.414062976837158, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08691780269145966, + "rewards/margins": 1.0284863710403442, + "rewards/rejected": -1.1154041290283203, + "step": 549 + }, + { + "epoch": 0.3421461897356143, + "grad_norm": 0.4225841760635376, + "learning_rate": 3.625e-05, + "log_odds_chosen": 11.029512405395508, + "log_odds_ratio": -0.003762052860110998, + "logits/chosen": 0.1677618771791458, + "logits/rejected": 0.9653577208518982, + "logps/chosen": -1.1759905815124512, + "logps/rejected": -11.536764144897461, + "loss": 3.2061, + "nll_loss": 3.2056827545166016, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11759904772043228, + "rewards/margins": 1.0360772609710693, + "rewards/rejected": -1.1536762714385986, + "step": 550 + }, + { + "epoch": 0.3427682737169518, + "grad_norm": 0.4881388247013092, + "learning_rate": 3.6225000000000006e-05, + "log_odds_chosen": 10.496110916137695, + "log_odds_ratio": -0.012914421036839485, + "logits/chosen": 0.15544462203979492, + "logits/rejected": 0.910711944103241, + "logps/chosen": -1.2286415100097656, + "logps/rejected": -11.356847763061523, + "loss": 2.8724, + "nll_loss": 2.871134042739868, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12286416441202164, + "rewards/margins": 1.0128206014633179, + "rewards/rejected": -1.1356847286224365, + "step": 551 + }, + { + "epoch": 0.3433903576982893, + "grad_norm": 0.5085164308547974, + "learning_rate": 3.62e-05, + "log_odds_chosen": 3.2137742042541504, + "log_odds_ratio": -0.2135535031557083, + "logits/chosen": 0.052660077810287476, + "logits/rejected": 0.18513916432857513, + "logps/chosen": -1.4653301239013672, + "logps/rejected": -4.4103498458862305, + "loss": 3.3976, + "nll_loss": 3.37619686126709, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14653301239013672, + "rewards/margins": 0.294501930475235, + "rewards/rejected": -0.4410349726676941, + "step": 552 + }, + { + "epoch": 0.34401244167962675, + "grad_norm": 0.45229020714759827, + "learning_rate": 3.6175000000000005e-05, + "log_odds_chosen": 6.850950241088867, + "log_odds_ratio": -0.22252152860164642, + "logits/chosen": 0.07053791731595993, + "logits/rejected": 0.5561230182647705, + "logps/chosen": -1.2072479724884033, + "logps/rejected": -7.82691764831543, + "loss": 2.8457, + "nll_loss": 2.8234715461730957, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12072480469942093, + "rewards/margins": 0.6619669198989868, + "rewards/rejected": -0.7826917767524719, + "step": 553 + }, + { + "epoch": 0.34463452566096425, + "grad_norm": 0.38006460666656494, + "learning_rate": 3.615e-05, + "log_odds_chosen": 4.494772911071777, + "log_odds_ratio": -0.40767160058021545, + "logits/chosen": 0.1025007963180542, + "logits/rejected": 0.28547078371047974, + "logps/chosen": -1.0280405282974243, + "logps/rejected": -5.124785900115967, + "loss": 3.0916, + "nll_loss": 3.050813674926758, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10280404984951019, + "rewards/margins": 0.4096745550632477, + "rewards/rejected": -0.5124785900115967, + "step": 554 + }, + { + "epoch": 0.3452566096423017, + "grad_norm": 0.41790586709976196, + "learning_rate": 3.6125000000000004e-05, + "log_odds_chosen": 9.663520812988281, + "log_odds_ratio": -0.1240549236536026, + "logits/chosen": 0.013118164613842964, + "logits/rejected": 0.7078239917755127, + "logps/chosen": -1.0113468170166016, + "logps/rejected": -10.269187927246094, + "loss": 2.3077, + "nll_loss": 2.2952585220336914, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10113468766212463, + "rewards/margins": 0.9257840514183044, + "rewards/rejected": -1.026918888092041, + "step": 555 + }, + { + "epoch": 0.3458786936236392, + "grad_norm": 0.42868050932884216, + "learning_rate": 3.61e-05, + "log_odds_chosen": 11.801397323608398, + "log_odds_ratio": -0.05558867007493973, + "logits/chosen": 0.11525575071573257, + "logits/rejected": 0.5333128571510315, + "logps/chosen": -0.6986356377601624, + "logps/rejected": -11.682586669921875, + "loss": 2.5445, + "nll_loss": 2.538949728012085, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06986355781555176, + "rewards/margins": 1.0983952283859253, + "rewards/rejected": -1.1682586669921875, + "step": 556 + }, + { + "epoch": 0.34650077760497666, + "grad_norm": 0.3941185772418976, + "learning_rate": 3.6075e-05, + "log_odds_chosen": 9.488451957702637, + "log_odds_ratio": -0.022738970816135406, + "logits/chosen": 0.2211320698261261, + "logits/rejected": 0.9476622343063354, + "logps/chosen": -1.3519847393035889, + "logps/rejected": -10.521648406982422, + "loss": 3.1268, + "nll_loss": 3.1244912147521973, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1351984739303589, + "rewards/margins": 0.9169663190841675, + "rewards/rejected": -1.0521647930145264, + "step": 557 + }, + { + "epoch": 0.34712286158631417, + "grad_norm": 0.5461376309394836, + "learning_rate": 3.605e-05, + "log_odds_chosen": 6.737714767456055, + "log_odds_ratio": -0.270801305770874, + "logits/chosen": 0.20157453417778015, + "logits/rejected": 0.566937267780304, + "logps/chosen": -1.1833453178405762, + "logps/rejected": -7.544872760772705, + "loss": 3.1792, + "nll_loss": 3.152073621749878, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11833453178405762, + "rewards/margins": 0.6361527442932129, + "rewards/rejected": -0.7544872760772705, + "step": 558 + }, + { + "epoch": 0.3477449455676516, + "grad_norm": 0.5232445597648621, + "learning_rate": 3.6025e-05, + "log_odds_chosen": 14.541240692138672, + "log_odds_ratio": -0.1170097142457962, + "logits/chosen": 0.005186443217098713, + "logits/rejected": 0.5835030674934387, + "logps/chosen": -1.1031867265701294, + "logps/rejected": -15.032424926757812, + "loss": 2.6264, + "nll_loss": 2.6146867275238037, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11031867563724518, + "rewards/margins": 1.3929238319396973, + "rewards/rejected": -1.5032424926757812, + "step": 559 + }, + { + "epoch": 0.3483670295489891, + "grad_norm": 0.40398097038269043, + "learning_rate": 3.6e-05, + "log_odds_chosen": 10.217870712280273, + "log_odds_ratio": -0.021281030029058456, + "logits/chosen": 0.26363763213157654, + "logits/rejected": 0.7080330848693848, + "logps/chosen": -1.2973829507827759, + "logps/rejected": -11.150710105895996, + "loss": 3.5862, + "nll_loss": 3.58408260345459, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12973830103874207, + "rewards/margins": 0.985332727432251, + "rewards/rejected": -1.1150710582733154, + "step": 560 + }, + { + "epoch": 0.3489891135303266, + "grad_norm": 0.3686501085758209, + "learning_rate": 3.5975e-05, + "log_odds_chosen": 11.192573547363281, + "log_odds_ratio": -0.00902550108730793, + "logits/chosen": 0.07758531719446182, + "logits/rejected": 0.7558944225311279, + "logps/chosen": -0.9165749549865723, + "logps/rejected": -11.570627212524414, + "loss": 2.5896, + "nll_loss": 2.5887372493743896, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09165749698877335, + "rewards/margins": 1.0654051303863525, + "rewards/rejected": -1.1570627689361572, + "step": 561 + }, + { + "epoch": 0.3496111975116641, + "grad_norm": 0.3752516508102417, + "learning_rate": 3.595e-05, + "log_odds_chosen": 13.246245384216309, + "log_odds_ratio": -0.1449487805366516, + "logits/chosen": 0.2694312334060669, + "logits/rejected": 1.2381070852279663, + "logps/chosen": -1.137485384941101, + "logps/rejected": -14.047223091125488, + "loss": 3.0143, + "nll_loss": 2.999814033508301, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11374853551387787, + "rewards/margins": 1.2909739017486572, + "rewards/rejected": -1.4047224521636963, + "step": 562 + }, + { + "epoch": 0.35023328149300154, + "grad_norm": 0.3949683904647827, + "learning_rate": 3.5925000000000006e-05, + "log_odds_chosen": 8.974077224731445, + "log_odds_ratio": -0.0277986042201519, + "logits/chosen": 0.18479128181934357, + "logits/rejected": 0.6321989297866821, + "logps/chosen": -0.7476387023925781, + "logps/rejected": -8.921815872192383, + "loss": 3.007, + "nll_loss": 3.004196882247925, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07476387172937393, + "rewards/margins": 0.8174177408218384, + "rewards/rejected": -0.8921815752983093, + "step": 563 + }, + { + "epoch": 0.35085536547433904, + "grad_norm": 5.831315994262695, + "learning_rate": 3.59e-05, + "log_odds_chosen": 9.392239570617676, + "log_odds_ratio": -0.17418979108333588, + "logits/chosen": 0.1219196617603302, + "logits/rejected": 0.9300506114959717, + "logps/chosen": -1.233995795249939, + "logps/rejected": -10.267557144165039, + "loss": 2.6961, + "nll_loss": 2.6787209510803223, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12339960038661957, + "rewards/margins": 0.9033560752868652, + "rewards/rejected": -1.026755690574646, + "step": 564 + }, + { + "epoch": 0.3514774494556765, + "grad_norm": 0.4015295207500458, + "learning_rate": 3.5875000000000005e-05, + "log_odds_chosen": 11.647599220275879, + "log_odds_ratio": -0.007850930094718933, + "logits/chosen": 0.2969825267791748, + "logits/rejected": 0.9183492660522461, + "logps/chosen": -1.142836332321167, + "logps/rejected": -12.2813720703125, + "loss": 3.3011, + "nll_loss": 3.3002769947052, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11428363621234894, + "rewards/margins": 1.1138535737991333, + "rewards/rejected": -1.2281371355056763, + "step": 565 + }, + { + "epoch": 0.352099533437014, + "grad_norm": 0.8392999768257141, + "learning_rate": 3.585e-05, + "log_odds_chosen": 15.985481262207031, + "log_odds_ratio": -0.0001611363550182432, + "logits/chosen": 0.33824992179870605, + "logits/rejected": 1.2024070024490356, + "logps/chosen": -1.6567131280899048, + "logps/rejected": -17.20331573486328, + "loss": 3.5828, + "nll_loss": 3.5828089714050293, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16567131876945496, + "rewards/margins": 1.5546603202819824, + "rewards/rejected": -1.7203315496444702, + "step": 566 + }, + { + "epoch": 0.35272161741835145, + "grad_norm": 0.4848373830318451, + "learning_rate": 3.5825000000000003e-05, + "log_odds_chosen": 13.249602317810059, + "log_odds_ratio": -0.12133760005235672, + "logits/chosen": 0.1779986023902893, + "logits/rejected": 1.0257817506790161, + "logps/chosen": -1.069126009941101, + "logps/rejected": -13.913080215454102, + "loss": 2.6563, + "nll_loss": 2.644139289855957, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10691259801387787, + "rewards/margins": 1.284395456314087, + "rewards/rejected": -1.391308069229126, + "step": 567 + }, + { + "epoch": 0.35334370139968896, + "grad_norm": 0.4035547375679016, + "learning_rate": 3.58e-05, + "log_odds_chosen": 13.095964431762695, + "log_odds_ratio": -0.006714683026075363, + "logits/chosen": 0.09633934497833252, + "logits/rejected": 0.7238105535507202, + "logps/chosen": -0.9360955357551575, + "logps/rejected": -13.495253562927246, + "loss": 2.7449, + "nll_loss": 2.744229555130005, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09360955655574799, + "rewards/margins": 1.255915880203247, + "rewards/rejected": -1.3495254516601562, + "step": 568 + }, + { + "epoch": 0.35396578538102647, + "grad_norm": 0.7661040425300598, + "learning_rate": 3.5775e-05, + "log_odds_chosen": 9.719877243041992, + "log_odds_ratio": -0.23540478944778442, + "logits/chosen": 0.3083692789077759, + "logits/rejected": 1.01234769821167, + "logps/chosen": -1.212859034538269, + "logps/rejected": -10.61805534362793, + "loss": 3.1301, + "nll_loss": 3.1065545082092285, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12128590047359467, + "rewards/margins": 0.9405196905136108, + "rewards/rejected": -1.0618056058883667, + "step": 569 + }, + { + "epoch": 0.3545878693623639, + "grad_norm": 0.3926217257976532, + "learning_rate": 3.575e-05, + "log_odds_chosen": 10.073858261108398, + "log_odds_ratio": -0.1235310509800911, + "logits/chosen": 0.13625189661979675, + "logits/rejected": 0.7520464658737183, + "logps/chosen": -1.2251389026641846, + "logps/rejected": -10.796701431274414, + "loss": 2.7511, + "nll_loss": 2.738753318786621, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12251389026641846, + "rewards/margins": 0.957156240940094, + "rewards/rejected": -1.0796700716018677, + "step": 570 + }, + { + "epoch": 0.3552099533437014, + "grad_norm": 0.8033375144004822, + "learning_rate": 3.5725e-05, + "log_odds_chosen": 4.0529279708862305, + "log_odds_ratio": -0.20374912023544312, + "logits/chosen": 0.1487824022769928, + "logits/rejected": 0.3493782877922058, + "logps/chosen": -1.0385345220565796, + "logps/rejected": -4.73582649230957, + "loss": 2.6228, + "nll_loss": 2.6024231910705566, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10385345667600632, + "rewards/margins": 0.369729220867157, + "rewards/rejected": -0.4735826551914215, + "step": 571 + }, + { + "epoch": 0.3558320373250389, + "grad_norm": 0.38423430919647217, + "learning_rate": 3.57e-05, + "log_odds_chosen": 7.926821708679199, + "log_odds_ratio": -0.0515306256711483, + "logits/chosen": 0.3775959610939026, + "logits/rejected": 0.7432839274406433, + "logps/chosen": -1.2405486106872559, + "logps/rejected": -8.742298126220703, + "loss": 3.7806, + "nll_loss": 3.775494337081909, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12405486404895782, + "rewards/margins": 0.7501749992370605, + "rewards/rejected": -0.874229907989502, + "step": 572 + }, + { + "epoch": 0.3564541213063764, + "grad_norm": 0.5379979610443115, + "learning_rate": 3.5675e-05, + "log_odds_chosen": 4.479024410247803, + "log_odds_ratio": -0.45168882608413696, + "logits/chosen": 0.07607246190309525, + "logits/rejected": 0.2333858758211136, + "logps/chosen": -1.1368803977966309, + "logps/rejected": -5.305628776550293, + "loss": 2.7889, + "nll_loss": 2.7437705993652344, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11368802934885025, + "rewards/margins": 0.41687485575675964, + "rewards/rejected": -0.5305629372596741, + "step": 573 + }, + { + "epoch": 0.35707620528771383, + "grad_norm": 1.3011409044265747, + "learning_rate": 3.565e-05, + "log_odds_chosen": 3.907680034637451, + "log_odds_ratio": -0.33975449204444885, + "logits/chosen": 0.2571965754032135, + "logits/rejected": 0.5251089930534363, + "logps/chosen": -1.092237949371338, + "logps/rejected": -4.7046799659729, + "loss": 3.1342, + "nll_loss": 3.1002023220062256, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10922379791736603, + "rewards/margins": 0.36124420166015625, + "rewards/rejected": -0.4704679846763611, + "step": 574 + }, + { + "epoch": 0.35769828926905134, + "grad_norm": 0.6529536247253418, + "learning_rate": 3.5625000000000005e-05, + "log_odds_chosen": 7.877595901489258, + "log_odds_ratio": -0.17618416249752045, + "logits/chosen": 0.3382919430732727, + "logits/rejected": 0.7670020461082458, + "logps/chosen": -1.239711880683899, + "logps/rejected": -8.82756233215332, + "loss": 3.5095, + "nll_loss": 3.4918410778045654, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12397119402885437, + "rewards/margins": 0.7587849497795105, + "rewards/rejected": -0.882756233215332, + "step": 575 + }, + { + "epoch": 0.3583203732503888, + "grad_norm": 0.4566708505153656, + "learning_rate": 3.56e-05, + "log_odds_chosen": 11.294231414794922, + "log_odds_ratio": -0.1296280026435852, + "logits/chosen": 0.2539590299129486, + "logits/rejected": 0.8094176054000854, + "logps/chosen": -1.7060835361480713, + "logps/rejected": -12.409913063049316, + "loss": 3.248, + "nll_loss": 3.234999656677246, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17060835659503937, + "rewards/margins": 1.070383071899414, + "rewards/rejected": -1.2409913539886475, + "step": 576 + }, + { + "epoch": 0.3589424572317263, + "grad_norm": 0.46674543619155884, + "learning_rate": 3.5575000000000004e-05, + "log_odds_chosen": 3.2900829315185547, + "log_odds_ratio": -0.31960904598236084, + "logits/chosen": 0.08701478689908981, + "logits/rejected": 0.1937023103237152, + "logps/chosen": -0.9611541628837585, + "logps/rejected": -3.7472667694091797, + "loss": 2.775, + "nll_loss": 2.743015766143799, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09611541777849197, + "rewards/margins": 0.27861127257347107, + "rewards/rejected": -0.37472671270370483, + "step": 577 + }, + { + "epoch": 0.35956454121306375, + "grad_norm": 0.3776179552078247, + "learning_rate": 3.555e-05, + "log_odds_chosen": 3.8524093627929688, + "log_odds_ratio": -0.1908583641052246, + "logits/chosen": 0.1532333493232727, + "logits/rejected": 0.4199594557285309, + "logps/chosen": -1.2285915613174438, + "logps/rejected": -4.827481269836426, + "loss": 3.0596, + "nll_loss": 3.0405075550079346, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1228591576218605, + "rewards/margins": 0.3598890006542206, + "rewards/rejected": -0.4827481508255005, + "step": 578 + }, + { + "epoch": 0.36018662519440126, + "grad_norm": 0.5118311643600464, + "learning_rate": 3.5525e-05, + "log_odds_chosen": 7.782455921173096, + "log_odds_ratio": -0.2796177864074707, + "logits/chosen": 0.08961069583892822, + "logits/rejected": 0.5031421780586243, + "logps/chosen": -1.2598867416381836, + "logps/rejected": -8.860183715820312, + "loss": 2.4754, + "nll_loss": 2.4474620819091797, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1259886771440506, + "rewards/margins": 0.760029673576355, + "rewards/rejected": -0.8860183954238892, + "step": 579 + }, + { + "epoch": 0.3608087091757387, + "grad_norm": 0.40134724974632263, + "learning_rate": 3.55e-05, + "log_odds_chosen": 10.93133544921875, + "log_odds_ratio": -0.1257864534854889, + "logits/chosen": 0.13825632631778717, + "logits/rejected": 0.43383318185806274, + "logps/chosen": -1.1258553266525269, + "logps/rejected": -11.62828540802002, + "loss": 3.1864, + "nll_loss": 3.173846960067749, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11258553713560104, + "rewards/margins": 1.0502430200576782, + "rewards/rejected": -1.1628284454345703, + "step": 580 + }, + { + "epoch": 0.3614307931570762, + "grad_norm": 0.37659627199172974, + "learning_rate": 3.5475e-05, + "log_odds_chosen": 9.985986709594727, + "log_odds_ratio": -0.02613000012934208, + "logits/chosen": 0.2525095045566559, + "logits/rejected": 0.599713146686554, + "logps/chosen": -1.3548399209976196, + "logps/rejected": -11.0252685546875, + "loss": 3.217, + "nll_loss": 3.214369535446167, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1354840099811554, + "rewards/margins": 0.967042863368988, + "rewards/rejected": -1.1025269031524658, + "step": 581 + }, + { + "epoch": 0.36205287713841366, + "grad_norm": 0.4002346098423004, + "learning_rate": 3.545e-05, + "log_odds_chosen": 7.260525226593018, + "log_odds_ratio": -0.22730308771133423, + "logits/chosen": 0.3146141767501831, + "logits/rejected": 0.7007775902748108, + "logps/chosen": -1.2889691591262817, + "logps/rejected": -8.326926231384277, + "loss": 3.4248, + "nll_loss": 3.4020333290100098, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12889692187309265, + "rewards/margins": 0.7037956714630127, + "rewards/rejected": -0.8326926231384277, + "step": 582 + }, + { + "epoch": 0.36267496111975117, + "grad_norm": 0.4641818404197693, + "learning_rate": 3.5425e-05, + "log_odds_chosen": 8.128804206848145, + "log_odds_ratio": -0.26333796977996826, + "logits/chosen": 0.2824775278568268, + "logits/rejected": 0.928533673286438, + "logps/chosen": -1.3320082426071167, + "logps/rejected": -9.260583877563477, + "loss": 2.9645, + "nll_loss": 2.9382100105285645, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13320083916187286, + "rewards/margins": 0.7928575873374939, + "rewards/rejected": -0.9260584115982056, + "step": 583 + }, + { + "epoch": 0.3632970451010886, + "grad_norm": 0.4908921718597412, + "learning_rate": 3.54e-05, + "log_odds_chosen": 7.648657321929932, + "log_odds_ratio": -0.20667998492717743, + "logits/chosen": 0.1102341040968895, + "logits/rejected": 0.6346576809883118, + "logps/chosen": -1.1072837114334106, + "logps/rejected": -8.343591690063477, + "loss": 2.6697, + "nll_loss": 2.6490468978881836, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11072837561368942, + "rewards/margins": 0.7236307859420776, + "rewards/rejected": -0.8343591094017029, + "step": 584 + }, + { + "epoch": 0.36391912908242613, + "grad_norm": 0.42618241906166077, + "learning_rate": 3.5375e-05, + "log_odds_chosen": 6.4050774574279785, + "log_odds_ratio": -0.31210067868232727, + "logits/chosen": 0.232399120926857, + "logits/rejected": 0.5041719079017639, + "logps/chosen": -1.3073663711547852, + "logps/rejected": -7.530215263366699, + "loss": 3.0651, + "nll_loss": 3.033935785293579, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13073663413524628, + "rewards/margins": 0.6222848892211914, + "rewards/rejected": -0.7530215382575989, + "step": 585 + }, + { + "epoch": 0.3645412130637636, + "grad_norm": 0.3700605034828186, + "learning_rate": 3.535e-05, + "log_odds_chosen": 9.741218566894531, + "log_odds_ratio": -0.2235887050628662, + "logits/chosen": 0.17234095931053162, + "logits/rejected": 0.7051503658294678, + "logps/chosen": -1.2577295303344727, + "logps/rejected": -10.757645606994629, + "loss": 2.9902, + "nll_loss": 2.967867851257324, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12577295303344727, + "rewards/margins": 0.9499915838241577, + "rewards/rejected": -1.075764536857605, + "step": 586 + }, + { + "epoch": 0.3651632970451011, + "grad_norm": 0.4975121021270752, + "learning_rate": 3.5325000000000005e-05, + "log_odds_chosen": 8.96474838256836, + "log_odds_ratio": -0.33923766016960144, + "logits/chosen": 0.3609507977962494, + "logits/rejected": 0.8460711240768433, + "logps/chosen": -1.3260118961334229, + "logps/rejected": -10.090775489807129, + "loss": 3.1647, + "nll_loss": 3.1307742595672607, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13260118663311005, + "rewards/margins": 0.8764763474464417, + "rewards/rejected": -1.009077548980713, + "step": 587 + }, + { + "epoch": 0.3657853810264386, + "grad_norm": 0.7693401575088501, + "learning_rate": 3.53e-05, + "log_odds_chosen": 3.7467236518859863, + "log_odds_ratio": -0.6245080828666687, + "logits/chosen": 0.09779280424118042, + "logits/rejected": 0.26720985770225525, + "logps/chosen": -1.4264414310455322, + "logps/rejected": -4.837800025939941, + "loss": 2.7459, + "nll_loss": 2.6834800243377686, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14264413714408875, + "rewards/margins": 0.3411359190940857, + "rewards/rejected": -0.48378005623817444, + "step": 588 + }, + { + "epoch": 0.36640746500777605, + "grad_norm": 0.41673484444618225, + "learning_rate": 3.5275000000000004e-05, + "log_odds_chosen": 7.740867614746094, + "log_odds_ratio": -0.17404800653457642, + "logits/chosen": 0.23524239659309387, + "logits/rejected": 0.43157610297203064, + "logps/chosen": -0.9836545586585999, + "logps/rejected": -8.333390235900879, + "loss": 3.4373, + "nll_loss": 3.419887065887451, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09836546331644058, + "rewards/margins": 0.7349735498428345, + "rewards/rejected": -0.8333390951156616, + "step": 589 + }, + { + "epoch": 0.36702954898911355, + "grad_norm": 0.5059419870376587, + "learning_rate": 3.525e-05, + "log_odds_chosen": 4.646327018737793, + "log_odds_ratio": -0.17816919088363647, + "logits/chosen": 0.14172425866127014, + "logits/rejected": 0.35883334279060364, + "logps/chosen": -1.3644582033157349, + "logps/rejected": -5.79020881652832, + "loss": 2.8427, + "nll_loss": 2.8248753547668457, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1364458203315735, + "rewards/margins": 0.442575067281723, + "rewards/rejected": -0.5790208578109741, + "step": 590 + }, + { + "epoch": 0.367651632970451, + "grad_norm": 0.45872944593429565, + "learning_rate": 3.5225e-05, + "log_odds_chosen": 14.634015083312988, + "log_odds_ratio": -0.07316752523183823, + "logits/chosen": 0.20786404609680176, + "logits/rejected": 1.1298377513885498, + "logps/chosen": -1.0286473035812378, + "logps/rejected": -15.15748405456543, + "loss": 2.9729, + "nll_loss": 2.965557098388672, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10286473482847214, + "rewards/margins": 1.4128836393356323, + "rewards/rejected": -1.5157485008239746, + "step": 591 + }, + { + "epoch": 0.3682737169517885, + "grad_norm": 0.678117036819458, + "learning_rate": 3.52e-05, + "log_odds_chosen": 6.029167175292969, + "log_odds_ratio": -0.36632686853408813, + "logits/chosen": 0.255694717168808, + "logits/rejected": 0.5321189761161804, + "logps/chosen": -0.940081000328064, + "logps/rejected": -6.712700843811035, + "loss": 3.3547, + "nll_loss": 3.3180642127990723, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09400809556245804, + "rewards/margins": 0.5772619843482971, + "rewards/rejected": -0.6712701320648193, + "step": 592 + }, + { + "epoch": 0.36889580093312596, + "grad_norm": 0.415947824716568, + "learning_rate": 3.5175e-05, + "log_odds_chosen": 9.287468910217285, + "log_odds_ratio": -0.33420291543006897, + "logits/chosen": 0.1860431730747223, + "logits/rejected": 0.6892718076705933, + "logps/chosen": -1.1524367332458496, + "logps/rejected": -10.236968994140625, + "loss": 3.0043, + "nll_loss": 2.9708549976348877, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11524367332458496, + "rewards/margins": 0.9084532260894775, + "rewards/rejected": -1.0236968994140625, + "step": 593 + }, + { + "epoch": 0.36951788491446347, + "grad_norm": 0.6838355660438538, + "learning_rate": 3.515e-05, + "log_odds_chosen": 11.52259635925293, + "log_odds_ratio": -0.16391010582447052, + "logits/chosen": 0.14682823419570923, + "logits/rejected": 0.7776925563812256, + "logps/chosen": -1.0399855375289917, + "logps/rejected": -12.213512420654297, + "loss": 2.6591, + "nll_loss": 2.642709493637085, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10399855673313141, + "rewards/margins": 1.1173527240753174, + "rewards/rejected": -1.2213512659072876, + "step": 594 + }, + { + "epoch": 0.3701399688958009, + "grad_norm": 0.5135987997055054, + "learning_rate": 3.5125e-05, + "log_odds_chosen": 10.131647109985352, + "log_odds_ratio": -0.15441890060901642, + "logits/chosen": 0.1124928668141365, + "logits/rejected": 0.4724903106689453, + "logps/chosen": -1.242720365524292, + "logps/rejected": -11.012077331542969, + "loss": 2.9245, + "nll_loss": 2.9090170860290527, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12427204102277756, + "rewards/margins": 0.9769356846809387, + "rewards/rejected": -1.1012077331542969, + "step": 595 + }, + { + "epoch": 0.3707620528771384, + "grad_norm": 40.30850601196289, + "learning_rate": 3.51e-05, + "log_odds_chosen": 12.990202903747559, + "log_odds_ratio": -0.9076607823371887, + "logits/chosen": 0.37599942088127136, + "logits/rejected": 1.0942538976669312, + "logps/chosen": -3.2952306270599365, + "logps/rejected": -15.915863037109375, + "loss": 3.9839, + "nll_loss": 3.893106460571289, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.32952308654785156, + "rewards/margins": 1.2620632648468018, + "rewards/rejected": -1.5915863513946533, + "step": 596 + }, + { + "epoch": 0.3713841368584759, + "grad_norm": 30.15015983581543, + "learning_rate": 3.5075000000000006e-05, + "log_odds_chosen": 10.713224411010742, + "log_odds_ratio": -0.17117588222026825, + "logits/chosen": 0.28844138979911804, + "logits/rejected": 0.7312687039375305, + "logps/chosen": -2.2271180152893066, + "logps/rejected": -12.698269844055176, + "loss": 3.5309, + "nll_loss": 3.5137851238250732, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.22271178662776947, + "rewards/margins": 1.0471150875091553, + "rewards/rejected": -1.269826889038086, + "step": 597 + }, + { + "epoch": 0.3720062208398134, + "grad_norm": 0.4343259036540985, + "learning_rate": 3.505e-05, + "log_odds_chosen": 16.90182876586914, + "log_odds_ratio": -0.0005982535076327622, + "logits/chosen": 0.18842889368534088, + "logits/rejected": 0.9794631600379944, + "logps/chosen": -1.1533726453781128, + "logps/rejected": -17.613147735595703, + "loss": 3.0408, + "nll_loss": 3.0407166481018066, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11533726751804352, + "rewards/margins": 1.64597749710083, + "rewards/rejected": -1.761314868927002, + "step": 598 + }, + { + "epoch": 0.37262830482115084, + "grad_norm": 0.38888421654701233, + "learning_rate": 3.5025000000000004e-05, + "log_odds_chosen": 6.894892692565918, + "log_odds_ratio": -0.13730782270431519, + "logits/chosen": 0.24703556299209595, + "logits/rejected": 0.6600065231323242, + "logps/chosen": -0.8034927845001221, + "logps/rejected": -7.0167083740234375, + "loss": 3.3792, + "nll_loss": 3.3654870986938477, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08034928143024445, + "rewards/margins": 0.6213215589523315, + "rewards/rejected": -0.7016708254814148, + "step": 599 + }, + { + "epoch": 0.37325038880248834, + "grad_norm": 0.3728528320789337, + "learning_rate": 3.5e-05, + "log_odds_chosen": 10.392243385314941, + "log_odds_ratio": -0.08781400322914124, + "logits/chosen": 0.14524298906326294, + "logits/rejected": 0.8344522714614868, + "logps/chosen": -1.0598855018615723, + "logps/rejected": -11.012116432189941, + "loss": 2.7949, + "nll_loss": 2.786162853240967, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10598855465650558, + "rewards/margins": 0.9952231645584106, + "rewards/rejected": -1.101211667060852, + "step": 600 + }, + { + "epoch": 0.3738724727838258, + "grad_norm": 0.4189922511577606, + "learning_rate": 3.4975e-05, + "log_odds_chosen": 2.0390243530273438, + "log_odds_ratio": -0.39055636525154114, + "logits/chosen": 0.08198841661214828, + "logits/rejected": 0.19098129868507385, + "logps/chosen": -1.0478918552398682, + "logps/rejected": -2.8190433979034424, + "loss": 2.9721, + "nll_loss": 2.932997226715088, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10478918254375458, + "rewards/margins": 0.17711517214775085, + "rewards/rejected": -0.28190433979034424, + "step": 601 + }, + { + "epoch": 0.3744945567651633, + "grad_norm": 0.4921775758266449, + "learning_rate": 3.495e-05, + "log_odds_chosen": 3.0136878490448, + "log_odds_ratio": -0.41104233264923096, + "logits/chosen": 0.2594601511955261, + "logits/rejected": 0.3218687176704407, + "logps/chosen": -1.2040386199951172, + "logps/rejected": -4.0521650314331055, + "loss": 3.3135, + "nll_loss": 3.2724430561065674, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12040386348962784, + "rewards/margins": 0.2848126292228699, + "rewards/rejected": -0.4052164852619171, + "step": 602 + }, + { + "epoch": 0.37511664074650075, + "grad_norm": 0.5495923161506653, + "learning_rate": 3.4925e-05, + "log_odds_chosen": 2.3447442054748535, + "log_odds_ratio": -0.38162410259246826, + "logits/chosen": 0.23126523196697235, + "logits/rejected": 0.14873181283473969, + "logps/chosen": -1.3720605373382568, + "logps/rejected": -3.5602757930755615, + "loss": 3.5564, + "nll_loss": 3.5182876586914062, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1372060626745224, + "rewards/margins": 0.21882151067256927, + "rewards/rejected": -0.3560275733470917, + "step": 603 + }, + { + "epoch": 0.37573872472783826, + "grad_norm": 0.5212722420692444, + "learning_rate": 3.49e-05, + "log_odds_chosen": 6.591350078582764, + "log_odds_ratio": -0.1355379819869995, + "logits/chosen": 0.07449732720851898, + "logits/rejected": 0.338863730430603, + "logps/chosen": -1.1350582838058472, + "logps/rejected": -7.260507583618164, + "loss": 2.7158, + "nll_loss": 2.7022814750671387, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11350581794977188, + "rewards/margins": 0.6125448942184448, + "rewards/rejected": -0.7260507941246033, + "step": 604 + }, + { + "epoch": 0.37636080870917576, + "grad_norm": 0.47101324796676636, + "learning_rate": 3.4875e-05, + "log_odds_chosen": 2.375208616256714, + "log_odds_ratio": -0.4302332103252411, + "logits/chosen": 0.042256325483322144, + "logits/rejected": 0.19507254660129547, + "logps/chosen": -1.2770401239395142, + "logps/rejected": -3.526949882507324, + "loss": 2.6779, + "nll_loss": 2.6349117755889893, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12770400941371918, + "rewards/margins": 0.22499097883701324, + "rewards/rejected": -0.3526949882507324, + "step": 605 + }, + { + "epoch": 0.3769828926905132, + "grad_norm": 0.534155547618866, + "learning_rate": 3.485e-05, + "log_odds_chosen": 4.362737655639648, + "log_odds_ratio": -0.16578631103038788, + "logits/chosen": -0.0013291984796524048, + "logits/rejected": 0.4363200068473816, + "logps/chosen": -1.247031569480896, + "logps/rejected": -5.332918167114258, + "loss": 2.764, + "nll_loss": 2.7474396228790283, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12470315396785736, + "rewards/margins": 0.4085886478424072, + "rewards/rejected": -0.5332918167114258, + "step": 606 + }, + { + "epoch": 0.3776049766718507, + "grad_norm": 0.5048543810844421, + "learning_rate": 3.4825e-05, + "log_odds_chosen": 3.234736919403076, + "log_odds_ratio": -0.36818602681159973, + "logits/chosen": 0.2557424008846283, + "logits/rejected": 0.39144402742385864, + "logps/chosen": -1.2747143507003784, + "logps/rejected": -4.362112998962402, + "loss": 3.2001, + "nll_loss": 3.163267135620117, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1274714320898056, + "rewards/margins": 0.3087398409843445, + "rewards/rejected": -0.4362112581729889, + "step": 607 + }, + { + "epoch": 0.3782270606531882, + "grad_norm": 0.3789282441139221, + "learning_rate": 3.48e-05, + "log_odds_chosen": 5.484085559844971, + "log_odds_ratio": -0.24823862314224243, + "logits/chosen": 0.07289771735668182, + "logits/rejected": 0.3960074782371521, + "logps/chosen": -1.0095453262329102, + "logps/rejected": -6.180008411407471, + "loss": 3.062, + "nll_loss": 3.037135124206543, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10095453262329102, + "rewards/margins": 0.517046332359314, + "rewards/rejected": -0.6180008053779602, + "step": 608 + }, + { + "epoch": 0.3788491446345257, + "grad_norm": 0.45806336402893066, + "learning_rate": 3.4775000000000005e-05, + "log_odds_chosen": 5.055695533752441, + "log_odds_ratio": -0.07001174986362457, + "logits/chosen": 0.29797932505607605, + "logits/rejected": 0.7238092422485352, + "logps/chosen": -1.1003090143203735, + "logps/rejected": -5.755534648895264, + "loss": 3.4197, + "nll_loss": 3.4127392768859863, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11003090441226959, + "rewards/margins": 0.4655225872993469, + "rewards/rejected": -0.5755534768104553, + "step": 609 + }, + { + "epoch": 0.37947122861586313, + "grad_norm": 0.4006802439689636, + "learning_rate": 3.475e-05, + "log_odds_chosen": 8.513465881347656, + "log_odds_ratio": -0.1920199692249298, + "logits/chosen": 0.18292070925235748, + "logits/rejected": 0.8109362721443176, + "logps/chosen": -1.025404691696167, + "logps/rejected": -8.980233192443848, + "loss": 3.2596, + "nll_loss": 3.2403993606567383, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10254047811031342, + "rewards/margins": 0.795482873916626, + "rewards/rejected": -0.8980233073234558, + "step": 610 + }, + { + "epoch": 0.38009331259720064, + "grad_norm": 0.49815765023231506, + "learning_rate": 3.4725000000000004e-05, + "log_odds_chosen": 5.950308799743652, + "log_odds_ratio": -0.26779890060424805, + "logits/chosen": 0.12611763179302216, + "logits/rejected": 0.2929439842700958, + "logps/chosen": -1.2592666149139404, + "logps/rejected": -6.927753925323486, + "loss": 3.173, + "nll_loss": 3.146212100982666, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.125926673412323, + "rewards/margins": 0.5668487548828125, + "rewards/rejected": -0.6927754878997803, + "step": 611 + }, + { + "epoch": 0.3807153965785381, + "grad_norm": 0.44551563262939453, + "learning_rate": 3.4699999999999996e-05, + "log_odds_chosen": 8.533232688903809, + "log_odds_ratio": -0.24877887964248657, + "logits/chosen": 0.3306812644004822, + "logits/rejected": 0.6930794715881348, + "logps/chosen": -0.9571521878242493, + "logps/rejected": -9.140741348266602, + "loss": 3.4917, + "nll_loss": 3.4668092727661133, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09571521729230881, + "rewards/margins": 0.8183589577674866, + "rewards/rejected": -0.914074182510376, + "step": 612 + }, + { + "epoch": 0.3813374805598756, + "grad_norm": 1.205088496208191, + "learning_rate": 3.4675e-05, + "log_odds_chosen": 11.991146087646484, + "log_odds_ratio": -0.0009046989143826067, + "logits/chosen": 0.16803990304470062, + "logits/rejected": 1.0780525207519531, + "logps/chosen": -1.2963323593139648, + "logps/rejected": -12.865903854370117, + "loss": 2.9082, + "nll_loss": 2.9080753326416016, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12963323295116425, + "rewards/margins": 1.1569572687149048, + "rewards/rejected": -1.2865904569625854, + "step": 613 + }, + { + "epoch": 0.38195956454121305, + "grad_norm": 0.6109058260917664, + "learning_rate": 3.465e-05, + "log_odds_chosen": 10.368602752685547, + "log_odds_ratio": -0.05110776424407959, + "logits/chosen": 0.15692661702632904, + "logits/rejected": 0.9058154821395874, + "logps/chosen": -1.3313194513320923, + "logps/rejected": -11.122682571411133, + "loss": 3.1653, + "nll_loss": 3.160191297531128, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1331319510936737, + "rewards/margins": 0.9791362881660461, + "rewards/rejected": -1.112268328666687, + "step": 614 + }, + { + "epoch": 0.38258164852255055, + "grad_norm": 0.3697699308395386, + "learning_rate": 3.4625e-05, + "log_odds_chosen": 6.597249984741211, + "log_odds_ratio": -0.18403266370296478, + "logits/chosen": 0.2341020703315735, + "logits/rejected": 0.8733110427856445, + "logps/chosen": -1.0065994262695312, + "logps/rejected": -7.087155342102051, + "loss": 3.0979, + "nll_loss": 3.0794601440429688, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10065993666648865, + "rewards/margins": 0.6080557107925415, + "rewards/rejected": -0.708715558052063, + "step": 615 + }, + { + "epoch": 0.383203732503888, + "grad_norm": 0.43355652689933777, + "learning_rate": 3.46e-05, + "log_odds_chosen": 5.680283069610596, + "log_odds_ratio": -0.3767112195491791, + "logits/chosen": 0.03800162300467491, + "logits/rejected": 0.27057909965515137, + "logps/chosen": -1.3413360118865967, + "logps/rejected": -6.765846252441406, + "loss": 2.745, + "nll_loss": 2.707333564758301, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13413360714912415, + "rewards/margins": 0.542451024055481, + "rewards/rejected": -0.6765846014022827, + "step": 616 + }, + { + "epoch": 0.3838258164852255, + "grad_norm": 0.3831072151660919, + "learning_rate": 3.4575e-05, + "log_odds_chosen": 5.973001480102539, + "log_odds_ratio": -0.13384968042373657, + "logits/chosen": 0.09891879558563232, + "logits/rejected": 0.39111006259918213, + "logps/chosen": -1.1477863788604736, + "logps/rejected": -6.674656867980957, + "loss": 3.2631, + "nll_loss": 3.2497549057006836, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11477864533662796, + "rewards/margins": 0.5526870489120483, + "rewards/rejected": -0.6674657464027405, + "step": 617 + }, + { + "epoch": 0.38444790046656296, + "grad_norm": 0.4581452012062073, + "learning_rate": 3.455e-05, + "log_odds_chosen": 5.603791236877441, + "log_odds_ratio": -0.1752859354019165, + "logits/chosen": 0.04793360084295273, + "logits/rejected": 0.3412993550300598, + "logps/chosen": -1.081144094467163, + "logps/rejected": -6.33277702331543, + "loss": 3.1135, + "nll_loss": 3.0959632396698, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10811440646648407, + "rewards/margins": 0.5251632928848267, + "rewards/rejected": -0.6332777142524719, + "step": 618 + }, + { + "epoch": 0.38506998444790047, + "grad_norm": 0.421712189912796, + "learning_rate": 3.4525e-05, + "log_odds_chosen": 5.768167972564697, + "log_odds_ratio": -0.22745099663734436, + "logits/chosen": 0.03324780613183975, + "logits/rejected": 0.42689570784568787, + "logps/chosen": -1.1148405075073242, + "logps/rejected": -6.4015655517578125, + "loss": 2.9409, + "nll_loss": 2.9181556701660156, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11148406565189362, + "rewards/margins": 0.5286725163459778, + "rewards/rejected": -0.6401565670967102, + "step": 619 + }, + { + "epoch": 0.3856920684292379, + "grad_norm": 0.4084291458129883, + "learning_rate": 3.45e-05, + "log_odds_chosen": 6.440693378448486, + "log_odds_ratio": -0.1236582025885582, + "logits/chosen": 0.10940997302532196, + "logits/rejected": 0.420942485332489, + "logps/chosen": -1.1581194400787354, + "logps/rejected": -6.912134170532227, + "loss": 3.3024, + "nll_loss": 3.2900736331939697, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11581194400787354, + "rewards/margins": 0.5754014849662781, + "rewards/rejected": -0.6912134289741516, + "step": 620 + }, + { + "epoch": 0.38631415241057543, + "grad_norm": 0.5068473219871521, + "learning_rate": 3.4475000000000005e-05, + "log_odds_chosen": 6.7337493896484375, + "log_odds_ratio": -0.1989458054304123, + "logits/chosen": 0.19425322115421295, + "logits/rejected": 0.316842645406723, + "logps/chosen": -1.122262716293335, + "logps/rejected": -7.526772499084473, + "loss": 3.0876, + "nll_loss": 3.0677075386047363, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11222627013921738, + "rewards/margins": 0.6404510140419006, + "rewards/rejected": -0.7526772618293762, + "step": 621 + }, + { + "epoch": 0.38693623639191294, + "grad_norm": 0.38466599583625793, + "learning_rate": 3.445e-05, + "log_odds_chosen": 5.886903762817383, + "log_odds_ratio": -0.2482847273349762, + "logits/chosen": 0.2649783492088318, + "logits/rejected": 0.7941878437995911, + "logps/chosen": -1.2445883750915527, + "logps/rejected": -6.8715128898620605, + "loss": 3.1309, + "nll_loss": 3.1060690879821777, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12445883452892303, + "rewards/margins": 0.5626925230026245, + "rewards/rejected": -0.687151312828064, + "step": 622 + }, + { + "epoch": 0.3875583203732504, + "grad_norm": 0.40377840399742126, + "learning_rate": 3.4425e-05, + "log_odds_chosen": 8.402645111083984, + "log_odds_ratio": -0.2527872323989868, + "logits/chosen": 0.14706459641456604, + "logits/rejected": 0.5948149561882019, + "logps/chosen": -1.120314598083496, + "logps/rejected": -9.159903526306152, + "loss": 3.1371, + "nll_loss": 3.111776828765869, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11203145235776901, + "rewards/margins": 0.8039589524269104, + "rewards/rejected": -0.9159903526306152, + "step": 623 + }, + { + "epoch": 0.3881804043545879, + "grad_norm": 0.41054055094718933, + "learning_rate": 3.4399999999999996e-05, + "log_odds_chosen": 6.867403984069824, + "log_odds_ratio": -0.08644460141658783, + "logits/chosen": 0.2943398952484131, + "logits/rejected": 0.6636469960212708, + "logps/chosen": -0.9363681077957153, + "logps/rejected": -7.202298641204834, + "loss": 3.3252, + "nll_loss": 3.3165183067321777, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09363681823015213, + "rewards/margins": 0.6265930533409119, + "rewards/rejected": -0.7202298641204834, + "step": 624 + }, + { + "epoch": 0.38880248833592534, + "grad_norm": 0.4753408432006836, + "learning_rate": 3.4375e-05, + "log_odds_chosen": 6.773974418640137, + "log_odds_ratio": -0.2092844694852829, + "logits/chosen": 0.29289454221725464, + "logits/rejected": 0.7853541374206543, + "logps/chosen": -1.3344194889068604, + "logps/rejected": -7.880761623382568, + "loss": 3.0419, + "nll_loss": 3.0209245681762695, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1334419548511505, + "rewards/margins": 0.6546341776847839, + "rewards/rejected": -0.7880761623382568, + "step": 625 + }, + { + "epoch": 0.38942457231726285, + "grad_norm": 0.38249650597572327, + "learning_rate": 3.435e-05, + "log_odds_chosen": 9.644584655761719, + "log_odds_ratio": -0.07656724750995636, + "logits/chosen": 0.32878345251083374, + "logits/rejected": 0.9979957342147827, + "logps/chosen": -0.9833849668502808, + "logps/rejected": -10.156517028808594, + "loss": 3.1942, + "nll_loss": 3.186497688293457, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09833849966526031, + "rewards/margins": 0.917313277721405, + "rewards/rejected": -1.015651822090149, + "step": 626 + }, + { + "epoch": 0.3900466562986003, + "grad_norm": 0.5387607216835022, + "learning_rate": 3.4325e-05, + "log_odds_chosen": 7.158836364746094, + "log_odds_ratio": -0.13461348414421082, + "logits/chosen": 0.11595956236124039, + "logits/rejected": 0.8866112232208252, + "logps/chosen": -1.2095855474472046, + "logps/rejected": -8.037429809570312, + "loss": 2.1498, + "nll_loss": 2.1363189220428467, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12095855176448822, + "rewards/margins": 0.6827844977378845, + "rewards/rejected": -0.8037430047988892, + "step": 627 + }, + { + "epoch": 0.3906687402799378, + "grad_norm": 18.14006233215332, + "learning_rate": 3.430000000000001e-05, + "log_odds_chosen": 8.21702766418457, + "log_odds_ratio": -0.22415418922901154, + "logits/chosen": 0.5142999887466431, + "logits/rejected": 0.9347355961799622, + "logps/chosen": -3.1413345336914062, + "logps/rejected": -11.10336685180664, + "loss": 3.7885, + "nll_loss": 3.7660984992980957, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.31413349509239197, + "rewards/margins": 0.7962032556533813, + "rewards/rejected": -1.1103367805480957, + "step": 628 + }, + { + "epoch": 0.39129082426127526, + "grad_norm": 0.5293332934379578, + "learning_rate": 3.4275e-05, + "log_odds_chosen": 9.935027122497559, + "log_odds_ratio": -0.0972500815987587, + "logits/chosen": 0.372843861579895, + "logits/rejected": 0.5359551906585693, + "logps/chosen": -1.0797069072723389, + "logps/rejected": -10.594209671020508, + "loss": 3.561, + "nll_loss": 3.551283121109009, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10797069221735, + "rewards/margins": 0.9514502882957458, + "rewards/rejected": -1.0594210624694824, + "step": 629 + }, + { + "epoch": 0.39191290824261277, + "grad_norm": 0.5737473368644714, + "learning_rate": 3.4250000000000006e-05, + "log_odds_chosen": 8.50829029083252, + "log_odds_ratio": -0.14946213364601135, + "logits/chosen": 0.21567851305007935, + "logits/rejected": 0.841961681842804, + "logps/chosen": -1.2254951000213623, + "logps/rejected": -9.41191291809082, + "loss": 2.6145, + "nll_loss": 2.599510669708252, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12254951894283295, + "rewards/margins": 0.8186418414115906, + "rewards/rejected": -0.9411913156509399, + "step": 630 + }, + { + "epoch": 0.3925349922239502, + "grad_norm": 0.5078374147415161, + "learning_rate": 3.4225e-05, + "log_odds_chosen": 8.175843238830566, + "log_odds_ratio": -0.14807230234146118, + "logits/chosen": 0.4468088746070862, + "logits/rejected": 0.967327356338501, + "logps/chosen": -0.9474899768829346, + "logps/rejected": -8.668539047241211, + "loss": 2.8416, + "nll_loss": 2.8268425464630127, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09474898874759674, + "rewards/margins": 0.7721049785614014, + "rewards/rejected": -0.8668538928031921, + "step": 631 + }, + { + "epoch": 0.3931570762052877, + "grad_norm": 0.43191152811050415, + "learning_rate": 3.4200000000000005e-05, + "log_odds_chosen": 4.073611736297607, + "log_odds_ratio": -0.16531451046466827, + "logits/chosen": 0.35050833225250244, + "logits/rejected": 0.45885440707206726, + "logps/chosen": -1.4971977472305298, + "logps/rejected": -5.289085388183594, + "loss": 3.174, + "nll_loss": 3.1574785709381104, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14971977472305298, + "rewards/margins": 0.3791887164115906, + "rewards/rejected": -0.5289084911346436, + "step": 632 + }, + { + "epoch": 0.3937791601866252, + "grad_norm": 0.48704293370246887, + "learning_rate": 3.4175000000000004e-05, + "log_odds_chosen": 4.4312849044799805, + "log_odds_ratio": -0.15987420082092285, + "logits/chosen": 0.3942926228046417, + "logits/rejected": 0.5708335638046265, + "logps/chosen": -1.2215713262557983, + "logps/rejected": -5.378739833831787, + "loss": 3.6228, + "nll_loss": 3.6067724227905273, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12215713411569595, + "rewards/margins": 0.41571688652038574, + "rewards/rejected": -0.5378739833831787, + "step": 633 + }, + { + "epoch": 0.3944012441679627, + "grad_norm": 0.5552240014076233, + "learning_rate": 3.415e-05, + "log_odds_chosen": 7.168707847595215, + "log_odds_ratio": -0.04835962504148483, + "logits/chosen": 0.1953887641429901, + "logits/rejected": 0.6829716563224792, + "logps/chosen": -0.9781539440155029, + "logps/rejected": -7.654153347015381, + "loss": 2.4954, + "nll_loss": 2.4905707836151123, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09781539440155029, + "rewards/margins": 0.6675999164581299, + "rewards/rejected": -0.765415370464325, + "step": 634 + }, + { + "epoch": 0.39502332814930013, + "grad_norm": 0.43904614448547363, + "learning_rate": 3.4125e-05, + "log_odds_chosen": 7.5145721435546875, + "log_odds_ratio": -0.01926579885184765, + "logits/chosen": 0.2834244668483734, + "logits/rejected": 0.6822664737701416, + "logps/chosen": -1.2787011861801147, + "logps/rejected": -8.410541534423828, + "loss": 3.4046, + "nll_loss": 3.4026920795440674, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1278701275587082, + "rewards/margins": 0.7131839990615845, + "rewards/rejected": -0.8410541415214539, + "step": 635 + }, + { + "epoch": 0.39564541213063764, + "grad_norm": 0.4341355860233307, + "learning_rate": 3.41e-05, + "log_odds_chosen": 7.731707572937012, + "log_odds_ratio": -0.02020195871591568, + "logits/chosen": 0.2916351556777954, + "logits/rejected": 0.7818939685821533, + "logps/chosen": -0.9819329977035522, + "logps/rejected": -8.171854972839355, + "loss": 2.8691, + "nll_loss": 2.8670694828033447, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09819329530000687, + "rewards/margins": 0.7189922332763672, + "rewards/rejected": -0.8171855211257935, + "step": 636 + }, + { + "epoch": 0.3962674961119751, + "grad_norm": 0.4297373294830322, + "learning_rate": 3.4075e-05, + "log_odds_chosen": 5.247067451477051, + "log_odds_ratio": -0.35224947333335876, + "logits/chosen": 0.16714119911193848, + "logits/rejected": 0.5036913752555847, + "logps/chosen": -1.042916178703308, + "logps/rejected": -6.024389743804932, + "loss": 2.7291, + "nll_loss": 2.6938908100128174, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10429162532091141, + "rewards/margins": 0.4981473386287689, + "rewards/rejected": -0.6024389266967773, + "step": 637 + }, + { + "epoch": 0.3968895800933126, + "grad_norm": 0.5243464112281799, + "learning_rate": 3.405e-05, + "log_odds_chosen": 8.308638572692871, + "log_odds_ratio": -0.2812676429748535, + "logits/chosen": 0.20677152276039124, + "logits/rejected": 0.3971843421459198, + "logps/chosen": -1.1305897235870361, + "logps/rejected": -9.007345199584961, + "loss": 3.0744, + "nll_loss": 3.0463199615478516, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11305898427963257, + "rewards/margins": 0.7876755595207214, + "rewards/rejected": -0.9007344841957092, + "step": 638 + }, + { + "epoch": 0.39751166407465005, + "grad_norm": 0.541507363319397, + "learning_rate": 3.4025e-05, + "log_odds_chosen": 5.680115699768066, + "log_odds_ratio": -0.08863461017608643, + "logits/chosen": 0.23920726776123047, + "logits/rejected": 0.6990938782691956, + "logps/chosen": -1.3689988851547241, + "logps/rejected": -6.771450996398926, + "loss": 2.8984, + "nll_loss": 2.889554023742676, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1368998885154724, + "rewards/margins": 0.5402451753616333, + "rewards/rejected": -0.6771451234817505, + "step": 639 + }, + { + "epoch": 0.39813374805598756, + "grad_norm": 0.44972142577171326, + "learning_rate": 3.4000000000000007e-05, + "log_odds_chosen": 5.760073661804199, + "log_odds_ratio": -0.20923081040382385, + "logits/chosen": 0.28065040707588196, + "logits/rejected": 0.6092424392700195, + "logps/chosen": -1.1194320917129517, + "logps/rejected": -6.457537651062012, + "loss": 2.9208, + "nll_loss": 2.899829864501953, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11194320023059845, + "rewards/margins": 0.533810555934906, + "rewards/rejected": -0.6457537412643433, + "step": 640 + }, + { + "epoch": 0.39875583203732506, + "grad_norm": 0.45881423354148865, + "learning_rate": 3.3975e-05, + "log_odds_chosen": 8.45309066772461, + "log_odds_ratio": -0.002736177993938327, + "logits/chosen": 0.1473774015903473, + "logits/rejected": 0.9392131567001343, + "logps/chosen": -1.1920280456542969, + "logps/rejected": -9.228044509887695, + "loss": 2.4849, + "nll_loss": 2.4846346378326416, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11920280009508133, + "rewards/margins": 0.8036016225814819, + "rewards/rejected": -0.9228044152259827, + "step": 641 + }, + { + "epoch": 0.3993779160186625, + "grad_norm": 0.4157197177410126, + "learning_rate": 3.3950000000000005e-05, + "log_odds_chosen": 7.536615371704102, + "log_odds_ratio": -0.09330558776855469, + "logits/chosen": 0.31922441720962524, + "logits/rejected": 0.7367990016937256, + "logps/chosen": -1.2460134029388428, + "logps/rejected": -8.397119522094727, + "loss": 3.217, + "nll_loss": 3.2076451778411865, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12460136413574219, + "rewards/margins": 0.7151105403900146, + "rewards/rejected": -0.8397119641304016, + "step": 642 + }, + { + "epoch": 0.4, + "grad_norm": 0.3730640113353729, + "learning_rate": 3.3925e-05, + "log_odds_chosen": 7.549066543579102, + "log_odds_ratio": -0.028249753639101982, + "logits/chosen": 0.43140530586242676, + "logits/rejected": 0.803489089012146, + "logps/chosen": -0.825218141078949, + "logps/rejected": -7.683599472045898, + "loss": 3.0414, + "nll_loss": 3.0385513305664062, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08252181112766266, + "rewards/margins": 0.6858382225036621, + "rewards/rejected": -0.7683599591255188, + "step": 643 + }, + { + "epoch": 0.4006220839813375, + "grad_norm": 0.5043125748634338, + "learning_rate": 3.3900000000000004e-05, + "log_odds_chosen": 12.2427978515625, + "log_odds_ratio": -0.06448502838611603, + "logits/chosen": 0.22592878341674805, + "logits/rejected": 0.6281089782714844, + "logps/chosen": -0.8235211968421936, + "logps/rejected": -12.127344131469727, + "loss": 3.0036, + "nll_loss": 2.997192859649658, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08235213160514832, + "rewards/margins": 1.1303822994232178, + "rewards/rejected": -1.2127344608306885, + "step": 644 + }, + { + "epoch": 0.401244167962675, + "grad_norm": 0.5275508165359497, + "learning_rate": 3.3875000000000003e-05, + "log_odds_chosen": 8.873658180236816, + "log_odds_ratio": -0.16682618856430054, + "logits/chosen": 0.2479245662689209, + "logits/rejected": 0.6735783219337463, + "logps/chosen": -1.1058787107467651, + "logps/rejected": -9.506242752075195, + "loss": 2.9091, + "nll_loss": 2.8924357891082764, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11058788001537323, + "rewards/margins": 0.8400364518165588, + "rewards/rejected": -0.9506243467330933, + "step": 645 + }, + { + "epoch": 0.40186625194401243, + "grad_norm": 0.47356879711151123, + "learning_rate": 3.385e-05, + "log_odds_chosen": 6.3066816329956055, + "log_odds_ratio": -0.24541525542736053, + "logits/chosen": 0.4849867820739746, + "logits/rejected": 0.8165839314460754, + "logps/chosen": -1.1368855237960815, + "logps/rejected": -6.978367805480957, + "loss": 3.2928, + "nll_loss": 3.268289089202881, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11368854343891144, + "rewards/margins": 0.5841482877731323, + "rewards/rejected": -0.6978368163108826, + "step": 646 + }, + { + "epoch": 0.40248833592534994, + "grad_norm": 0.5155513882637024, + "learning_rate": 3.3825e-05, + "log_odds_chosen": 5.060737609863281, + "log_odds_ratio": -0.2929985821247101, + "logits/chosen": 0.2869529724121094, + "logits/rejected": 0.567604124546051, + "logps/chosen": -1.1311206817626953, + "logps/rejected": -5.907896995544434, + "loss": 3.0702, + "nll_loss": 3.0408787727355957, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11311206221580505, + "rewards/margins": 0.47767767310142517, + "rewards/rejected": -0.5907896757125854, + "step": 647 + }, + { + "epoch": 0.4031104199066874, + "grad_norm": 0.4957614243030548, + "learning_rate": 3.38e-05, + "log_odds_chosen": 7.915012836456299, + "log_odds_ratio": -0.07648847997188568, + "logits/chosen": 0.3334338963031769, + "logits/rejected": 0.8731058239936829, + "logps/chosen": -0.9914820790290833, + "logps/rejected": -8.388053894042969, + "loss": 2.8132, + "nll_loss": 2.805509090423584, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0991482138633728, + "rewards/margins": 0.7396571636199951, + "rewards/rejected": -0.8388054370880127, + "step": 648 + }, + { + "epoch": 0.4037325038880249, + "grad_norm": 0.48051124811172485, + "learning_rate": 3.3775e-05, + "log_odds_chosen": 11.63776683807373, + "log_odds_ratio": -0.05890395864844322, + "logits/chosen": 0.15852481126785278, + "logits/rejected": 0.6033845543861389, + "logps/chosen": -0.6961344480514526, + "logps/rejected": -11.388937950134277, + "loss": 2.9851, + "nll_loss": 2.9792487621307373, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06961344182491302, + "rewards/margins": 1.0692802667617798, + "rewards/rejected": -1.138893723487854, + "step": 649 + }, + { + "epoch": 0.40435458786936235, + "grad_norm": 0.5910434722900391, + "learning_rate": 3.375000000000001e-05, + "log_odds_chosen": 13.171907424926758, + "log_odds_ratio": -0.08223630487918854, + "logits/chosen": 0.15894870460033417, + "logits/rejected": 0.8937514424324036, + "logps/chosen": -1.2927449941635132, + "logps/rejected": -14.164291381835938, + "loss": 2.5011, + "nll_loss": 2.4928855895996094, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12927450239658356, + "rewards/margins": 1.2871546745300293, + "rewards/rejected": -1.4164291620254517, + "step": 650 + }, + { + "epoch": 0.40497667185069985, + "grad_norm": 2.0554184913635254, + "learning_rate": 3.3725e-05, + "log_odds_chosen": 13.773082733154297, + "log_odds_ratio": -0.07272257655858994, + "logits/chosen": 0.4365695118904114, + "logits/rejected": 1.2860937118530273, + "logps/chosen": -1.0678213834762573, + "logps/rejected": -14.37375545501709, + "loss": 2.8618, + "nll_loss": 2.8545725345611572, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10678213834762573, + "rewards/margins": 1.330593466758728, + "rewards/rejected": -1.437375545501709, + "step": 651 + }, + { + "epoch": 0.4055987558320373, + "grad_norm": 2.0059454441070557, + "learning_rate": 3.3700000000000006e-05, + "log_odds_chosen": 13.936979293823242, + "log_odds_ratio": -0.09149403125047684, + "logits/chosen": 0.09907475113868713, + "logits/rejected": 0.763411283493042, + "logps/chosen": -1.1428678035736084, + "logps/rejected": -14.637571334838867, + "loss": 2.3492, + "nll_loss": 2.3400137424468994, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11428678035736084, + "rewards/margins": 1.3494703769683838, + "rewards/rejected": -1.4637572765350342, + "step": 652 + }, + { + "epoch": 0.4062208398133748, + "grad_norm": 0.46556660532951355, + "learning_rate": 3.3675e-05, + "log_odds_chosen": 7.606475830078125, + "log_odds_ratio": -0.18322187662124634, + "logits/chosen": 0.3579789996147156, + "logits/rejected": 0.8734443783760071, + "logps/chosen": -1.0546009540557861, + "logps/rejected": -8.23729419708252, + "loss": 3.08, + "nll_loss": 3.0616695880889893, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10546009242534637, + "rewards/margins": 0.7182692885398865, + "rewards/rejected": -0.8237293362617493, + "step": 653 + }, + { + "epoch": 0.40684292379471226, + "grad_norm": 0.4319630563259125, + "learning_rate": 3.3650000000000005e-05, + "log_odds_chosen": 9.043044090270996, + "log_odds_ratio": -0.17355716228485107, + "logits/chosen": 0.38297635316848755, + "logits/rejected": 0.8766739964485168, + "logps/chosen": -1.290746808052063, + "logps/rejected": -10.054768562316895, + "loss": 3.6632, + "nll_loss": 3.6458590030670166, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12907469272613525, + "rewards/margins": 0.8764022588729858, + "rewards/rejected": -1.005476951599121, + "step": 654 + }, + { + "epoch": 0.40746500777604977, + "grad_norm": 0.586379885673523, + "learning_rate": 3.3625000000000004e-05, + "log_odds_chosen": 10.436687469482422, + "log_odds_ratio": -0.08339973539113998, + "logits/chosen": 0.01694488525390625, + "logits/rejected": 0.7056764364242554, + "logps/chosen": -1.175871729850769, + "logps/rejected": -11.233123779296875, + "loss": 2.2024, + "nll_loss": 2.1940736770629883, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11758717149496078, + "rewards/margins": 1.0057252645492554, + "rewards/rejected": -1.1233124732971191, + "step": 655 + }, + { + "epoch": 0.4080870917573872, + "grad_norm": 0.5463007688522339, + "learning_rate": 3.3600000000000004e-05, + "log_odds_chosen": 14.558086395263672, + "log_odds_ratio": -0.020145747810602188, + "logits/chosen": 0.1619691252708435, + "logits/rejected": 0.8365485668182373, + "logps/chosen": -1.254634141921997, + "logps/rejected": -15.396265029907227, + "loss": 2.7958, + "nll_loss": 2.793788433074951, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12546342611312866, + "rewards/margins": 1.4141631126403809, + "rewards/rejected": -1.5396264791488647, + "step": 656 + }, + { + "epoch": 0.40870917573872473, + "grad_norm": 1.0127180814743042, + "learning_rate": 3.3575e-05, + "log_odds_chosen": 10.627214431762695, + "log_odds_ratio": -0.025262095034122467, + "logits/chosen": 0.15809279680252075, + "logits/rejected": 1.015772819519043, + "logps/chosen": -1.144571304321289, + "logps/rejected": -11.349461555480957, + "loss": 2.5338, + "nll_loss": 2.5312976837158203, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11445712298154831, + "rewards/margins": 1.020488977432251, + "rewards/rejected": -1.1349461078643799, + "step": 657 + }, + { + "epoch": 0.40933125972006223, + "grad_norm": 0.5098369121551514, + "learning_rate": 3.355e-05, + "log_odds_chosen": 7.605299949645996, + "log_odds_ratio": -0.18348553776741028, + "logits/chosen": 0.14395418763160706, + "logits/rejected": 0.305155873298645, + "logps/chosen": -1.009643793106079, + "logps/rejected": -8.116901397705078, + "loss": 2.8782, + "nll_loss": 2.859889507293701, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10096438229084015, + "rewards/margins": 0.7107258439064026, + "rewards/rejected": -0.8116902112960815, + "step": 658 + }, + { + "epoch": 0.4099533437013997, + "grad_norm": 0.49793869256973267, + "learning_rate": 3.3525e-05, + "log_odds_chosen": 5.807119369506836, + "log_odds_ratio": -0.18563133478164673, + "logits/chosen": 0.1726893037557602, + "logits/rejected": 0.5850039720535278, + "logps/chosen": -1.2937705516815186, + "logps/rejected": -6.849543571472168, + "loss": 2.8515, + "nll_loss": 2.8329498767852783, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12937705218791962, + "rewards/margins": 0.555577278137207, + "rewards/rejected": -0.6849542856216431, + "step": 659 + }, + { + "epoch": 0.4105754276827372, + "grad_norm": 0.4080248773097992, + "learning_rate": 3.35e-05, + "log_odds_chosen": 7.798429489135742, + "log_odds_ratio": -0.033232737332582474, + "logits/chosen": 0.16965456306934357, + "logits/rejected": 0.5689667463302612, + "logps/chosen": -0.8590775728225708, + "logps/rejected": -8.06137466430664, + "loss": 3.1818, + "nll_loss": 3.178438663482666, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08590776473283768, + "rewards/margins": 0.7202296853065491, + "rewards/rejected": -0.8061374425888062, + "step": 660 + }, + { + "epoch": 0.41119751166407464, + "grad_norm": 1.0456312894821167, + "learning_rate": 3.3475e-05, + "log_odds_chosen": 4.1362528800964355, + "log_odds_ratio": -0.4419694244861603, + "logits/chosen": 0.1379992961883545, + "logits/rejected": 0.17854338884353638, + "logps/chosen": -1.8760737180709839, + "logps/rejected": -5.827506065368652, + "loss": 3.5102, + "nll_loss": 3.4659581184387207, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.18760737776756287, + "rewards/margins": 0.39514321088790894, + "rewards/rejected": -0.5827505588531494, + "step": 661 + }, + { + "epoch": 0.41181959564541215, + "grad_norm": 0.45937567949295044, + "learning_rate": 3.345000000000001e-05, + "log_odds_chosen": 6.3919997215271, + "log_odds_ratio": -0.18915338814258575, + "logits/chosen": 0.06113801896572113, + "logits/rejected": 0.46323448419570923, + "logps/chosen": -0.9827958345413208, + "logps/rejected": -6.9341278076171875, + "loss": 2.761, + "nll_loss": 2.742082118988037, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09827959537506104, + "rewards/margins": 0.5951332449913025, + "rewards/rejected": -0.6934128403663635, + "step": 662 + }, + { + "epoch": 0.4124416796267496, + "grad_norm": 0.7443048357963562, + "learning_rate": 3.3425e-05, + "log_odds_chosen": 3.1607913970947266, + "log_odds_ratio": -0.3298979699611664, + "logits/chosen": -0.002762638032436371, + "logits/rejected": 0.15435074269771576, + "logps/chosen": -1.0491981506347656, + "logps/rejected": -3.879762649536133, + "loss": 2.6448, + "nll_loss": 2.6117641925811768, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10491982102394104, + "rewards/margins": 0.28305643796920776, + "rewards/rejected": -0.3879762291908264, + "step": 663 + }, + { + "epoch": 0.4130637636080871, + "grad_norm": 0.5198607444763184, + "learning_rate": 3.3400000000000005e-05, + "log_odds_chosen": 5.001226425170898, + "log_odds_ratio": -0.33517199754714966, + "logits/chosen": 0.09879479557275772, + "logits/rejected": 0.3197081387042999, + "logps/chosen": -1.149256944656372, + "logps/rejected": -5.96102237701416, + "loss": 2.7095, + "nll_loss": 2.6760072708129883, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11492569744586945, + "rewards/margins": 0.48117655515670776, + "rewards/rejected": -0.596102237701416, + "step": 664 + }, + { + "epoch": 0.41368584758942456, + "grad_norm": 0.6322446465492249, + "learning_rate": 3.3375e-05, + "log_odds_chosen": 8.142166137695312, + "log_odds_ratio": -0.5253583788871765, + "logits/chosen": 0.10554549843072891, + "logits/rejected": 0.32331740856170654, + "logps/chosen": -1.1191916465759277, + "logps/rejected": -8.784271240234375, + "loss": 3.2475, + "nll_loss": 3.194953441619873, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11191916465759277, + "rewards/margins": 0.7665079832077026, + "rewards/rejected": -0.8784270882606506, + "step": 665 + }, + { + "epoch": 0.41430793157076207, + "grad_norm": 0.3725062906742096, + "learning_rate": 3.3350000000000004e-05, + "log_odds_chosen": 2.328892707824707, + "log_odds_ratio": -0.46152687072753906, + "logits/chosen": 0.1010928526520729, + "logits/rejected": 0.24071413278579712, + "logps/chosen": -1.1850922107696533, + "logps/rejected": -3.2830145359039307, + "loss": 3.2168, + "nll_loss": 3.170639753341675, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1185092180967331, + "rewards/margins": 0.20979222655296326, + "rewards/rejected": -0.32830148935317993, + "step": 666 + }, + { + "epoch": 0.4149300155520995, + "grad_norm": 0.6286746263504028, + "learning_rate": 3.3325000000000004e-05, + "log_odds_chosen": 4.8994221687316895, + "log_odds_ratio": -0.09839385747909546, + "logits/chosen": 0.04399656876921654, + "logits/rejected": 0.19438612461090088, + "logps/chosen": -1.0532360076904297, + "logps/rejected": -5.433768272399902, + "loss": 2.3705, + "nll_loss": 2.360666275024414, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10532359033823013, + "rewards/margins": 0.43805328011512756, + "rewards/rejected": -0.5433768630027771, + "step": 667 + }, + { + "epoch": 0.415552099533437, + "grad_norm": 0.509515106678009, + "learning_rate": 3.33e-05, + "log_odds_chosen": 1.514922857284546, + "log_odds_ratio": -0.4025351405143738, + "logits/chosen": 0.14341723918914795, + "logits/rejected": 0.2947719693183899, + "logps/chosen": -1.520890474319458, + "logps/rejected": -2.891176223754883, + "loss": 3.3778, + "nll_loss": 3.3375864028930664, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15208904445171356, + "rewards/margins": 0.13702860474586487, + "rewards/rejected": -0.28911763429641724, + "step": 668 + }, + { + "epoch": 0.4161741835147745, + "grad_norm": 0.42655518651008606, + "learning_rate": 3.3275e-05, + "log_odds_chosen": 5.49832820892334, + "log_odds_ratio": -0.04279167205095291, + "logits/chosen": 0.17684528231620789, + "logits/rejected": 0.4185295104980469, + "logps/chosen": -1.1025071144104004, + "logps/rejected": -6.174997329711914, + "loss": 3.0329, + "nll_loss": 3.0286574363708496, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11025071889162064, + "rewards/margins": 0.5072489976882935, + "rewards/rejected": -0.6174997091293335, + "step": 669 + }, + { + "epoch": 0.416796267496112, + "grad_norm": 0.45711860060691833, + "learning_rate": 3.325e-05, + "log_odds_chosen": 5.088669300079346, + "log_odds_ratio": -0.17721472680568695, + "logits/chosen": 0.21003636717796326, + "logits/rejected": 0.4002099633216858, + "logps/chosen": -0.9949367046356201, + "logps/rejected": -5.5031514167785645, + "loss": 3.3491, + "nll_loss": 3.33135724067688, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09949367493391037, + "rewards/margins": 0.45082151889801025, + "rewards/rejected": -0.5503151416778564, + "step": 670 + }, + { + "epoch": 0.41741835147744943, + "grad_norm": 0.4432086646556854, + "learning_rate": 3.3225e-05, + "log_odds_chosen": 7.916843414306641, + "log_odds_ratio": -0.1536819189786911, + "logits/chosen": 0.14663207530975342, + "logits/rejected": 0.4883155822753906, + "logps/chosen": -0.9252321124076843, + "logps/rejected": -8.24629020690918, + "loss": 2.6296, + "nll_loss": 2.6141862869262695, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09252320975065231, + "rewards/margins": 0.7321057915687561, + "rewards/rejected": -0.8246290683746338, + "step": 671 + }, + { + "epoch": 0.41804043545878694, + "grad_norm": 0.37037086486816406, + "learning_rate": 3.32e-05, + "log_odds_chosen": 6.104862213134766, + "log_odds_ratio": -0.2211923599243164, + "logits/chosen": 0.14146625995635986, + "logits/rejected": 0.2648002505302429, + "logps/chosen": -1.257808804512024, + "logps/rejected": -6.962691307067871, + "loss": 3.0737, + "nll_loss": 3.0516157150268555, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1257808804512024, + "rewards/margins": 0.5704882144927979, + "rewards/rejected": -0.696269154548645, + "step": 672 + }, + { + "epoch": 0.4186625194401244, + "grad_norm": 0.3770119845867157, + "learning_rate": 3.3175e-05, + "log_odds_chosen": 7.011809349060059, + "log_odds_ratio": -0.11317376792430878, + "logits/chosen": 0.364218533039093, + "logits/rejected": 0.4188498854637146, + "logps/chosen": -0.8818821907043457, + "logps/rejected": -7.176725387573242, + "loss": 3.8569, + "nll_loss": 3.845541477203369, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08818823099136353, + "rewards/margins": 0.6294843554496765, + "rewards/rejected": -0.71767258644104, + "step": 673 + }, + { + "epoch": 0.4192846034214619, + "grad_norm": 0.3674079179763794, + "learning_rate": 3.3150000000000006e-05, + "log_odds_chosen": 4.395227432250977, + "log_odds_ratio": -0.3885990381240845, + "logits/chosen": 0.1841599941253662, + "logits/rejected": 0.3264680504798889, + "logps/chosen": -1.2810193300247192, + "logps/rejected": -5.530620098114014, + "loss": 3.2217, + "nll_loss": 3.182861804962158, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12810194492340088, + "rewards/margins": 0.42496004700660706, + "rewards/rejected": -0.5530620217323303, + "step": 674 + }, + { + "epoch": 0.4199066874027994, + "grad_norm": 0.3473321199417114, + "learning_rate": 3.3125e-05, + "log_odds_chosen": 7.033344745635986, + "log_odds_ratio": -0.09653866291046143, + "logits/chosen": 0.022332118824124336, + "logits/rejected": 0.26113685965538025, + "logps/chosen": -0.9488670825958252, + "logps/rejected": -7.321063041687012, + "loss": 2.7077, + "nll_loss": 2.698029041290283, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09488672018051147, + "rewards/margins": 0.6372196078300476, + "rewards/rejected": -0.7321063280105591, + "step": 675 + }, + { + "epoch": 0.42052877138413686, + "grad_norm": 0.5475688576698303, + "learning_rate": 3.3100000000000005e-05, + "log_odds_chosen": 4.550343036651611, + "log_odds_ratio": -0.4450215697288513, + "logits/chosen": 0.22917914390563965, + "logits/rejected": 0.44544917345046997, + "logps/chosen": -1.1535780429840088, + "logps/rejected": -5.414106369018555, + "loss": 3.0895, + "nll_loss": 3.044969320297241, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11535780131816864, + "rewards/margins": 0.42605286836624146, + "rewards/rejected": -0.5414106249809265, + "step": 676 + }, + { + "epoch": 0.42115085536547436, + "grad_norm": 0.6116684079170227, + "learning_rate": 3.3075e-05, + "log_odds_chosen": 2.995664596557617, + "log_odds_ratio": -0.6276533603668213, + "logits/chosen": 0.14690755307674408, + "logits/rejected": 0.3131594657897949, + "logps/chosen": -1.151829719543457, + "logps/rejected": -3.948768138885498, + "loss": 2.504, + "nll_loss": 2.4411871433258057, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11518297344446182, + "rewards/margins": 0.2796938121318817, + "rewards/rejected": -0.39487677812576294, + "step": 677 + }, + { + "epoch": 0.4217729393468118, + "grad_norm": 0.4240874946117401, + "learning_rate": 3.3050000000000004e-05, + "log_odds_chosen": 5.284928321838379, + "log_odds_ratio": -0.183100625872612, + "logits/chosen": 0.1283799111843109, + "logits/rejected": 0.2122117280960083, + "logps/chosen": -0.9375210404396057, + "logps/rejected": -5.698101997375488, + "loss": 3.3374, + "nll_loss": 3.319121837615967, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09375210851430893, + "rewards/margins": 0.47605806589126587, + "rewards/rejected": -0.569810152053833, + "step": 678 + }, + { + "epoch": 0.4223950233281493, + "grad_norm": 0.3675094246864319, + "learning_rate": 3.3025e-05, + "log_odds_chosen": 8.166585922241211, + "log_odds_ratio": -0.18817320466041565, + "logits/chosen": 0.23240971565246582, + "logits/rejected": 0.30073827505111694, + "logps/chosen": -0.8773943185806274, + "logps/rejected": -8.555867195129395, + "loss": 3.5129, + "nll_loss": 3.4940483570098877, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08773943036794662, + "rewards/margins": 0.7678471803665161, + "rewards/rejected": -0.8555866479873657, + "step": 679 + }, + { + "epoch": 0.4230171073094868, + "grad_norm": 0.3875802159309387, + "learning_rate": 3.3e-05, + "log_odds_chosen": 7.042316913604736, + "log_odds_ratio": -0.33785781264305115, + "logits/chosen": 0.2113608419895172, + "logits/rejected": 0.20565426349639893, + "logps/chosen": -1.0790364742279053, + "logps/rejected": -7.6537089347839355, + "loss": 3.4207, + "nll_loss": 3.3869504928588867, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10790365934371948, + "rewards/margins": 0.6574673056602478, + "rewards/rejected": -0.7653709053993225, + "step": 680 + }, + { + "epoch": 0.4236391912908243, + "grad_norm": 0.47894734144210815, + "learning_rate": 3.2975e-05, + "log_odds_chosen": 8.363826751708984, + "log_odds_ratio": -0.121647410094738, + "logits/chosen": 0.16830061376094818, + "logits/rejected": 0.7813224196434021, + "logps/chosen": -1.120478630065918, + "logps/rejected": -9.116507530212402, + "loss": 2.7327, + "nll_loss": 2.720547676086426, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11204788088798523, + "rewards/margins": 0.7996028661727905, + "rewards/rejected": -0.9116507768630981, + "step": 681 + }, + { + "epoch": 0.42426127527216173, + "grad_norm": 0.5742572546005249, + "learning_rate": 3.295e-05, + "log_odds_chosen": 8.133566856384277, + "log_odds_ratio": -0.07514964044094086, + "logits/chosen": 0.10974755883216858, + "logits/rejected": 0.43608176708221436, + "logps/chosen": -0.74546879529953, + "logps/rejected": -8.146604537963867, + "loss": 2.561, + "nll_loss": 2.553471803665161, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07454688102006912, + "rewards/margins": 0.7401136159896851, + "rewards/rejected": -0.8146604299545288, + "step": 682 + }, + { + "epoch": 0.42488335925349924, + "grad_norm": 0.4656708240509033, + "learning_rate": 3.2925e-05, + "log_odds_chosen": 5.774636745452881, + "log_odds_ratio": -0.13835880160331726, + "logits/chosen": 0.3287077248096466, + "logits/rejected": 0.6844013929367065, + "logps/chosen": -1.2045843601226807, + "logps/rejected": -6.656024932861328, + "loss": 3.3318, + "nll_loss": 3.3180010318756104, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12045843154191971, + "rewards/margins": 0.5451440811157227, + "rewards/rejected": -0.6656025052070618, + "step": 683 + }, + { + "epoch": 0.4255054432348367, + "grad_norm": 0.4819745421409607, + "learning_rate": 3.29e-05, + "log_odds_chosen": 11.101076126098633, + "log_odds_ratio": -0.05002117529511452, + "logits/chosen": 0.4243965446949005, + "logits/rejected": 1.0198917388916016, + "logps/chosen": -1.1497440338134766, + "logps/rejected": -11.876899719238281, + "loss": 3.3201, + "nll_loss": 3.315087080001831, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11497440934181213, + "rewards/margins": 1.0727156400680542, + "rewards/rejected": -1.187690019607544, + "step": 684 + }, + { + "epoch": 0.4261275272161742, + "grad_norm": 0.5185660719871521, + "learning_rate": 3.2875e-05, + "log_odds_chosen": 11.11820125579834, + "log_odds_ratio": -0.0768592581152916, + "logits/chosen": 0.29388269782066345, + "logits/rejected": 0.9650092124938965, + "logps/chosen": -1.116542100906372, + "logps/rejected": -11.861469268798828, + "loss": 2.4838, + "nll_loss": 2.4760661125183105, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11165420711040497, + "rewards/margins": 1.0744926929473877, + "rewards/rejected": -1.186146855354309, + "step": 685 + }, + { + "epoch": 0.42674961119751165, + "grad_norm": 0.6650855541229248, + "learning_rate": 3.2850000000000006e-05, + "log_odds_chosen": 5.216774940490723, + "log_odds_ratio": -0.18936312198638916, + "logits/chosen": 0.33131909370422363, + "logits/rejected": 0.5103744864463806, + "logps/chosen": -1.0327638387680054, + "logps/rejected": -5.904963493347168, + "loss": 2.7629, + "nll_loss": 2.7439844608306885, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10327637195587158, + "rewards/margins": 0.48721998929977417, + "rewards/rejected": -0.5904964208602905, + "step": 686 + }, + { + "epoch": 0.42737169517884915, + "grad_norm": 0.48068517446517944, + "learning_rate": 3.2825e-05, + "log_odds_chosen": 7.785678386688232, + "log_odds_ratio": -0.4847630262374878, + "logits/chosen": 0.31015443801879883, + "logits/rejected": 0.8882300853729248, + "logps/chosen": -1.257796049118042, + "logps/rejected": -8.93236255645752, + "loss": 3.1948, + "nll_loss": 3.146324872970581, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12577959895133972, + "rewards/margins": 0.767456591129303, + "rewards/rejected": -0.8932361602783203, + "step": 687 + }, + { + "epoch": 0.4279937791601866, + "grad_norm": 0.5043879747390747, + "learning_rate": 3.2800000000000004e-05, + "log_odds_chosen": 9.975924491882324, + "log_odds_ratio": -0.09700936079025269, + "logits/chosen": 0.4725090265274048, + "logits/rejected": 0.9532057642936707, + "logps/chosen": -1.3929022550582886, + "logps/rejected": -11.091150283813477, + "loss": 3.2531, + "nll_loss": 3.2434072494506836, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1392902284860611, + "rewards/margins": 0.9698247909545898, + "rewards/rejected": -1.1091151237487793, + "step": 688 + }, + { + "epoch": 0.4286158631415241, + "grad_norm": 0.379336953163147, + "learning_rate": 3.2775e-05, + "log_odds_chosen": 6.284054279327393, + "log_odds_ratio": -0.14844508469104767, + "logits/chosen": 0.22702661156654358, + "logits/rejected": 0.30767202377319336, + "logps/chosen": -0.7351300120353699, + "logps/rejected": -6.354801654815674, + "loss": 3.1944, + "nll_loss": 3.179506778717041, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07351300120353699, + "rewards/margins": 0.561967134475708, + "rewards/rejected": -0.6354801058769226, + "step": 689 + }, + { + "epoch": 0.42923794712286156, + "grad_norm": 0.3885799050331116, + "learning_rate": 3.275e-05, + "log_odds_chosen": 8.412586212158203, + "log_odds_ratio": -0.14217258989810944, + "logits/chosen": 0.307309627532959, + "logits/rejected": 0.5339474081993103, + "logps/chosen": -1.0682765245437622, + "logps/rejected": -9.073349952697754, + "loss": 3.3597, + "nll_loss": 3.3454573154449463, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10682766139507294, + "rewards/margins": 0.8005073070526123, + "rewards/rejected": -0.9073349833488464, + "step": 690 + }, + { + "epoch": 0.42986003110419907, + "grad_norm": 0.4812905192375183, + "learning_rate": 3.2725e-05, + "log_odds_chosen": 4.128777503967285, + "log_odds_ratio": -0.2909623384475708, + "logits/chosen": 0.2246783971786499, + "logits/rejected": 0.3128810524940491, + "logps/chosen": -0.982389509677887, + "logps/rejected": -4.593428134918213, + "loss": 3.1373, + "nll_loss": 3.1081809997558594, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09823895990848541, + "rewards/margins": 0.361103892326355, + "rewards/rejected": -0.4593428373336792, + "step": 691 + }, + { + "epoch": 0.4304821150855365, + "grad_norm": 0.4728715121746063, + "learning_rate": 3.27e-05, + "log_odds_chosen": 8.828766822814941, + "log_odds_ratio": -0.32048314809799194, + "logits/chosen": 0.239055335521698, + "logits/rejected": 0.45625340938568115, + "logps/chosen": -0.9721330404281616, + "logps/rejected": -9.417234420776367, + "loss": 3.3518, + "nll_loss": 3.319707155227661, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09721331298351288, + "rewards/margins": 0.8445100784301758, + "rewards/rejected": -0.9417234659194946, + "step": 692 + }, + { + "epoch": 0.431104199066874, + "grad_norm": 0.6615074872970581, + "learning_rate": 3.2675e-05, + "log_odds_chosen": 3.707711696624756, + "log_odds_ratio": -0.5199106931686401, + "logits/chosen": 0.13658814132213593, + "logits/rejected": 0.24072976410388947, + "logps/chosen": -1.3815832138061523, + "logps/rejected": -4.977440357208252, + "loss": 3.5352, + "nll_loss": 3.483212947845459, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13815832138061523, + "rewards/margins": 0.359585702419281, + "rewards/rejected": -0.49774405360221863, + "step": 693 + }, + { + "epoch": 0.43172628304821153, + "grad_norm": 0.5314946174621582, + "learning_rate": 3.265e-05, + "log_odds_chosen": 12.457331657409668, + "log_odds_ratio": -0.14057813584804535, + "logits/chosen": 0.2213817536830902, + "logits/rejected": 0.8500862121582031, + "logps/chosen": -1.0103843212127686, + "logps/rejected": -12.985528945922852, + "loss": 2.7115, + "nll_loss": 2.697409152984619, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10103843361139297, + "rewards/margins": 1.197514295578003, + "rewards/rejected": -1.2985528707504272, + "step": 694 + }, + { + "epoch": 0.432348367029549, + "grad_norm": 0.47115829586982727, + "learning_rate": 3.2625e-05, + "log_odds_chosen": 16.424076080322266, + "log_odds_ratio": -0.10967248678207397, + "logits/chosen": 0.17491212487220764, + "logits/rejected": 0.6353819370269775, + "logps/chosen": -1.1987203359603882, + "logps/rejected": -17.27529525756836, + "loss": 3.0507, + "nll_loss": 3.0397396087646484, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11987203359603882, + "rewards/margins": 1.6076574325561523, + "rewards/rejected": -1.727529525756836, + "step": 695 + }, + { + "epoch": 0.4329704510108865, + "grad_norm": 0.4146907329559326, + "learning_rate": 3.26e-05, + "log_odds_chosen": 10.058050155639648, + "log_odds_ratio": -0.004779008217155933, + "logits/chosen": 0.33498451113700867, + "logits/rejected": 0.9141867160797119, + "logps/chosen": -1.1475989818572998, + "logps/rejected": -10.654457092285156, + "loss": 3.33, + "nll_loss": 3.329507350921631, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1147598996758461, + "rewards/margins": 0.9506858587265015, + "rewards/rejected": -1.0654456615447998, + "step": 696 + }, + { + "epoch": 0.43359253499222394, + "grad_norm": 0.771777868270874, + "learning_rate": 3.2575e-05, + "log_odds_chosen": 5.501579761505127, + "log_odds_ratio": -0.4520403742790222, + "logits/chosen": 0.11421041190624237, + "logits/rejected": 0.4084751307964325, + "logps/chosen": -1.05423903465271, + "logps/rejected": -6.390564441680908, + "loss": 2.4735, + "nll_loss": 2.4283266067504883, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10542389750480652, + "rewards/margins": 0.5336325764656067, + "rewards/rejected": -0.6390565037727356, + "step": 697 + }, + { + "epoch": 0.43421461897356145, + "grad_norm": 0.41613906621932983, + "learning_rate": 3.2550000000000005e-05, + "log_odds_chosen": 11.464178085327148, + "log_odds_ratio": -0.3043498396873474, + "logits/chosen": 0.23589307069778442, + "logits/rejected": 0.7353752851486206, + "logps/chosen": -1.2409882545471191, + "logps/rejected": -12.499031066894531, + "loss": 3.0398, + "nll_loss": 3.0093822479248047, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12409882992506027, + "rewards/margins": 1.1258044242858887, + "rewards/rejected": -1.2499030828475952, + "step": 698 + }, + { + "epoch": 0.4348367029548989, + "grad_norm": 0.41328713297843933, + "learning_rate": 3.2525e-05, + "log_odds_chosen": 10.817594528198242, + "log_odds_ratio": -0.10718300193548203, + "logits/chosen": 0.25043705105781555, + "logits/rejected": 0.6431066393852234, + "logps/chosen": -0.8873676061630249, + "logps/rejected": -11.163918495178223, + "loss": 3.1033, + "nll_loss": 3.0926289558410645, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08873676508665085, + "rewards/margins": 1.0276551246643066, + "rewards/rejected": -1.1163920164108276, + "step": 699 + }, + { + "epoch": 0.4354587869362364, + "grad_norm": 0.4063067138195038, + "learning_rate": 3.2500000000000004e-05, + "log_odds_chosen": 12.294190406799316, + "log_odds_ratio": -0.12343909591436386, + "logits/chosen": 0.33206403255462646, + "logits/rejected": 0.7490406632423401, + "logps/chosen": -0.9793272614479065, + "logps/rejected": -12.811656951904297, + "loss": 3.2405, + "nll_loss": 3.2281508445739746, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09793273359537125, + "rewards/margins": 1.1832330226898193, + "rewards/rejected": -1.2811657190322876, + "step": 700 + }, + { + "epoch": 0.43608087091757386, + "grad_norm": 9.865860939025879, + "learning_rate": 3.2474999999999997e-05, + "log_odds_chosen": 8.678180694580078, + "log_odds_ratio": -0.7480700016021729, + "logits/chosen": 0.48775994777679443, + "logits/rejected": 1.0069730281829834, + "logps/chosen": -3.711902618408203, + "logps/rejected": -12.224882125854492, + "loss": 3.892, + "nll_loss": 3.8171887397766113, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.37119027972221375, + "rewards/margins": 0.8512980341911316, + "rewards/rejected": -1.2224884033203125, + "step": 701 + }, + { + "epoch": 0.43670295489891137, + "grad_norm": 15.485320091247559, + "learning_rate": 3.245e-05, + "log_odds_chosen": 10.974223136901855, + "log_odds_ratio": -0.2399260401725769, + "logits/chosen": 0.2739264965057373, + "logits/rejected": 0.861457347869873, + "logps/chosen": -1.5031821727752686, + "logps/rejected": -12.213601112365723, + "loss": 3.1174, + "nll_loss": 3.093437671661377, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1503182202577591, + "rewards/margins": 1.0710419416427612, + "rewards/rejected": -1.221360206604004, + "step": 702 + }, + { + "epoch": 0.4373250388802488, + "grad_norm": 0.6337563395500183, + "learning_rate": 3.2425e-05, + "log_odds_chosen": 10.547877311706543, + "log_odds_ratio": -0.17687174677848816, + "logits/chosen": 0.16326065361499786, + "logits/rejected": 0.6496604084968567, + "logps/chosen": -1.749812364578247, + "logps/rejected": -12.043559074401855, + "loss": 3.144, + "nll_loss": 3.126358985900879, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1749812364578247, + "rewards/margins": 1.0293747186660767, + "rewards/rejected": -1.2043559551239014, + "step": 703 + }, + { + "epoch": 0.4379471228615863, + "grad_norm": 0.4531756043434143, + "learning_rate": 3.24e-05, + "log_odds_chosen": 7.200477600097656, + "log_odds_ratio": -0.31999287009239197, + "logits/chosen": 0.2357894331216812, + "logits/rejected": 0.5349158644676208, + "logps/chosen": -1.258987545967102, + "logps/rejected": -8.253671646118164, + "loss": 3.0629, + "nll_loss": 3.030871868133545, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12589874863624573, + "rewards/margins": 0.6994683146476746, + "rewards/rejected": -0.8253670930862427, + "step": 704 + }, + { + "epoch": 0.4385692068429238, + "grad_norm": 0.9053670167922974, + "learning_rate": 3.2375e-05, + "log_odds_chosen": 10.318868637084961, + "log_odds_ratio": -0.1663799285888672, + "logits/chosen": 0.26653003692626953, + "logits/rejected": 0.845885157585144, + "logps/chosen": -1.1329401731491089, + "logps/rejected": -11.080015182495117, + "loss": 2.6064, + "nll_loss": 2.589715003967285, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11329401284456253, + "rewards/margins": 0.994707465171814, + "rewards/rejected": -1.108001470565796, + "step": 705 + }, + { + "epoch": 0.4391912908242613, + "grad_norm": 0.7091290950775146, + "learning_rate": 3.235e-05, + "log_odds_chosen": 5.621170997619629, + "log_odds_ratio": -0.23729656636714935, + "logits/chosen": 0.16418294608592987, + "logits/rejected": 0.46262502670288086, + "logps/chosen": -1.284653663635254, + "logps/rejected": -6.603672027587891, + "loss": 2.5671, + "nll_loss": 2.543408155441284, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12846536934375763, + "rewards/margins": 0.5319017767906189, + "rewards/rejected": -0.6603671908378601, + "step": 706 + }, + { + "epoch": 0.43981337480559873, + "grad_norm": 0.4145105183124542, + "learning_rate": 3.2325e-05, + "log_odds_chosen": 11.086092948913574, + "log_odds_ratio": -0.12887164950370789, + "logits/chosen": 0.19201578199863434, + "logits/rejected": 0.8350374102592468, + "logps/chosen": -1.087384581565857, + "logps/rejected": -11.732353210449219, + "loss": 2.7902, + "nll_loss": 2.77732515335083, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10873845964670181, + "rewards/margins": 1.0644968748092651, + "rewards/rejected": -1.1732354164123535, + "step": 707 + }, + { + "epoch": 0.44043545878693624, + "grad_norm": 0.4064066708087921, + "learning_rate": 3.2300000000000006e-05, + "log_odds_chosen": 9.882148742675781, + "log_odds_ratio": -0.08438535779714584, + "logits/chosen": 0.2323889285326004, + "logits/rejected": 0.5477718710899353, + "logps/chosen": -1.0335191488265991, + "logps/rejected": -10.476679801940918, + "loss": 3.3664, + "nll_loss": 3.3579728603363037, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10335192084312439, + "rewards/margins": 0.944316029548645, + "rewards/rejected": -1.0476679801940918, + "step": 708 + }, + { + "epoch": 0.4410575427682737, + "grad_norm": 0.46960577368736267, + "learning_rate": 3.2275e-05, + "log_odds_chosen": 9.509929656982422, + "log_odds_ratio": -0.04786451533436775, + "logits/chosen": 0.3471740186214447, + "logits/rejected": 0.7918584942817688, + "logps/chosen": -1.0728213787078857, + "logps/rejected": -10.050264358520508, + "loss": 3.4729, + "nll_loss": 3.468099355697632, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1072821393609047, + "rewards/margins": 0.8977442979812622, + "rewards/rejected": -1.0050263404846191, + "step": 709 + }, + { + "epoch": 0.4416796267496112, + "grad_norm": 0.4803198277950287, + "learning_rate": 3.2250000000000005e-05, + "log_odds_chosen": 9.757268905639648, + "log_odds_ratio": -0.057732146233320236, + "logits/chosen": 0.23063698410987854, + "logits/rejected": 0.6390030384063721, + "logps/chosen": -1.1186230182647705, + "logps/rejected": -10.331436157226562, + "loss": 2.9499, + "nll_loss": 2.944155693054199, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11186229437589645, + "rewards/margins": 0.9212814569473267, + "rewards/rejected": -1.0331437587738037, + "step": 710 + }, + { + "epoch": 0.4423017107309487, + "grad_norm": 0.5018370747566223, + "learning_rate": 3.2225e-05, + "log_odds_chosen": 7.726747512817383, + "log_odds_ratio": -0.13159742951393127, + "logits/chosen": 0.19366326928138733, + "logits/rejected": 0.6917151808738708, + "logps/chosen": -1.2360267639160156, + "logps/rejected": -8.579374313354492, + "loss": 2.7357, + "nll_loss": 2.722532272338867, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12360267341136932, + "rewards/margins": 0.7343348264694214, + "rewards/rejected": -0.8579374551773071, + "step": 711 + }, + { + "epoch": 0.44292379471228616, + "grad_norm": 0.38528966903686523, + "learning_rate": 3.2200000000000003e-05, + "log_odds_chosen": 7.1198835372924805, + "log_odds_ratio": -0.08342044800519943, + "logits/chosen": 0.08595260232686996, + "logits/rejected": 0.31091731786727905, + "logps/chosen": -0.8347682952880859, + "logps/rejected": -7.294371604919434, + "loss": 2.9365, + "nll_loss": 2.9281294345855713, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08347682654857635, + "rewards/margins": 0.6459603309631348, + "rewards/rejected": -0.7294372320175171, + "step": 712 + }, + { + "epoch": 0.44354587869362366, + "grad_norm": 0.4061030447483063, + "learning_rate": 3.2175e-05, + "log_odds_chosen": 10.162424087524414, + "log_odds_ratio": -0.04226357489824295, + "logits/chosen": 0.4276748597621918, + "logits/rejected": 0.6757638454437256, + "logps/chosen": -1.3077592849731445, + "logps/rejected": -11.131202697753906, + "loss": 3.7205, + "nll_loss": 3.7162415981292725, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13077592849731445, + "rewards/margins": 0.982344388961792, + "rewards/rejected": -1.1131203174591064, + "step": 713 + }, + { + "epoch": 0.4441679626749611, + "grad_norm": 0.4195731282234192, + "learning_rate": 3.215e-05, + "log_odds_chosen": 6.164430618286133, + "log_odds_ratio": -0.18825969099998474, + "logits/chosen": 0.3099673092365265, + "logits/rejected": 0.4048096835613251, + "logps/chosen": -1.0167288780212402, + "logps/rejected": -6.674760818481445, + "loss": 3.1177, + "nll_loss": 3.098839282989502, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10167289525270462, + "rewards/margins": 0.5658032298088074, + "rewards/rejected": -0.6674761176109314, + "step": 714 + }, + { + "epoch": 0.4447900466562986, + "grad_norm": 0.45698559284210205, + "learning_rate": 3.2125e-05, + "log_odds_chosen": 6.022528648376465, + "log_odds_ratio": -0.13800564408302307, + "logits/chosen": 0.34452202916145325, + "logits/rejected": 0.5671758055686951, + "logps/chosen": -1.0680052042007446, + "logps/rejected": -6.720895290374756, + "loss": 3.0917, + "nll_loss": 3.077878475189209, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10680052638053894, + "rewards/margins": 0.5652889609336853, + "rewards/rejected": -0.6720895171165466, + "step": 715 + }, + { + "epoch": 0.4454121306376361, + "grad_norm": 0.5258144736289978, + "learning_rate": 3.21e-05, + "log_odds_chosen": 7.468056678771973, + "log_odds_ratio": -0.05660433694720268, + "logits/chosen": 0.29723796248435974, + "logits/rejected": 0.726962685585022, + "logps/chosen": -1.1292157173156738, + "logps/rejected": -8.210090637207031, + "loss": 2.5706, + "nll_loss": 2.564983367919922, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1129215732216835, + "rewards/margins": 0.7080875039100647, + "rewards/rejected": -0.8210090398788452, + "step": 716 + }, + { + "epoch": 0.4460342146189736, + "grad_norm": 0.6525778770446777, + "learning_rate": 3.2075e-05, + "log_odds_chosen": 8.634244918823242, + "log_odds_ratio": -0.2098091095685959, + "logits/chosen": 0.3678334951400757, + "logits/rejected": 0.6341312527656555, + "logps/chosen": -1.0692224502563477, + "logps/rejected": -8.939923286437988, + "loss": 3.4355, + "nll_loss": 3.4145524501800537, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10692223906517029, + "rewards/margins": 0.7870701551437378, + "rewards/rejected": -0.8939923644065857, + "step": 717 + }, + { + "epoch": 0.44665629860031103, + "grad_norm": 0.44750717282295227, + "learning_rate": 3.205e-05, + "log_odds_chosen": 3.406766653060913, + "log_odds_ratio": -0.31649070978164673, + "logits/chosen": 0.3066599369049072, + "logits/rejected": 0.2885739207267761, + "logps/chosen": -1.0535868406295776, + "logps/rejected": -4.101262092590332, + "loss": 3.3799, + "nll_loss": 3.3482348918914795, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10535868257284164, + "rewards/margins": 0.3047674894332886, + "rewards/rejected": -0.4101262092590332, + "step": 718 + }, + { + "epoch": 0.44727838258164854, + "grad_norm": 0.46216386556625366, + "learning_rate": 3.2025e-05, + "log_odds_chosen": 9.778112411499023, + "log_odds_ratio": -0.0030376752838492393, + "logits/chosen": 0.33077576756477356, + "logits/rejected": 0.7403644323348999, + "logps/chosen": -1.0820248126983643, + "logps/rejected": -10.346675872802734, + "loss": 3.2615, + "nll_loss": 3.261215925216675, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1082024872303009, + "rewards/margins": 0.9264651536941528, + "rewards/rejected": -1.034667730331421, + "step": 719 + }, + { + "epoch": 0.447900466562986, + "grad_norm": 0.749276340007782, + "learning_rate": 3.2000000000000005e-05, + "log_odds_chosen": 5.970105171203613, + "log_odds_ratio": -0.3304394483566284, + "logits/chosen": 0.16680698096752167, + "logits/rejected": 0.3060448467731476, + "logps/chosen": -1.435762882232666, + "logps/rejected": -7.024744033813477, + "loss": 2.9031, + "nll_loss": 2.870039463043213, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14357629418373108, + "rewards/margins": 0.5588980913162231, + "rewards/rejected": -0.7024743556976318, + "step": 720 + }, + { + "epoch": 0.4485225505443235, + "grad_norm": 0.539753794670105, + "learning_rate": 3.1975e-05, + "log_odds_chosen": 3.475248098373413, + "log_odds_ratio": -0.529831051826477, + "logits/chosen": 0.2159595638513565, + "logits/rejected": 0.42169877886772156, + "logps/chosen": -1.6044065952301025, + "logps/rejected": -5.004334926605225, + "loss": 3.2259, + "nll_loss": 3.172966241836548, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.16044065356254578, + "rewards/margins": 0.33999279141426086, + "rewards/rejected": -0.5004334449768066, + "step": 721 + }, + { + "epoch": 0.44914463452566095, + "grad_norm": 0.452215313911438, + "learning_rate": 3.1950000000000004e-05, + "log_odds_chosen": 7.956733226776123, + "log_odds_ratio": -0.3105076849460602, + "logits/chosen": 0.16990184783935547, + "logits/rejected": 0.48880094289779663, + "logps/chosen": -1.11954927444458, + "logps/rejected": -8.724953651428223, + "loss": 2.9063, + "nll_loss": 2.8752598762512207, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1119549423456192, + "rewards/margins": 0.7605404257774353, + "rewards/rejected": -0.8724953532218933, + "step": 722 + }, + { + "epoch": 0.44976671850699845, + "grad_norm": 0.5523648262023926, + "learning_rate": 3.1925e-05, + "log_odds_chosen": 6.903494358062744, + "log_odds_ratio": -0.2759135067462921, + "logits/chosen": 0.0760892778635025, + "logits/rejected": 0.32330164313316345, + "logps/chosen": -1.1168400049209595, + "logps/rejected": -7.690395832061768, + "loss": 2.3726, + "nll_loss": 2.344987392425537, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11168399453163147, + "rewards/margins": 0.6573556065559387, + "rewards/rejected": -0.7690396308898926, + "step": 723 + }, + { + "epoch": 0.4503888024883359, + "grad_norm": 0.4526844322681427, + "learning_rate": 3.19e-05, + "log_odds_chosen": 7.047250270843506, + "log_odds_ratio": -0.1251072883605957, + "logits/chosen": 0.1379421055316925, + "logits/rejected": 0.5924164652824402, + "logps/chosen": -1.2250146865844727, + "logps/rejected": -7.931028842926025, + "loss": 2.6565, + "nll_loss": 2.643982172012329, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12250147014856339, + "rewards/margins": 0.6706014275550842, + "rewards/rejected": -0.7931028604507446, + "step": 724 + }, + { + "epoch": 0.4510108864696734, + "grad_norm": 0.5021094083786011, + "learning_rate": 3.1875e-05, + "log_odds_chosen": 9.610315322875977, + "log_odds_ratio": -0.24255721271038055, + "logits/chosen": 0.3092350363731384, + "logits/rejected": 0.9811382293701172, + "logps/chosen": -1.3039846420288086, + "logps/rejected": -10.703886032104492, + "loss": 2.5964, + "nll_loss": 2.572134256362915, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1303984671831131, + "rewards/margins": 0.9399901628494263, + "rewards/rejected": -1.0703885555267334, + "step": 725 + }, + { + "epoch": 0.45163297045101086, + "grad_norm": 6.453852653503418, + "learning_rate": 3.185e-05, + "log_odds_chosen": 9.646251678466797, + "log_odds_ratio": -0.11725477129220963, + "logits/chosen": 0.49717283248901367, + "logits/rejected": 0.8251327276229858, + "logps/chosen": -1.5012346506118774, + "logps/rejected": -10.704265594482422, + "loss": 3.4432, + "nll_loss": 3.431502103805542, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1501234769821167, + "rewards/margins": 0.9203031659126282, + "rewards/rejected": -1.0704265832901, + "step": 726 + }, + { + "epoch": 0.45225505443234837, + "grad_norm": 0.407627135515213, + "learning_rate": 3.1825e-05, + "log_odds_chosen": 6.178986549377441, + "log_odds_ratio": -0.25438594818115234, + "logits/chosen": 0.1292475461959839, + "logits/rejected": 0.43548843264579773, + "logps/chosen": -0.9622341990470886, + "logps/rejected": -6.761863708496094, + "loss": 2.5614, + "nll_loss": 2.5359420776367188, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09622342884540558, + "rewards/margins": 0.5799629092216492, + "rewards/rejected": -0.6761863231658936, + "step": 727 + }, + { + "epoch": 0.4528771384136858, + "grad_norm": 0.5671166777610779, + "learning_rate": 3.18e-05, + "log_odds_chosen": 9.908763885498047, + "log_odds_ratio": -0.15785574913024902, + "logits/chosen": 0.35942667722702026, + "logits/rejected": 0.8982104063034058, + "logps/chosen": -1.2879177331924438, + "logps/rejected": -10.947426795959473, + "loss": 3.2627, + "nll_loss": 3.2469372749328613, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12879177927970886, + "rewards/margins": 0.9659509062767029, + "rewards/rejected": -1.094742774963379, + "step": 728 + }, + { + "epoch": 0.4534992223950233, + "grad_norm": 0.4187334179878235, + "learning_rate": 3.1775e-05, + "log_odds_chosen": 6.857314586639404, + "log_odds_ratio": -0.12401160597801208, + "logits/chosen": 0.29477638006210327, + "logits/rejected": 0.567139744758606, + "logps/chosen": -0.97846519947052, + "logps/rejected": -7.214424133300781, + "loss": 3.2737, + "nll_loss": 3.2612991333007812, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09784651547670364, + "rewards/margins": 0.6235959529876709, + "rewards/rejected": -0.721442461013794, + "step": 729 + }, + { + "epoch": 0.45412130637636083, + "grad_norm": 0.43191951513290405, + "learning_rate": 3.175e-05, + "log_odds_chosen": 11.218259811401367, + "log_odds_ratio": -0.068722665309906, + "logits/chosen": 0.43568962812423706, + "logits/rejected": 1.0645880699157715, + "logps/chosen": -1.2674078941345215, + "logps/rejected": -12.176996231079102, + "loss": 3.3798, + "nll_loss": 3.372969388961792, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12674078345298767, + "rewards/margins": 1.090958833694458, + "rewards/rejected": -1.217699646949768, + "step": 730 + }, + { + "epoch": 0.4547433903576983, + "grad_norm": 1.9741532802581787, + "learning_rate": 3.1725e-05, + "log_odds_chosen": 7.206732273101807, + "log_odds_ratio": -0.13450881838798523, + "logits/chosen": 0.30000701546669006, + "logits/rejected": 0.480673611164093, + "logps/chosen": -1.4034446477890015, + "logps/rejected": -8.346330642700195, + "loss": 3.3633, + "nll_loss": 3.3498318195343018, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14034447073936462, + "rewards/margins": 0.6942886114120483, + "rewards/rejected": -0.8346331119537354, + "step": 731 + }, + { + "epoch": 0.4553654743390358, + "grad_norm": 0.3933872878551483, + "learning_rate": 3.1700000000000005e-05, + "log_odds_chosen": 9.562494277954102, + "log_odds_ratio": -0.07410041987895966, + "logits/chosen": 0.34438636898994446, + "logits/rejected": 0.6179068088531494, + "logps/chosen": -0.7376449704170227, + "logps/rejected": -9.597999572753906, + "loss": 3.4239, + "nll_loss": 3.4165120124816895, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07376450300216675, + "rewards/margins": 0.8860355615615845, + "rewards/rejected": -0.9598000049591064, + "step": 732 + }, + { + "epoch": 0.45598755832037324, + "grad_norm": 0.4037986695766449, + "learning_rate": 3.1675e-05, + "log_odds_chosen": 7.081986427307129, + "log_odds_ratio": -0.10163398832082748, + "logits/chosen": 0.18077480792999268, + "logits/rejected": 0.4592246115207672, + "logps/chosen": -0.9602686166763306, + "logps/rejected": -7.555391311645508, + "loss": 2.9457, + "nll_loss": 2.935537576675415, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09602686762809753, + "rewards/margins": 0.6595123410224915, + "rewards/rejected": -0.7555391788482666, + "step": 733 + }, + { + "epoch": 0.45660964230171075, + "grad_norm": 0.5620434284210205, + "learning_rate": 3.1650000000000004e-05, + "log_odds_chosen": 5.531484603881836, + "log_odds_ratio": -0.32697606086730957, + "logits/chosen": 0.1813567876815796, + "logits/rejected": 0.3070491850376129, + "logps/chosen": -1.2444071769714355, + "logps/rejected": -6.4295549392700195, + "loss": 3.2722, + "nll_loss": 3.239509105682373, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12444071471691132, + "rewards/margins": 0.5185148119926453, + "rewards/rejected": -0.642955482006073, + "step": 734 + }, + { + "epoch": 0.4572317262830482, + "grad_norm": 0.4302683174610138, + "learning_rate": 3.1624999999999996e-05, + "log_odds_chosen": 6.523910999298096, + "log_odds_ratio": -0.14284959435462952, + "logits/chosen": 0.04424915090203285, + "logits/rejected": 0.13490360975265503, + "logps/chosen": -0.9167971611022949, + "logps/rejected": -6.748834133148193, + "loss": 3.1965, + "nll_loss": 3.1821680068969727, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09167972207069397, + "rewards/margins": 0.5832037329673767, + "rewards/rejected": -0.6748834848403931, + "step": 735 + }, + { + "epoch": 0.4578538102643857, + "grad_norm": 0.6089425086975098, + "learning_rate": 3.16e-05, + "log_odds_chosen": 5.9218363761901855, + "log_odds_ratio": -0.2695227563381195, + "logits/chosen": 0.2000441998243332, + "logits/rejected": 0.4127127230167389, + "logps/chosen": -1.3729795217514038, + "logps/rejected": -7.12251091003418, + "loss": 3.1966, + "nll_loss": 3.1696414947509766, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13729795813560486, + "rewards/margins": 0.5749530792236328, + "rewards/rejected": -0.7122510671615601, + "step": 736 + }, + { + "epoch": 0.45847589424572316, + "grad_norm": 0.5367254018783569, + "learning_rate": 3.1575e-05, + "log_odds_chosen": 6.432736396789551, + "log_odds_ratio": -0.16896378993988037, + "logits/chosen": 0.06318940222263336, + "logits/rejected": 0.297978013753891, + "logps/chosen": -1.222300410270691, + "logps/rejected": -7.110202312469482, + "loss": 3.0173, + "nll_loss": 3.0003702640533447, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12223003804683685, + "rewards/margins": 0.5887902975082397, + "rewards/rejected": -0.7110202312469482, + "step": 737 + }, + { + "epoch": 0.45909797822706067, + "grad_norm": 4.201266288757324, + "learning_rate": 3.155e-05, + "log_odds_chosen": 5.493893623352051, + "log_odds_ratio": -0.45646530389785767, + "logits/chosen": 0.21096059679985046, + "logits/rejected": 0.23016388714313507, + "logps/chosen": -1.8003606796264648, + "logps/rejected": -7.092996597290039, + "loss": 3.7453, + "nll_loss": 3.6996266841888428, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.18003606796264648, + "rewards/margins": 0.5292636156082153, + "rewards/rejected": -0.7092996835708618, + "step": 738 + }, + { + "epoch": 0.4597200622083981, + "grad_norm": 0.41682955622673035, + "learning_rate": 3.1525e-05, + "log_odds_chosen": 3.3369433879852295, + "log_odds_ratio": -0.2148256152868271, + "logits/chosen": 0.12079880386590958, + "logits/rejected": 0.19746747612953186, + "logps/chosen": -1.0625683069229126, + "logps/rejected": -3.9425437450408936, + "loss": 3.5171, + "nll_loss": 3.4956541061401367, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10625684261322021, + "rewards/margins": 0.2879975736141205, + "rewards/rejected": -0.3942543864250183, + "step": 739 + }, + { + "epoch": 0.4603421461897356, + "grad_norm": 0.589731752872467, + "learning_rate": 3.15e-05, + "log_odds_chosen": 3.7177412509918213, + "log_odds_ratio": -0.39108750224113464, + "logits/chosen": -0.020437847822904587, + "logits/rejected": 0.1704711765050888, + "logps/chosen": -1.1689900159835815, + "logps/rejected": -4.5871758460998535, + "loss": 2.4873, + "nll_loss": 2.4481828212738037, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11689899861812592, + "rewards/margins": 0.34181857109069824, + "rewards/rejected": -0.45871755480766296, + "step": 740 + }, + { + "epoch": 0.4609642301710731, + "grad_norm": 0.6688488721847534, + "learning_rate": 3.1475e-05, + "log_odds_chosen": 4.912108421325684, + "log_odds_ratio": -0.13216131925582886, + "logits/chosen": 0.22601565718650818, + "logits/rejected": 0.3431988060474396, + "logps/chosen": -1.2438921928405762, + "logps/rejected": -5.821355819702148, + "loss": 3.6307, + "nll_loss": 3.617436408996582, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12438922375440598, + "rewards/margins": 0.45774632692337036, + "rewards/rejected": -0.5821355581283569, + "step": 741 + }, + { + "epoch": 0.4615863141524106, + "grad_norm": 0.433036744594574, + "learning_rate": 3.145e-05, + "log_odds_chosen": 3.096634864807129, + "log_odds_ratio": -0.2476002722978592, + "logits/chosen": 0.043339017778635025, + "logits/rejected": -0.03985293209552765, + "logps/chosen": -0.9964567422866821, + "logps/rejected": -3.488633155822754, + "loss": 3.2937, + "nll_loss": 3.26889705657959, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09964566677808762, + "rewards/margins": 0.2492176592350006, + "rewards/rejected": -0.34886330366134644, + "step": 742 + }, + { + "epoch": 0.46220839813374803, + "grad_norm": 0.6005847454071045, + "learning_rate": 3.1425e-05, + "log_odds_chosen": 6.896453857421875, + "log_odds_ratio": -0.1344524621963501, + "logits/chosen": 0.08107919245958328, + "logits/rejected": 0.26698002219200134, + "logps/chosen": -0.9853373169898987, + "logps/rejected": -7.432549953460693, + "loss": 2.531, + "nll_loss": 2.5175111293792725, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09853372722864151, + "rewards/margins": 0.644721269607544, + "rewards/rejected": -0.7432550191879272, + "step": 743 + }, + { + "epoch": 0.46283048211508554, + "grad_norm": 0.5875562429428101, + "learning_rate": 3.1400000000000004e-05, + "log_odds_chosen": 6.2522687911987305, + "log_odds_ratio": -0.14717459678649902, + "logits/chosen": 0.0802212506532669, + "logits/rejected": 0.36846601963043213, + "logps/chosen": -1.2279167175292969, + "logps/rejected": -7.11676025390625, + "loss": 2.9711, + "nll_loss": 2.9563703536987305, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12279167771339417, + "rewards/margins": 0.5888843536376953, + "rewards/rejected": -0.7116760015487671, + "step": 744 + }, + { + "epoch": 0.463452566096423, + "grad_norm": 0.6242512464523315, + "learning_rate": 3.1375e-05, + "log_odds_chosen": 8.841582298278809, + "log_odds_ratio": -0.2243223786354065, + "logits/chosen": 0.14888420701026917, + "logits/rejected": 0.574272096157074, + "logps/chosen": -1.1364891529083252, + "logps/rejected": -9.670944213867188, + "loss": 2.973, + "nll_loss": 2.9505813121795654, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1136489063501358, + "rewards/margins": 0.8534455299377441, + "rewards/rejected": -0.9670944213867188, + "step": 745 + }, + { + "epoch": 0.4640746500777605, + "grad_norm": 0.39740875363349915, + "learning_rate": 3.135e-05, + "log_odds_chosen": 6.4931817054748535, + "log_odds_ratio": -0.07528342306613922, + "logits/chosen": 0.23848633468151093, + "logits/rejected": 0.4403890371322632, + "logps/chosen": -1.1312795877456665, + "logps/rejected": -7.221208572387695, + "loss": 3.546, + "nll_loss": 3.5384349822998047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11312796920537949, + "rewards/margins": 0.608992874622345, + "rewards/rejected": -0.7221208214759827, + "step": 746 + }, + { + "epoch": 0.464696734059098, + "grad_norm": 0.5737400650978088, + "learning_rate": 3.1324999999999996e-05, + "log_odds_chosen": 7.452863693237305, + "log_odds_ratio": -0.15671104192733765, + "logits/chosen": 0.03789973258972168, + "logits/rejected": 0.1605812907218933, + "logps/chosen": -0.8898376226425171, + "logps/rejected": -7.745999336242676, + "loss": 3.0129, + "nll_loss": 2.997223377227783, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08898375928401947, + "rewards/margins": 0.6856162548065186, + "rewards/rejected": -0.7745999097824097, + "step": 747 + }, + { + "epoch": 0.46531881804043546, + "grad_norm": 0.4806496500968933, + "learning_rate": 3.13e-05, + "log_odds_chosen": 5.818041801452637, + "log_odds_ratio": -0.1867186278104782, + "logits/chosen": 0.1712789386510849, + "logits/rejected": 0.29497987031936646, + "logps/chosen": -0.8955323696136475, + "logps/rejected": -6.180495262145996, + "loss": 3.3477, + "nll_loss": 3.3290135860443115, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08955323696136475, + "rewards/margins": 0.5284963846206665, + "rewards/rejected": -0.6180496215820312, + "step": 748 + }, + { + "epoch": 0.46594090202177296, + "grad_norm": 0.6791718006134033, + "learning_rate": 3.1275e-05, + "log_odds_chosen": 5.693668842315674, + "log_odds_ratio": -0.15988114476203918, + "logits/chosen": 0.11964002251625061, + "logits/rejected": 0.40675055980682373, + "logps/chosen": -0.9922550916671753, + "logps/rejected": -6.27172327041626, + "loss": 2.7005, + "nll_loss": 2.68455171585083, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09922550618648529, + "rewards/margins": 0.5279468297958374, + "rewards/rejected": -0.6271723508834839, + "step": 749 + }, + { + "epoch": 0.4665629860031104, + "grad_norm": 0.6238471865653992, + "learning_rate": 3.125e-05, + "log_odds_chosen": 3.5400168895721436, + "log_odds_ratio": -0.42935463786125183, + "logits/chosen": 0.0866260677576065, + "logits/rejected": 0.37089765071868896, + "logps/chosen": -1.1119719743728638, + "logps/rejected": -4.440646171569824, + "loss": 2.7092, + "nll_loss": 2.666236400604248, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11119719594717026, + "rewards/margins": 0.33286744356155396, + "rewards/rejected": -0.4440646469593048, + "step": 750 + }, + { + "epoch": 0.4671850699844479, + "grad_norm": 0.5068716406822205, + "learning_rate": 3.122500000000001e-05, + "log_odds_chosen": 10.899909019470215, + "log_odds_ratio": -0.00871206633746624, + "logits/chosen": -0.01668960601091385, + "logits/rejected": 0.5333446264266968, + "logps/chosen": -1.2172091007232666, + "logps/rejected": -11.635497093200684, + "loss": 2.5222, + "nll_loss": 2.521331310272217, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12172091007232666, + "rewards/margins": 1.0418287515640259, + "rewards/rejected": -1.1635496616363525, + "step": 751 + }, + { + "epoch": 0.46780715396578537, + "grad_norm": 1.516399621963501, + "learning_rate": 3.12e-05, + "log_odds_chosen": 7.541528701782227, + "log_odds_ratio": -0.1825413852930069, + "logits/chosen": 0.006808616686612368, + "logits/rejected": 0.20838546752929688, + "logps/chosen": -1.2133069038391113, + "logps/rejected": -8.299160957336426, + "loss": 2.7663, + "nll_loss": 2.7480356693267822, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12133070081472397, + "rewards/margins": 0.7085853815078735, + "rewards/rejected": -0.8299161195755005, + "step": 752 + }, + { + "epoch": 0.4684292379471229, + "grad_norm": 0.43352144956588745, + "learning_rate": 3.1175000000000006e-05, + "log_odds_chosen": 7.610816955566406, + "log_odds_ratio": -0.33424389362335205, + "logits/chosen": 0.17765414714813232, + "logits/rejected": 0.3236091732978821, + "logps/chosen": -1.1976127624511719, + "logps/rejected": -8.585079193115234, + "loss": 3.4346, + "nll_loss": 3.4012205600738525, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11976128816604614, + "rewards/margins": 0.738746702671051, + "rewards/rejected": -0.8585079908370972, + "step": 753 + }, + { + "epoch": 0.46905132192846033, + "grad_norm": 0.4248764216899872, + "learning_rate": 3.115e-05, + "log_odds_chosen": 8.949850082397461, + "log_odds_ratio": -0.18056482076644897, + "logits/chosen": 0.1752690225839615, + "logits/rejected": 0.3496873676776886, + "logps/chosen": -0.8166213035583496, + "logps/rejected": -9.080907821655273, + "loss": 3.2551, + "nll_loss": 3.23703932762146, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0816621258854866, + "rewards/margins": 0.8264287710189819, + "rewards/rejected": -0.9080908894538879, + "step": 754 + }, + { + "epoch": 0.46967340590979784, + "grad_norm": 0.5243033170700073, + "learning_rate": 3.1125000000000004e-05, + "log_odds_chosen": 12.745326042175293, + "log_odds_ratio": -0.11361424624919891, + "logits/chosen": 0.1282683163881302, + "logits/rejected": 0.7375682592391968, + "logps/chosen": -1.205803394317627, + "logps/rejected": -13.61036205291748, + "loss": 2.9496, + "nll_loss": 2.938230276107788, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12058034539222717, + "rewards/margins": 1.2404558658599854, + "rewards/rejected": -1.3610363006591797, + "step": 755 + }, + { + "epoch": 0.4702954898911353, + "grad_norm": 0.5427597761154175, + "learning_rate": 3.1100000000000004e-05, + "log_odds_chosen": 10.578163146972656, + "log_odds_ratio": -0.34553229808807373, + "logits/chosen": 0.16474664211273193, + "logits/rejected": 0.526168704032898, + "logps/chosen": -1.3434865474700928, + "logps/rejected": -11.748912811279297, + "loss": 3.4079, + "nll_loss": 3.3733925819396973, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13434866070747375, + "rewards/margins": 1.040542721748352, + "rewards/rejected": -1.1748913526535034, + "step": 756 + }, + { + "epoch": 0.4709175738724728, + "grad_norm": 0.47395461797714233, + "learning_rate": 3.1075e-05, + "log_odds_chosen": 4.500313758850098, + "log_odds_ratio": -0.33959078788757324, + "logits/chosen": 0.07310894876718521, + "logits/rejected": 0.23603522777557373, + "logps/chosen": -1.0726346969604492, + "logps/rejected": -5.353082656860352, + "loss": 2.9118, + "nll_loss": 2.8778131008148193, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1072634682059288, + "rewards/margins": 0.428044855594635, + "rewards/rejected": -0.535308301448822, + "step": 757 + }, + { + "epoch": 0.47153965785381025, + "grad_norm": 0.5754133462905884, + "learning_rate": 3.105e-05, + "log_odds_chosen": 9.807731628417969, + "log_odds_ratio": -0.16157642006874084, + "logits/chosen": 0.05956118553876877, + "logits/rejected": 0.5986432433128357, + "logps/chosen": -1.3279142379760742, + "logps/rejected": -10.884170532226562, + "loss": 3.1166, + "nll_loss": 3.100430488586426, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1327914297580719, + "rewards/margins": 0.9556256532669067, + "rewards/rejected": -1.0884170532226562, + "step": 758 + }, + { + "epoch": 0.47216174183514775, + "grad_norm": 0.48073136806488037, + "learning_rate": 3.1025e-05, + "log_odds_chosen": 7.494086265563965, + "log_odds_ratio": -0.2150120735168457, + "logits/chosen": 0.2095496952533722, + "logits/rejected": 0.4400617778301239, + "logps/chosen": -1.2367106676101685, + "logps/rejected": -8.398553848266602, + "loss": 3.5941, + "nll_loss": 3.5726468563079834, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12367106974124908, + "rewards/margins": 0.7161843776702881, + "rewards/rejected": -0.839855432510376, + "step": 759 + }, + { + "epoch": 0.4727838258164852, + "grad_norm": 0.5509316325187683, + "learning_rate": 3.1e-05, + "log_odds_chosen": 7.781867504119873, + "log_odds_ratio": -0.1630517691373825, + "logits/chosen": 0.12625464797019958, + "logits/rejected": 0.6091369390487671, + "logps/chosen": -1.2734575271606445, + "logps/rejected": -8.757940292358398, + "loss": 2.8018, + "nll_loss": 2.7855429649353027, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1273457556962967, + "rewards/margins": 0.748448371887207, + "rewards/rejected": -0.8757941126823425, + "step": 760 + }, + { + "epoch": 0.4734059097978227, + "grad_norm": 0.5899271368980408, + "learning_rate": 3.0975e-05, + "log_odds_chosen": 2.6958911418914795, + "log_odds_ratio": -0.5016304850578308, + "logits/chosen": 0.13957029581069946, + "logits/rejected": 0.23938274383544922, + "logps/chosen": -1.2764240503311157, + "logps/rejected": -3.8734121322631836, + "loss": 3.2298, + "nll_loss": 3.1796460151672363, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1276424080133438, + "rewards/margins": 0.2596988081932068, + "rewards/rejected": -0.3873412013053894, + "step": 761 + }, + { + "epoch": 0.47402799377916016, + "grad_norm": 0.5521773099899292, + "learning_rate": 3.095e-05, + "log_odds_chosen": 3.7203664779663086, + "log_odds_ratio": -0.439362496137619, + "logits/chosen": -0.036664508283138275, + "logits/rejected": 0.10708197951316833, + "logps/chosen": -0.9352907538414001, + "logps/rejected": -4.322984218597412, + "loss": 2.5678, + "nll_loss": 2.52388596534729, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09352907538414001, + "rewards/margins": 0.3387693762779236, + "rewards/rejected": -0.4322984218597412, + "step": 762 + }, + { + "epoch": 0.47465007776049767, + "grad_norm": 0.5061217546463013, + "learning_rate": 3.0925000000000006e-05, + "log_odds_chosen": 12.82931137084961, + "log_odds_ratio": -0.07551414519548416, + "logits/chosen": 0.04354046285152435, + "logits/rejected": 0.9819294810295105, + "logps/chosen": -1.4580919742584229, + "logps/rejected": -13.935245513916016, + "loss": 2.7455, + "nll_loss": 2.7379651069641113, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14580918848514557, + "rewards/margins": 1.2477153539657593, + "rewards/rejected": -1.393524408340454, + "step": 763 + }, + { + "epoch": 0.4752721617418352, + "grad_norm": 0.6883053183555603, + "learning_rate": 3.09e-05, + "log_odds_chosen": 7.964182376861572, + "log_odds_ratio": -0.46369829773902893, + "logits/chosen": 0.12046404182910919, + "logits/rejected": 0.6274876594543457, + "logps/chosen": -1.3564209938049316, + "logps/rejected": -9.131847381591797, + "loss": 2.7681, + "nll_loss": 2.7217345237731934, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13564209640026093, + "rewards/margins": 0.7775427103042603, + "rewards/rejected": -0.9131847620010376, + "step": 764 + }, + { + "epoch": 0.4758942457231726, + "grad_norm": 0.44783955812454224, + "learning_rate": 3.0875000000000005e-05, + "log_odds_chosen": 9.665321350097656, + "log_odds_ratio": -0.1914052665233612, + "logits/chosen": 0.05238525569438934, + "logits/rejected": 0.6160183548927307, + "logps/chosen": -1.1813290119171143, + "logps/rejected": -10.583022117614746, + "loss": 2.9187, + "nll_loss": 2.8995184898376465, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11813290417194366, + "rewards/margins": 0.9401693940162659, + "rewards/rejected": -1.0583022832870483, + "step": 765 + }, + { + "epoch": 0.47651632970451013, + "grad_norm": 0.48133954405784607, + "learning_rate": 3.0850000000000004e-05, + "log_odds_chosen": 10.398935317993164, + "log_odds_ratio": -0.2939707934856415, + "logits/chosen": 0.196578711271286, + "logits/rejected": 0.7910341024398804, + "logps/chosen": -1.453446626663208, + "logps/rejected": -11.644519805908203, + "loss": 3.0305, + "nll_loss": 3.0011115074157715, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14534467458724976, + "rewards/margins": 1.0191072225570679, + "rewards/rejected": -1.1644519567489624, + "step": 766 + }, + { + "epoch": 0.4771384136858476, + "grad_norm": 0.7429364323616028, + "learning_rate": 3.0825000000000004e-05, + "log_odds_chosen": 8.819571495056152, + "log_odds_ratio": -0.17902621626853943, + "logits/chosen": 0.0832553580403328, + "logits/rejected": 0.9124725461006165, + "logps/chosen": -1.139229416847229, + "logps/rejected": -9.641944885253906, + "loss": 2.4093, + "nll_loss": 2.391360282897949, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11392293870449066, + "rewards/margins": 0.8502715826034546, + "rewards/rejected": -0.9641945958137512, + "step": 767 + }, + { + "epoch": 0.4777604976671851, + "grad_norm": 0.45216140151023865, + "learning_rate": 3.08e-05, + "log_odds_chosen": 10.36209774017334, + "log_odds_ratio": -0.033273592591285706, + "logits/chosen": 0.0013059796765446663, + "logits/rejected": 0.4774538576602936, + "logps/chosen": -0.9916945695877075, + "logps/rejected": -10.634331703186035, + "loss": 2.6309, + "nll_loss": 2.627547264099121, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09916945546865463, + "rewards/margins": 0.9642637372016907, + "rewards/rejected": -1.063433289527893, + "step": 768 + }, + { + "epoch": 0.47838258164852254, + "grad_norm": 0.6611552834510803, + "learning_rate": 3.0775e-05, + "log_odds_chosen": 13.858925819396973, + "log_odds_ratio": -0.059990376234054565, + "logits/chosen": 0.07334032654762268, + "logits/rejected": 0.49123767018318176, + "logps/chosen": -0.9776344299316406, + "logps/rejected": -14.378206253051758, + "loss": 3.101, + "nll_loss": 3.0949764251708984, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09776344150304794, + "rewards/margins": 1.340057134628296, + "rewards/rejected": -1.437820553779602, + "step": 769 + }, + { + "epoch": 0.47900466562986005, + "grad_norm": 2.7856853008270264, + "learning_rate": 3.075e-05, + "log_odds_chosen": 11.939106941223145, + "log_odds_ratio": -0.11589374393224716, + "logits/chosen": 0.24869629740715027, + "logits/rejected": 0.7966226935386658, + "logps/chosen": -1.8126963376998901, + "logps/rejected": -13.446033477783203, + "loss": 3.6146, + "nll_loss": 3.603039264678955, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18126963078975677, + "rewards/margins": 1.163333773612976, + "rewards/rejected": -1.3446035385131836, + "step": 770 + }, + { + "epoch": 0.4796267496111975, + "grad_norm": 0.4278852641582489, + "learning_rate": 3.0725e-05, + "log_odds_chosen": 16.7786865234375, + "log_odds_ratio": -0.15743397176265717, + "logits/chosen": 0.27345868945121765, + "logits/rejected": 0.9987517595291138, + "logps/chosen": -0.9886462092399597, + "logps/rejected": -17.29168128967285, + "loss": 3.3687, + "nll_loss": 3.3529460430145264, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09886462986469269, + "rewards/margins": 1.6303035020828247, + "rewards/rejected": -1.729168176651001, + "step": 771 + }, + { + "epoch": 0.480248833592535, + "grad_norm": 0.4929329454898834, + "learning_rate": 3.07e-05, + "log_odds_chosen": 13.330595970153809, + "log_odds_ratio": -0.08029691874980927, + "logits/chosen": 0.1175004094839096, + "logits/rejected": 0.8302413821220398, + "logps/chosen": -1.2270184755325317, + "logps/rejected": -14.165221214294434, + "loss": 2.9649, + "nll_loss": 2.956918478012085, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12270184606313705, + "rewards/margins": 1.2938203811645508, + "rewards/rejected": -1.4165221452713013, + "step": 772 + }, + { + "epoch": 0.48087091757387246, + "grad_norm": 0.4613707661628723, + "learning_rate": 3.067500000000001e-05, + "log_odds_chosen": 17.801786422729492, + "log_odds_ratio": -0.0008926771115511656, + "logits/chosen": 0.22919896245002747, + "logits/rejected": 1.450141429901123, + "logps/chosen": -1.3054852485656738, + "logps/rejected": -18.71816635131836, + "loss": 2.5083, + "nll_loss": 2.5082015991210938, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13054853677749634, + "rewards/margins": 1.7412681579589844, + "rewards/rejected": -1.8718167543411255, + "step": 773 + }, + { + "epoch": 0.48149300155520997, + "grad_norm": 0.4349910020828247, + "learning_rate": 3.065e-05, + "log_odds_chosen": 16.952882766723633, + "log_odds_ratio": -0.07804927229881287, + "logits/chosen": -0.0008221510797739029, + "logits/rejected": 1.0307400226593018, + "logps/chosen": -1.0544698238372803, + "logps/rejected": -17.315820693969727, + "loss": 2.7886, + "nll_loss": 2.7808361053466797, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10544697940349579, + "rewards/margins": 1.6261351108551025, + "rewards/rejected": -1.7315820455551147, + "step": 774 + }, + { + "epoch": 0.4821150855365474, + "grad_norm": 0.45079338550567627, + "learning_rate": 3.0625000000000006e-05, + "log_odds_chosen": 6.942625045776367, + "log_odds_ratio": -0.2755284011363983, + "logits/chosen": 0.1854054182767868, + "logits/rejected": 0.424145370721817, + "logps/chosen": -1.04248046875, + "logps/rejected": -7.667983055114746, + "loss": 3.4672, + "nll_loss": 3.4396426677703857, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1042480394244194, + "rewards/margins": 0.6625502705574036, + "rewards/rejected": -0.7667983174324036, + "step": 775 + }, + { + "epoch": 0.4827371695178849, + "grad_norm": 1.2325921058654785, + "learning_rate": 3.06e-05, + "log_odds_chosen": 17.07767105102539, + "log_odds_ratio": -0.07737934589385986, + "logits/chosen": 0.209869846701622, + "logits/rejected": 1.219171404838562, + "logps/chosen": -1.1802769899368286, + "logps/rejected": -17.847057342529297, + "loss": 2.9043, + "nll_loss": 2.89656138420105, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1180276945233345, + "rewards/margins": 1.6666781902313232, + "rewards/rejected": -1.7847058773040771, + "step": 776 + }, + { + "epoch": 0.4833592534992224, + "grad_norm": 0.4120030999183655, + "learning_rate": 3.0575000000000005e-05, + "log_odds_chosen": 13.921056747436523, + "log_odds_ratio": -0.22028662264347076, + "logits/chosen": 0.2784203588962555, + "logits/rejected": 1.1734328269958496, + "logps/chosen": -1.1785310506820679, + "logps/rejected": -14.79477596282959, + "loss": 3.0951, + "nll_loss": 3.0730538368225098, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11785309761762619, + "rewards/margins": 1.3616244792938232, + "rewards/rejected": -1.4794776439666748, + "step": 777 + }, + { + "epoch": 0.4839813374805599, + "grad_norm": 0.5162467956542969, + "learning_rate": 3.0550000000000004e-05, + "log_odds_chosen": 12.977121353149414, + "log_odds_ratio": -0.20910266041755676, + "logits/chosen": 0.2095002979040146, + "logits/rejected": 0.5365079641342163, + "logps/chosen": -1.314969539642334, + "logps/rejected": -14.050061225891113, + "loss": 3.1444, + "nll_loss": 3.123497486114502, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13149695098400116, + "rewards/margins": 1.2735092639923096, + "rewards/rejected": -1.4050061702728271, + "step": 778 + }, + { + "epoch": 0.48460342146189733, + "grad_norm": 0.4763834774494171, + "learning_rate": 3.0525e-05, + "log_odds_chosen": 20.971172332763672, + "log_odds_ratio": -2.3841880647523794e-07, + "logits/chosen": 0.3068065345287323, + "logits/rejected": 1.4703824520111084, + "logps/chosen": -1.2722759246826172, + "logps/rejected": -21.902528762817383, + "loss": 3.1538, + "nll_loss": 3.153752088546753, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12722758948802948, + "rewards/margins": 2.0630252361297607, + "rewards/rejected": -2.1902527809143066, + "step": 779 + }, + { + "epoch": 0.48522550544323484, + "grad_norm": 0.45172974467277527, + "learning_rate": 3.05e-05, + "log_odds_chosen": 11.323856353759766, + "log_odds_ratio": -0.1366121768951416, + "logits/chosen": 0.18574324250221252, + "logits/rejected": 0.7099617719650269, + "logps/chosen": -0.6922950744628906, + "logps/rejected": -11.323362350463867, + "loss": 3.2448, + "nll_loss": 3.2311220169067383, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.06922950595617294, + "rewards/margins": 1.0631067752838135, + "rewards/rejected": -1.1323362588882446, + "step": 780 + }, + { + "epoch": 0.4858475894245723, + "grad_norm": 0.32547134160995483, + "learning_rate": 3.0475000000000002e-05, + "log_odds_chosen": 13.120780944824219, + "log_odds_ratio": -0.2054150402545929, + "logits/chosen": 0.3427540361881256, + "logits/rejected": 0.6512297987937927, + "logps/chosen": -1.148818016052246, + "logps/rejected": -13.904681205749512, + "loss": 3.6809, + "nll_loss": 3.6603195667266846, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11488181352615356, + "rewards/margins": 1.2755863666534424, + "rewards/rejected": -1.3904681205749512, + "step": 781 + }, + { + "epoch": 0.4864696734059098, + "grad_norm": 0.4283800721168518, + "learning_rate": 3.045e-05, + "log_odds_chosen": 15.075250625610352, + "log_odds_ratio": -0.05576830729842186, + "logits/chosen": 0.07458168268203735, + "logits/rejected": 0.8056328296661377, + "logps/chosen": -1.0403079986572266, + "logps/rejected": -15.61276912689209, + "loss": 2.6876, + "nll_loss": 2.682063341140747, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1040308028459549, + "rewards/margins": 1.4572460651397705, + "rewards/rejected": -1.561276912689209, + "step": 782 + }, + { + "epoch": 0.4870917573872473, + "grad_norm": 7.5606160163879395, + "learning_rate": 3.0425000000000004e-05, + "log_odds_chosen": 15.987709999084473, + "log_odds_ratio": -0.09875954687595367, + "logits/chosen": 0.29279085993766785, + "logits/rejected": 1.0353281497955322, + "logps/chosen": -1.5241639614105225, + "logps/rejected": -17.19584846496582, + "loss": 3.2454, + "nll_loss": 3.2354772090911865, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15241639316082, + "rewards/margins": 1.5671684741973877, + "rewards/rejected": -1.7195848226547241, + "step": 783 + }, + { + "epoch": 0.48771384136858476, + "grad_norm": 0.6330939531326294, + "learning_rate": 3.04e-05, + "log_odds_chosen": 7.304858684539795, + "log_odds_ratio": -0.5831460952758789, + "logits/chosen": 0.21207121014595032, + "logits/rejected": 0.46470144391059875, + "logps/chosen": -1.3569834232330322, + "logps/rejected": -8.61102294921875, + "loss": 3.3432, + "nll_loss": 3.284919261932373, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1356983482837677, + "rewards/margins": 0.725403904914856, + "rewards/rejected": -0.8611023426055908, + "step": 784 + }, + { + "epoch": 0.48833592534992226, + "grad_norm": 0.43546679615974426, + "learning_rate": 3.0375000000000003e-05, + "log_odds_chosen": 7.85859489440918, + "log_odds_ratio": -0.39412838220596313, + "logits/chosen": 0.12173283845186234, + "logits/rejected": 0.5120882391929626, + "logps/chosen": -1.2129175662994385, + "logps/rejected": -8.616832733154297, + "loss": 3.2616, + "nll_loss": 3.2221570014953613, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12129175662994385, + "rewards/margins": 0.7403914928436279, + "rewards/rejected": -0.861683189868927, + "step": 785 + }, + { + "epoch": 0.4889580093312597, + "grad_norm": 0.5060476064682007, + "learning_rate": 3.035e-05, + "log_odds_chosen": 8.088102340698242, + "log_odds_ratio": -0.1689227670431137, + "logits/chosen": 0.18627770245075226, + "logits/rejected": 0.8996896743774414, + "logps/chosen": -1.2733261585235596, + "logps/rejected": -9.087573051452637, + "loss": 3.05, + "nll_loss": 3.0331501960754395, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12733261287212372, + "rewards/margins": 0.7814247012138367, + "rewards/rejected": -0.908757209777832, + "step": 786 + }, + { + "epoch": 0.4895800933125972, + "grad_norm": 0.5102691054344177, + "learning_rate": 3.0325000000000002e-05, + "log_odds_chosen": 9.025958061218262, + "log_odds_ratio": -0.18867120146751404, + "logits/chosen": 0.2037818729877472, + "logits/rejected": 0.6893376708030701, + "logps/chosen": -1.0546302795410156, + "logps/rejected": -9.737627983093262, + "loss": 2.8651, + "nll_loss": 2.84627366065979, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10546302795410156, + "rewards/margins": 0.8682998418807983, + "rewards/rejected": -0.9737628698348999, + "step": 787 + }, + { + "epoch": 0.49020217729393467, + "grad_norm": 0.43984538316726685, + "learning_rate": 3.03e-05, + "log_odds_chosen": 10.870232582092285, + "log_odds_ratio": -0.10683636367321014, + "logits/chosen": 0.13067655265331268, + "logits/rejected": 0.9502900838851929, + "logps/chosen": -0.9612005352973938, + "logps/rejected": -11.30851936340332, + "loss": 2.3156, + "nll_loss": 2.304933547973633, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09612004458904266, + "rewards/margins": 1.0347318649291992, + "rewards/rejected": -1.1308519840240479, + "step": 788 + }, + { + "epoch": 0.4908242612752722, + "grad_norm": 0.41433024406433105, + "learning_rate": 3.0275000000000004e-05, + "log_odds_chosen": 7.59851598739624, + "log_odds_ratio": -0.12431486696004868, + "logits/chosen": 0.08263087272644043, + "logits/rejected": 0.3450954556465149, + "logps/chosen": -1.0177639722824097, + "logps/rejected": -8.221351623535156, + "loss": 3.0721, + "nll_loss": 3.0597083568573, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10177639871835709, + "rewards/margins": 0.7203587293624878, + "rewards/rejected": -0.8221350908279419, + "step": 789 + }, + { + "epoch": 0.49144634525660963, + "grad_norm": 0.480323851108551, + "learning_rate": 3.025e-05, + "log_odds_chosen": 3.6505227088928223, + "log_odds_ratio": -0.4678669273853302, + "logits/chosen": 0.12035234272480011, + "logits/rejected": 0.24431809782981873, + "logps/chosen": -1.2138129472732544, + "logps/rejected": -4.56633996963501, + "loss": 2.912, + "nll_loss": 2.865224599838257, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12138129770755768, + "rewards/margins": 0.33525270223617554, + "rewards/rejected": -0.456633985042572, + "step": 790 + }, + { + "epoch": 0.49206842923794714, + "grad_norm": 0.5126906037330627, + "learning_rate": 3.0225000000000003e-05, + "log_odds_chosen": 14.473326683044434, + "log_odds_ratio": -7.871995330788195e-05, + "logits/chosen": 0.212377667427063, + "logits/rejected": 1.0169252157211304, + "logps/chosen": -1.15001380443573, + "logps/rejected": -15.20506763458252, + "loss": 3.0556, + "nll_loss": 3.0555496215820312, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.115001380443573, + "rewards/margins": 1.4055054187774658, + "rewards/rejected": -1.520506739616394, + "step": 791 + }, + { + "epoch": 0.4926905132192846, + "grad_norm": 0.5260676145553589, + "learning_rate": 3.02e-05, + "log_odds_chosen": 9.164556503295898, + "log_odds_ratio": -0.20497088134288788, + "logits/chosen": 0.14021986722946167, + "logits/rejected": 0.6322394609451294, + "logps/chosen": -1.1137261390686035, + "logps/rejected": -9.867830276489258, + "loss": 3.3855, + "nll_loss": 3.3650379180908203, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11137261986732483, + "rewards/margins": 0.8754103183746338, + "rewards/rejected": -0.986782968044281, + "step": 792 + }, + { + "epoch": 0.4933125972006221, + "grad_norm": 1.3812450170516968, + "learning_rate": 3.0175e-05, + "log_odds_chosen": 7.82905912399292, + "log_odds_ratio": -0.5109982490539551, + "logits/chosen": 0.2852308452129364, + "logits/rejected": 0.6695021390914917, + "logps/chosen": -1.8619670867919922, + "logps/rejected": -9.53437614440918, + "loss": 3.489, + "nll_loss": 3.4378857612609863, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1861967146396637, + "rewards/margins": 0.7672408819198608, + "rewards/rejected": -0.9534375667572021, + "step": 793 + }, + { + "epoch": 0.49393468118195955, + "grad_norm": 0.47631436586380005, + "learning_rate": 3.015e-05, + "log_odds_chosen": 10.75469970703125, + "log_odds_ratio": -0.24386245012283325, + "logits/chosen": 0.14563864469528198, + "logits/rejected": 0.9450300335884094, + "logps/chosen": -1.1033132076263428, + "logps/rejected": -11.575775146484375, + "loss": 2.6249, + "nll_loss": 2.6005167961120605, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11033132672309875, + "rewards/margins": 1.0472462177276611, + "rewards/rejected": -1.1575775146484375, + "step": 794 + }, + { + "epoch": 0.49455676516329705, + "grad_norm": 0.4538606107234955, + "learning_rate": 3.0125000000000004e-05, + "log_odds_chosen": 8.877276420593262, + "log_odds_ratio": -0.19892564415931702, + "logits/chosen": 0.2505427896976471, + "logits/rejected": 0.9329760670661926, + "logps/chosen": -1.0743852853775024, + "logps/rejected": -9.534322738647461, + "loss": 2.7392, + "nll_loss": 2.7193334102630615, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10743853449821472, + "rewards/margins": 0.8459938764572144, + "rewards/rejected": -0.9534323215484619, + "step": 795 + }, + { + "epoch": 0.4951788491446345, + "grad_norm": 0.5034440755844116, + "learning_rate": 3.01e-05, + "log_odds_chosen": 6.5391740798950195, + "log_odds_ratio": -0.3063565492630005, + "logits/chosen": 0.1824992597103119, + "logits/rejected": 0.32490694522857666, + "logps/chosen": -1.5990245342254639, + "logps/rejected": -7.915644645690918, + "loss": 3.681, + "nll_loss": 3.6503705978393555, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1599024385213852, + "rewards/margins": 0.6316620111465454, + "rewards/rejected": -0.7915644645690918, + "step": 796 + }, + { + "epoch": 0.495800933125972, + "grad_norm": 0.4987301528453827, + "learning_rate": 3.0075000000000003e-05, + "log_odds_chosen": 13.378631591796875, + "log_odds_ratio": -0.15653789043426514, + "logits/chosen": 0.19882658123970032, + "logits/rejected": 1.0811190605163574, + "logps/chosen": -0.9882148504257202, + "logps/rejected": -13.973001480102539, + "loss": 2.8765, + "nll_loss": 2.860856056213379, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0988214910030365, + "rewards/margins": 1.2984788417816162, + "rewards/rejected": -1.3973002433776855, + "step": 797 + }, + { + "epoch": 0.49642301710730946, + "grad_norm": 0.35390564799308777, + "learning_rate": 3.0050000000000002e-05, + "log_odds_chosen": 11.163237571716309, + "log_odds_ratio": -0.11993933469057083, + "logits/chosen": 0.24885781109333038, + "logits/rejected": 0.7615518569946289, + "logps/chosen": -1.0514477491378784, + "logps/rejected": -11.754963874816895, + "loss": 3.3127, + "nll_loss": 3.3006787300109863, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10514476895332336, + "rewards/margins": 1.0703516006469727, + "rewards/rejected": -1.1754963397979736, + "step": 798 + }, + { + "epoch": 0.49704510108864697, + "grad_norm": 0.46855729818344116, + "learning_rate": 3.0025000000000005e-05, + "log_odds_chosen": 14.509871482849121, + "log_odds_ratio": -0.06569898873567581, + "logits/chosen": 0.13805624842643738, + "logits/rejected": 0.8150652050971985, + "logps/chosen": -1.496653437614441, + "logps/rejected": -15.731345176696777, + "loss": 3.0396, + "nll_loss": 3.0330357551574707, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14966535568237305, + "rewards/margins": 1.4234691858291626, + "rewards/rejected": -1.573134422302246, + "step": 799 + }, + { + "epoch": 0.4976671850699845, + "grad_norm": 0.418714702129364, + "learning_rate": 3e-05, + "log_odds_chosen": 6.016261577606201, + "log_odds_ratio": -0.35451096296310425, + "logits/chosen": 0.012198593467473984, + "logits/rejected": 0.27333053946495056, + "logps/chosen": -1.07492995262146, + "logps/rejected": -6.799544334411621, + "loss": 2.815, + "nll_loss": 2.7795400619506836, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10749299824237823, + "rewards/margins": 0.5724614262580872, + "rewards/rejected": -0.6799544095993042, + "step": 800 + }, + { + "epoch": 0.4982892690513219, + "grad_norm": 0.45310264825820923, + "learning_rate": 2.9975000000000004e-05, + "log_odds_chosen": 12.761698722839355, + "log_odds_ratio": -0.23440547287464142, + "logits/chosen": 0.07036770135164261, + "logits/rejected": 0.8573406338691711, + "logps/chosen": -1.2407668828964233, + "logps/rejected": -13.764123916625977, + "loss": 2.4648, + "nll_loss": 2.4413938522338867, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.124076709151268, + "rewards/margins": 1.252335786819458, + "rewards/rejected": -1.3764123916625977, + "step": 801 + }, + { + "epoch": 0.49891135303265943, + "grad_norm": 0.47506943345069885, + "learning_rate": 2.995e-05, + "log_odds_chosen": 9.560124397277832, + "log_odds_ratio": -0.45071908831596375, + "logits/chosen": 0.16752222180366516, + "logits/rejected": 0.6287195086479187, + "logps/chosen": -1.2660199403762817, + "logps/rejected": -10.690040588378906, + "loss": 2.8457, + "nll_loss": 2.8006627559661865, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12660199403762817, + "rewards/margins": 0.942402184009552, + "rewards/rejected": -1.0690041780471802, + "step": 802 + }, + { + "epoch": 0.4995334370139969, + "grad_norm": 0.47123369574546814, + "learning_rate": 2.9925000000000002e-05, + "log_odds_chosen": 7.347402572631836, + "log_odds_ratio": -0.20708920061588287, + "logits/chosen": 0.1796862781047821, + "logits/rejected": 0.46771129965782166, + "logps/chosen": -0.9793645739555359, + "logps/rejected": -7.834238052368164, + "loss": 3.2493, + "nll_loss": 3.2286105155944824, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09793645888566971, + "rewards/margins": 0.6854873895645142, + "rewards/rejected": -0.7834238409996033, + "step": 803 + }, + { + "epoch": 0.5001555209953343, + "grad_norm": 0.4224500358104706, + "learning_rate": 2.9900000000000002e-05, + "log_odds_chosen": 9.55636215209961, + "log_odds_ratio": -0.20841999351978302, + "logits/chosen": 0.25128045678138733, + "logits/rejected": 0.9328861832618713, + "logps/chosen": -1.126585602760315, + "logps/rejected": -10.326457977294922, + "loss": 3.3699, + "nll_loss": 3.3491053581237793, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11265856772661209, + "rewards/margins": 0.9199872612953186, + "rewards/rejected": -1.03264582157135, + "step": 804 + }, + { + "epoch": 0.5007776049766719, + "grad_norm": 0.6790135502815247, + "learning_rate": 2.9875000000000004e-05, + "log_odds_chosen": 10.101418495178223, + "log_odds_ratio": -0.3796658217906952, + "logits/chosen": 0.1424853503704071, + "logits/rejected": 0.5358273386955261, + "logps/chosen": -1.3166006803512573, + "logps/rejected": -11.287535667419434, + "loss": 2.6253, + "nll_loss": 2.5873122215270996, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1316600739955902, + "rewards/margins": 0.9970934391021729, + "rewards/rejected": -1.1287535429000854, + "step": 805 + }, + { + "epoch": 0.5013996889580093, + "grad_norm": 0.5378172993659973, + "learning_rate": 2.985e-05, + "log_odds_chosen": 12.793437957763672, + "log_odds_ratio": -0.16563086211681366, + "logits/chosen": 0.2660048007965088, + "logits/rejected": 1.0401332378387451, + "logps/chosen": -1.2829506397247314, + "logps/rejected": -13.792176246643066, + "loss": 2.8793, + "nll_loss": 2.8627707958221436, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12829506397247314, + "rewards/margins": 1.250922679901123, + "rewards/rejected": -1.3792178630828857, + "step": 806 + }, + { + "epoch": 0.5020217729393468, + "grad_norm": 0.44759759306907654, + "learning_rate": 2.9825000000000003e-05, + "log_odds_chosen": 8.082733154296875, + "log_odds_ratio": -0.29205119609832764, + "logits/chosen": 0.08207601308822632, + "logits/rejected": 0.7699632048606873, + "logps/chosen": -1.0160481929779053, + "logps/rejected": -8.517202377319336, + "loss": 2.4715, + "nll_loss": 2.442284107208252, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10160481929779053, + "rewards/margins": 0.7501154541969299, + "rewards/rejected": -0.8517202734947205, + "step": 807 + }, + { + "epoch": 0.5026438569206843, + "grad_norm": 12.664875030517578, + "learning_rate": 2.98e-05, + "log_odds_chosen": 9.370549201965332, + "log_odds_ratio": -0.25263848900794983, + "logits/chosen": 0.18772029876708984, + "logits/rejected": 0.5844571590423584, + "logps/chosen": -1.6902470588684082, + "logps/rejected": -10.821871757507324, + "loss": 3.2462, + "nll_loss": 3.220900774002075, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16902472078800201, + "rewards/margins": 0.9131625294685364, + "rewards/rejected": -1.0821871757507324, + "step": 808 + }, + { + "epoch": 0.5032659409020218, + "grad_norm": 0.785666823387146, + "learning_rate": 2.9775000000000002e-05, + "log_odds_chosen": 11.042867660522461, + "log_odds_ratio": -0.21754157543182373, + "logits/chosen": 0.30363738536834717, + "logits/rejected": 0.7481663227081299, + "logps/chosen": -1.3917642831802368, + "logps/rejected": -12.1473388671875, + "loss": 3.4199, + "nll_loss": 3.398167133331299, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13917642831802368, + "rewards/margins": 1.0755574703216553, + "rewards/rejected": -1.2147338390350342, + "step": 809 + }, + { + "epoch": 0.5038880248833593, + "grad_norm": 0.6624091267585754, + "learning_rate": 2.975e-05, + "log_odds_chosen": 12.086601257324219, + "log_odds_ratio": -0.16491734981536865, + "logits/chosen": 0.1394706517457962, + "logits/rejected": 0.6047402620315552, + "logps/chosen": -1.2884442806243896, + "logps/rejected": -13.085152626037598, + "loss": 2.5779, + "nll_loss": 2.561450481414795, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12884442508220673, + "rewards/margins": 1.179670810699463, + "rewards/rejected": -1.3085153102874756, + "step": 810 + }, + { + "epoch": 0.5045101088646967, + "grad_norm": 0.35120460391044617, + "learning_rate": 2.9725000000000004e-05, + "log_odds_chosen": 6.397805690765381, + "log_odds_ratio": -0.39501500129699707, + "logits/chosen": 0.25109362602233887, + "logits/rejected": 0.48641854524612427, + "logps/chosen": -1.0249440670013428, + "logps/rejected": -7.021490097045898, + "loss": 3.4616, + "nll_loss": 3.4220848083496094, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10249441117048264, + "rewards/margins": 0.5996546149253845, + "rewards/rejected": -0.7021490335464478, + "step": 811 + }, + { + "epoch": 0.5051321928460342, + "grad_norm": 0.5609656572341919, + "learning_rate": 2.97e-05, + "log_odds_chosen": 15.698429107666016, + "log_odds_ratio": -0.017468160018324852, + "logits/chosen": 0.3717779815196991, + "logits/rejected": 1.3647180795669556, + "logps/chosen": -1.1899399757385254, + "logps/rejected": -16.393325805664062, + "loss": 3.1869, + "nll_loss": 3.1851115226745605, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11899399757385254, + "rewards/margins": 1.520338773727417, + "rewards/rejected": -1.63933265209198, + "step": 812 + }, + { + "epoch": 0.5057542768273717, + "grad_norm": 0.436008095741272, + "learning_rate": 2.9675000000000003e-05, + "log_odds_chosen": 17.379844665527344, + "log_odds_ratio": -0.048989683389663696, + "logits/chosen": 0.2973853647708893, + "logits/rejected": 1.2079492807388306, + "logps/chosen": -1.2121381759643555, + "logps/rejected": -18.229488372802734, + "loss": 3.0875, + "nll_loss": 3.0826025009155273, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12121382355690002, + "rewards/margins": 1.7017349004745483, + "rewards/rejected": -1.822948694229126, + "step": 813 + }, + { + "epoch": 0.5063763608087092, + "grad_norm": 0.4556023180484772, + "learning_rate": 2.965e-05, + "log_odds_chosen": 17.50226402282715, + "log_odds_ratio": -0.00011185341281816363, + "logits/chosen": 0.230424702167511, + "logits/rejected": 0.8504374027252197, + "logps/chosen": -1.146848440170288, + "logps/rejected": -18.064584732055664, + "loss": 3.1448, + "nll_loss": 3.1448068618774414, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11468484997749329, + "rewards/margins": 1.6917736530303955, + "rewards/rejected": -1.8064584732055664, + "step": 814 + }, + { + "epoch": 0.5069984447900466, + "grad_norm": 0.3557608723640442, + "learning_rate": 2.9625000000000002e-05, + "log_odds_chosen": 9.623047828674316, + "log_odds_ratio": -0.2141369879245758, + "logits/chosen": 0.22592505812644958, + "logits/rejected": 0.46705248951911926, + "logps/chosen": -1.2357347011566162, + "logps/rejected": -10.610895156860352, + "loss": 3.3321, + "nll_loss": 3.3106772899627686, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12357348203659058, + "rewards/margins": 0.9375160336494446, + "rewards/rejected": -1.0610895156860352, + "step": 815 + }, + { + "epoch": 0.5076205287713841, + "grad_norm": 0.4675641655921936, + "learning_rate": 2.96e-05, + "log_odds_chosen": 14.062185287475586, + "log_odds_ratio": -0.10161672532558441, + "logits/chosen": 0.18339157104492188, + "logits/rejected": 1.1679893732070923, + "logps/chosen": -1.142388105392456, + "logps/rejected": -14.841217994689941, + "loss": 2.7793, + "nll_loss": 2.769094705581665, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11423881351947784, + "rewards/margins": 1.3698830604553223, + "rewards/rejected": -1.4841217994689941, + "step": 816 + }, + { + "epoch": 0.5082426127527216, + "grad_norm": 0.3435609042644501, + "learning_rate": 2.9575000000000004e-05, + "log_odds_chosen": 11.574292182922363, + "log_odds_ratio": -0.1866357922554016, + "logits/chosen": 0.3857420086860657, + "logits/rejected": 1.0843833684921265, + "logps/chosen": -1.4344037771224976, + "logps/rejected": -12.822274208068848, + "loss": 3.5625, + "nll_loss": 3.543877124786377, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1434403955936432, + "rewards/margins": 1.1387871503829956, + "rewards/rejected": -1.2822275161743164, + "step": 817 + }, + { + "epoch": 0.5088646967340591, + "grad_norm": 0.463178813457489, + "learning_rate": 2.955e-05, + "log_odds_chosen": 11.81800651550293, + "log_odds_ratio": -0.18851415812969208, + "logits/chosen": 0.285900741815567, + "logits/rejected": 1.0203773975372314, + "logps/chosen": -1.2619681358337402, + "logps/rejected": -12.845138549804688, + "loss": 3.0084, + "nll_loss": 2.9895620346069336, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12619681656360626, + "rewards/margins": 1.1583170890808105, + "rewards/rejected": -1.2845139503479004, + "step": 818 + }, + { + "epoch": 0.5094867807153965, + "grad_norm": 0.38956987857818604, + "learning_rate": 2.9525000000000003e-05, + "log_odds_chosen": 14.569768905639648, + "log_odds_ratio": -0.0029117946978658438, + "logits/chosen": 0.19263651967048645, + "logits/rejected": 1.1464442014694214, + "logps/chosen": -0.9850395917892456, + "logps/rejected": -15.035454750061035, + "loss": 2.6472, + "nll_loss": 2.646864891052246, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0985039621591568, + "rewards/margins": 1.4050414562225342, + "rewards/rejected": -1.5035455226898193, + "step": 819 + }, + { + "epoch": 0.5101088646967341, + "grad_norm": 0.3614479601383209, + "learning_rate": 2.95e-05, + "log_odds_chosen": 13.27166748046875, + "log_odds_ratio": -0.0006157811731100082, + "logits/chosen": 0.2117461860179901, + "logits/rejected": 1.3962442874908447, + "logps/chosen": -1.2849613428115845, + "logps/rejected": -14.16645622253418, + "loss": 3.0077, + "nll_loss": 3.0076475143432617, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12849614024162292, + "rewards/margins": 1.2881494760513306, + "rewards/rejected": -1.4166457653045654, + "step": 820 + }, + { + "epoch": 0.5107309486780716, + "grad_norm": 0.4717678129673004, + "learning_rate": 2.9475e-05, + "log_odds_chosen": 10.178583145141602, + "log_odds_ratio": -0.13750000298023224, + "logits/chosen": 0.20223943889141083, + "logits/rejected": 1.0256826877593994, + "logps/chosen": -0.9736385345458984, + "logps/rejected": -10.707265853881836, + "loss": 2.6019, + "nll_loss": 2.5881075859069824, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09736385941505432, + "rewards/margins": 0.9733626842498779, + "rewards/rejected": -1.0707266330718994, + "step": 821 + }, + { + "epoch": 0.511353032659409, + "grad_norm": 0.37724214792251587, + "learning_rate": 2.945e-05, + "log_odds_chosen": 15.004103660583496, + "log_odds_ratio": -0.0004430253757163882, + "logits/chosen": 0.3318445086479187, + "logits/rejected": 0.974646806716919, + "logps/chosen": -1.1180328130722046, + "logps/rejected": -15.67713451385498, + "loss": 3.4453, + "nll_loss": 3.4452362060546875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11180329322814941, + "rewards/margins": 1.4559102058410645, + "rewards/rejected": -1.5677136182785034, + "step": 822 + }, + { + "epoch": 0.5119751166407465, + "grad_norm": 0.4549922049045563, + "learning_rate": 2.9425000000000004e-05, + "log_odds_chosen": 15.360553741455078, + "log_odds_ratio": -0.050239816308021545, + "logits/chosen": 0.44666236639022827, + "logits/rejected": 1.4149678945541382, + "logps/chosen": -1.1860952377319336, + "logps/rejected": -15.948168754577637, + "loss": 3.2089, + "nll_loss": 3.203843593597412, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11860951781272888, + "rewards/margins": 1.4762073755264282, + "rewards/rejected": -1.5948166847229004, + "step": 823 + }, + { + "epoch": 0.512597200622084, + "grad_norm": 0.535690426826477, + "learning_rate": 2.94e-05, + "log_odds_chosen": 8.157598495483398, + "log_odds_ratio": -0.32862338423728943, + "logits/chosen": 0.19047969579696655, + "logits/rejected": 0.8336578011512756, + "logps/chosen": -1.151352882385254, + "logps/rejected": -8.862951278686523, + "loss": 2.6731, + "nll_loss": 2.6401925086975098, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11513528972864151, + "rewards/margins": 0.771159827709198, + "rewards/rejected": -0.8862950801849365, + "step": 824 + }, + { + "epoch": 0.5132192846034215, + "grad_norm": 0.38767126202583313, + "learning_rate": 2.9375000000000003e-05, + "log_odds_chosen": 8.195636749267578, + "log_odds_ratio": -0.2015450894832611, + "logits/chosen": 0.21387703716754913, + "logits/rejected": 0.6429730653762817, + "logps/chosen": -1.1645689010620117, + "logps/rejected": -8.997748374938965, + "loss": 3.2326, + "nll_loss": 3.212407112121582, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11645688116550446, + "rewards/margins": 0.7833179831504822, + "rewards/rejected": -0.8997748494148254, + "step": 825 + }, + { + "epoch": 0.5138413685847589, + "grad_norm": 0.5092079043388367, + "learning_rate": 2.935e-05, + "log_odds_chosen": 5.449245929718018, + "log_odds_ratio": -0.11578687280416489, + "logits/chosen": 0.24035300314426422, + "logits/rejected": 0.5837621092796326, + "logps/chosen": -1.1935652494430542, + "logps/rejected": -6.307559490203857, + "loss": 3.0066, + "nll_loss": 2.9950246810913086, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11935652047395706, + "rewards/margins": 0.5113994479179382, + "rewards/rejected": -0.6307559013366699, + "step": 826 + }, + { + "epoch": 0.5144634525660964, + "grad_norm": 0.6121360063552856, + "learning_rate": 2.9325e-05, + "log_odds_chosen": 8.71769905090332, + "log_odds_ratio": -0.22162342071533203, + "logits/chosen": 0.2978130578994751, + "logits/rejected": 0.6275981664657593, + "logps/chosen": -1.168921709060669, + "logps/rejected": -9.59075927734375, + "loss": 3.1936, + "nll_loss": 3.171393871307373, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11689215898513794, + "rewards/margins": 0.8421837687492371, + "rewards/rejected": -0.959075927734375, + "step": 827 + }, + { + "epoch": 0.5150855365474339, + "grad_norm": 0.4831790626049042, + "learning_rate": 2.93e-05, + "log_odds_chosen": 14.293399810791016, + "log_odds_ratio": -0.09568853676319122, + "logits/chosen": 0.3437972664833069, + "logits/rejected": 1.154407262802124, + "logps/chosen": -1.0440468788146973, + "logps/rejected": -14.955587387084961, + "loss": 3.0785, + "nll_loss": 3.068910837173462, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10440470278263092, + "rewards/margins": 1.3911540508270264, + "rewards/rejected": -1.495558500289917, + "step": 828 + }, + { + "epoch": 0.5157076205287714, + "grad_norm": 0.8993392586708069, + "learning_rate": 2.9275000000000003e-05, + "log_odds_chosen": 12.711183547973633, + "log_odds_ratio": -0.09046018123626709, + "logits/chosen": 0.41627803444862366, + "logits/rejected": 1.3701362609863281, + "logps/chosen": -1.0129324197769165, + "logps/rejected": -13.203245162963867, + "loss": 2.6354, + "nll_loss": 2.626333236694336, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10129325091838837, + "rewards/margins": 1.2190312147140503, + "rewards/rejected": -1.3203246593475342, + "step": 829 + }, + { + "epoch": 0.5163297045101088, + "grad_norm": 0.5971187949180603, + "learning_rate": 2.925e-05, + "log_odds_chosen": 7.836894989013672, + "log_odds_ratio": -0.3798048496246338, + "logits/chosen": 0.3004685938358307, + "logits/rejected": 0.724410355091095, + "logps/chosen": -1.366215705871582, + "logps/rejected": -9.068084716796875, + "loss": 3.2621, + "nll_loss": 3.224073886871338, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13662157952785492, + "rewards/margins": 0.7701869606971741, + "rewards/rejected": -0.9068085551261902, + "step": 830 + }, + { + "epoch": 0.5169517884914463, + "grad_norm": 0.36452698707580566, + "learning_rate": 2.9225000000000002e-05, + "log_odds_chosen": 15.304241180419922, + "log_odds_ratio": -0.0003950538521166891, + "logits/chosen": 0.3968985080718994, + "logits/rejected": 1.0415889024734497, + "logps/chosen": -1.0593630075454712, + "logps/rejected": -15.823348999023438, + "loss": 3.9988, + "nll_loss": 3.9988064765930176, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10593629628419876, + "rewards/margins": 1.4763985872268677, + "rewards/rejected": -1.5823347568511963, + "step": 831 + }, + { + "epoch": 0.5175738724727839, + "grad_norm": 1.9738948345184326, + "learning_rate": 2.9199999999999998e-05, + "log_odds_chosen": 12.84214973449707, + "log_odds_ratio": -0.1361134946346283, + "logits/chosen": 0.08501138538122177, + "logits/rejected": 0.4218248426914215, + "logps/chosen": -1.936566710472107, + "logps/rejected": -14.241930961608887, + "loss": 3.0152, + "nll_loss": 3.0015807151794434, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.19365663826465607, + "rewards/margins": 1.2305364608764648, + "rewards/rejected": -1.4241931438446045, + "step": 832 + }, + { + "epoch": 0.5181959564541213, + "grad_norm": 0.4944245219230652, + "learning_rate": 2.9175e-05, + "log_odds_chosen": 20.146743774414062, + "log_odds_ratio": -9.030426554090809e-06, + "logits/chosen": 0.2090873271226883, + "logits/rejected": 1.6611508131027222, + "logps/chosen": -1.001164197921753, + "logps/rejected": -20.59023094177246, + "loss": 2.5188, + "nll_loss": 2.5187838077545166, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10011641681194305, + "rewards/margins": 1.958906650543213, + "rewards/rejected": -2.059023141860962, + "step": 833 + }, + { + "epoch": 0.5188180404354588, + "grad_norm": 1.0386806726455688, + "learning_rate": 2.915e-05, + "log_odds_chosen": 10.240373611450195, + "log_odds_ratio": -0.17523886263370514, + "logits/chosen": 0.2517136335372925, + "logits/rejected": 0.601931095123291, + "logps/chosen": -1.1842358112335205, + "logps/rejected": -11.045554161071777, + "loss": 3.3807, + "nll_loss": 3.363182544708252, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11842358112335205, + "rewards/margins": 0.9861319661140442, + "rewards/rejected": -1.1045554876327515, + "step": 834 + }, + { + "epoch": 0.5194401244167963, + "grad_norm": 4.311429023742676, + "learning_rate": 2.9125000000000003e-05, + "log_odds_chosen": 12.021062850952148, + "log_odds_ratio": -0.011027473025023937, + "logits/chosen": 0.22837401926517487, + "logits/rejected": 0.7856446504592896, + "logps/chosen": -1.6762373447418213, + "logps/rejected": -13.32023811340332, + "loss": 2.9735, + "nll_loss": 2.9724273681640625, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16762374341487885, + "rewards/margins": 1.1644001007080078, + "rewards/rejected": -1.3320238590240479, + "step": 835 + }, + { + "epoch": 0.5200622083981338, + "grad_norm": 1.8414102792739868, + "learning_rate": 2.91e-05, + "log_odds_chosen": 13.802828788757324, + "log_odds_ratio": -0.06723160296678543, + "logits/chosen": 0.19973208010196686, + "logits/rejected": 1.1031876802444458, + "logps/chosen": -1.8014975786209106, + "logps/rejected": -15.22963809967041, + "loss": 2.9892, + "nll_loss": 2.982433319091797, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18014976382255554, + "rewards/margins": 1.3428139686584473, + "rewards/rejected": -1.5229637622833252, + "step": 836 + }, + { + "epoch": 0.5206842923794712, + "grad_norm": 0.47048988938331604, + "learning_rate": 2.9075000000000002e-05, + "log_odds_chosen": 16.698774337768555, + "log_odds_ratio": -1.1175909548910568e-06, + "logits/chosen": 0.33147627115249634, + "logits/rejected": 1.3666659593582153, + "logps/chosen": -1.1629831790924072, + "logps/rejected": -17.404766082763672, + "loss": 3.2272, + "nll_loss": 3.2272276878356934, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11629832535982132, + "rewards/margins": 1.624178171157837, + "rewards/rejected": -1.7404766082763672, + "step": 837 + }, + { + "epoch": 0.5213063763608087, + "grad_norm": 1.6325106620788574, + "learning_rate": 2.9049999999999998e-05, + "log_odds_chosen": 4.975150108337402, + "log_odds_ratio": -0.32594966888427734, + "logits/chosen": 0.16253390908241272, + "logits/rejected": 0.3543585538864136, + "logps/chosen": -1.0561513900756836, + "logps/rejected": -5.589508056640625, + "loss": 3.1341, + "nll_loss": 3.1014583110809326, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10561515390872955, + "rewards/margins": 0.453335702419281, + "rewards/rejected": -0.5589507818222046, + "step": 838 + }, + { + "epoch": 0.5219284603421462, + "grad_norm": 0.4523632228374481, + "learning_rate": 2.9025e-05, + "log_odds_chosen": 13.034244537353516, + "log_odds_ratio": -0.0004373805713839829, + "logits/chosen": 0.23802269995212555, + "logits/rejected": 1.02839994430542, + "logps/chosen": -1.0669258832931519, + "logps/rejected": -13.587167739868164, + "loss": 2.9686, + "nll_loss": 2.968569755554199, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1066925898194313, + "rewards/margins": 1.2520242929458618, + "rewards/rejected": -1.3587168455123901, + "step": 839 + }, + { + "epoch": 0.5225505443234837, + "grad_norm": 0.46523481607437134, + "learning_rate": 2.9e-05, + "log_odds_chosen": 5.310853958129883, + "log_odds_ratio": -0.19879606366157532, + "logits/chosen": 0.12425953149795532, + "logits/rejected": 0.24234220385551453, + "logps/chosen": -0.9893929362297058, + "logps/rejected": -5.892968654632568, + "loss": 3.0965, + "nll_loss": 3.0765981674194336, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09893929213285446, + "rewards/margins": 0.49035757780075073, + "rewards/rejected": -0.5892968773841858, + "step": 840 + }, + { + "epoch": 0.5231726283048211, + "grad_norm": 0.43103811144828796, + "learning_rate": 2.8975000000000003e-05, + "log_odds_chosen": 7.345240592956543, + "log_odds_ratio": -0.3413081169128418, + "logits/chosen": 0.015453029423952103, + "logits/rejected": 0.28171202540397644, + "logps/chosen": -0.8644816279411316, + "logps/rejected": -7.795204162597656, + "loss": 2.733, + "nll_loss": 2.6988906860351562, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08644817024469376, + "rewards/margins": 0.6930723190307617, + "rewards/rejected": -0.7795204520225525, + "step": 841 + }, + { + "epoch": 0.5237947122861586, + "grad_norm": 0.4897030293941498, + "learning_rate": 2.895e-05, + "log_odds_chosen": 4.109506607055664, + "log_odds_ratio": -0.5368872284889221, + "logits/chosen": 0.2146613895893097, + "logits/rejected": 0.4014909565448761, + "logps/chosen": -1.3654534816741943, + "logps/rejected": -5.355642318725586, + "loss": 3.0509, + "nll_loss": 2.997218132019043, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1365453451871872, + "rewards/margins": 0.39901888370513916, + "rewards/rejected": -0.5355641841888428, + "step": 842 + }, + { + "epoch": 0.5244167962674962, + "grad_norm": 0.47838544845581055, + "learning_rate": 2.8925000000000002e-05, + "log_odds_chosen": 2.614036798477173, + "log_odds_ratio": -0.4196464419364929, + "logits/chosen": 0.04910755902528763, + "logits/rejected": 0.2252475917339325, + "logps/chosen": -1.306127667427063, + "logps/rejected": -3.7415530681610107, + "loss": 2.6839, + "nll_loss": 2.641903877258301, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13061276078224182, + "rewards/margins": 0.24354252219200134, + "rewards/rejected": -0.37415528297424316, + "step": 843 + }, + { + "epoch": 0.5250388802488336, + "grad_norm": 0.329008549451828, + "learning_rate": 2.8899999999999998e-05, + "log_odds_chosen": 4.632389068603516, + "log_odds_ratio": -0.3140389621257782, + "logits/chosen": 0.2015816867351532, + "logits/rejected": 0.3459731340408325, + "logps/chosen": -1.283074975013733, + "logps/rejected": -5.729971408843994, + "loss": 3.4623, + "nll_loss": 3.430915355682373, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1283074915409088, + "rewards/margins": 0.44468966126441956, + "rewards/rejected": -0.5729971528053284, + "step": 844 + }, + { + "epoch": 0.5256609642301711, + "grad_norm": 0.43985334038734436, + "learning_rate": 2.8875e-05, + "log_odds_chosen": 1.6101486682891846, + "log_odds_ratio": -0.4912552237510681, + "logits/chosen": 0.13369233906269073, + "logits/rejected": 0.1671290248632431, + "logps/chosen": -1.0167124271392822, + "logps/rejected": -2.167614698410034, + "loss": 2.8231, + "nll_loss": 2.7740092277526855, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1016712337732315, + "rewards/margins": 0.11509023606777191, + "rewards/rejected": -0.2167614847421646, + "step": 845 + }, + { + "epoch": 0.5262830482115085, + "grad_norm": 0.4271375834941864, + "learning_rate": 2.885e-05, + "log_odds_chosen": 7.256542205810547, + "log_odds_ratio": -0.037149470299482346, + "logits/chosen": 0.24337813258171082, + "logits/rejected": 0.6470229625701904, + "logps/chosen": -0.8154711723327637, + "logps/rejected": -7.411262512207031, + "loss": 3.2228, + "nll_loss": 3.219062328338623, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08154712617397308, + "rewards/margins": 0.6595790386199951, + "rewards/rejected": -0.7411262392997742, + "step": 846 + }, + { + "epoch": 0.5269051321928461, + "grad_norm": 1.1752288341522217, + "learning_rate": 2.8825000000000003e-05, + "log_odds_chosen": 5.8892974853515625, + "log_odds_ratio": -0.18083639442920685, + "logits/chosen": 0.22460246086120605, + "logits/rejected": 0.6375922560691833, + "logps/chosen": -1.37664794921875, + "logps/rejected": -7.0137481689453125, + "loss": 3.0231, + "nll_loss": 3.0050535202026367, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1376648098230362, + "rewards/margins": 0.5637100338935852, + "rewards/rejected": -0.701374888420105, + "step": 847 + }, + { + "epoch": 0.5275272161741835, + "grad_norm": 0.3938504457473755, + "learning_rate": 2.88e-05, + "log_odds_chosen": 6.437436580657959, + "log_odds_ratio": -0.1518944650888443, + "logits/chosen": 0.2168705314397812, + "logits/rejected": 0.40676745772361755, + "logps/chosen": -1.264991283416748, + "logps/rejected": -7.434615135192871, + "loss": 3.5069, + "nll_loss": 3.4916627407073975, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12649914622306824, + "rewards/margins": 0.6169624328613281, + "rewards/rejected": -0.7434616088867188, + "step": 848 + }, + { + "epoch": 0.528149300155521, + "grad_norm": 0.36738061904907227, + "learning_rate": 2.8775e-05, + "log_odds_chosen": 4.217384338378906, + "log_odds_ratio": -0.19647136330604553, + "logits/chosen": 0.1279819905757904, + "logits/rejected": 0.30928248167037964, + "logps/chosen": -1.111844539642334, + "logps/rejected": -4.994659423828125, + "loss": 3.0897, + "nll_loss": 3.0701029300689697, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11118445545434952, + "rewards/margins": 0.3882814049720764, + "rewards/rejected": -0.4994658827781677, + "step": 849 + }, + { + "epoch": 0.5287713841368584, + "grad_norm": 0.49180299043655396, + "learning_rate": 2.8749999999999997e-05, + "log_odds_chosen": 3.4940242767333984, + "log_odds_ratio": -0.29053792357444763, + "logits/chosen": 0.3198105990886688, + "logits/rejected": 0.5212329030036926, + "logps/chosen": -1.107250452041626, + "logps/rejected": -4.281674385070801, + "loss": 3.3457, + "nll_loss": 3.3166162967681885, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1107250452041626, + "rewards/margins": 0.317442387342453, + "rewards/rejected": -0.4281674325466156, + "step": 850 + }, + { + "epoch": 0.529393468118196, + "grad_norm": 0.5136086940765381, + "learning_rate": 2.8725e-05, + "log_odds_chosen": 4.076441764831543, + "log_odds_ratio": -0.2946397066116333, + "logits/chosen": -0.006016634404659271, + "logits/rejected": 0.35069453716278076, + "logps/chosen": -1.2343401908874512, + "logps/rejected": -5.004025936126709, + "loss": 2.3004, + "nll_loss": 2.270923376083374, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12343402206897736, + "rewards/margins": 0.3769685626029968, + "rewards/rejected": -0.500402569770813, + "step": 851 + }, + { + "epoch": 0.5300155520995334, + "grad_norm": 0.42883893847465515, + "learning_rate": 2.87e-05, + "log_odds_chosen": 2.978586196899414, + "log_odds_ratio": -0.30816155672073364, + "logits/chosen": 0.10813058167695999, + "logits/rejected": 0.12170670181512833, + "logps/chosen": -0.927642822265625, + "logps/rejected": -3.484469413757324, + "loss": 3.1087, + "nll_loss": 3.0778818130493164, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09276428818702698, + "rewards/margins": 0.25568264722824097, + "rewards/rejected": -0.34844693541526794, + "step": 852 + }, + { + "epoch": 0.5306376360808709, + "grad_norm": 0.4703337550163269, + "learning_rate": 2.8675000000000002e-05, + "log_odds_chosen": 4.520643711090088, + "log_odds_ratio": -0.3384997248649597, + "logits/chosen": 0.1783015877008438, + "logits/rejected": 0.42561283707618713, + "logps/chosen": -1.0047498941421509, + "logps/rejected": -5.196669101715088, + "loss": 2.9576, + "nll_loss": 2.923743486404419, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10047499090433121, + "rewards/margins": 0.41919195652008057, + "rewards/rejected": -0.5196669101715088, + "step": 853 + }, + { + "epoch": 0.5312597200622085, + "grad_norm": 0.5501247644424438, + "learning_rate": 2.865e-05, + "log_odds_chosen": 3.5761749744415283, + "log_odds_ratio": -0.1852397471666336, + "logits/chosen": 0.2851634919643402, + "logits/rejected": 0.3833601772785187, + "logps/chosen": -1.2000057697296143, + "logps/rejected": -4.437448024749756, + "loss": 3.2604, + "nll_loss": 3.241886615753174, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12000058591365814, + "rewards/margins": 0.3237442076206207, + "rewards/rejected": -0.44374483823776245, + "step": 854 + }, + { + "epoch": 0.5318818040435459, + "grad_norm": 0.4883931577205658, + "learning_rate": 2.8625e-05, + "log_odds_chosen": 4.188264846801758, + "log_odds_ratio": -0.2911498546600342, + "logits/chosen": 0.11234842240810394, + "logits/rejected": 0.4352606236934662, + "logps/chosen": -1.0508315563201904, + "logps/rejected": -4.962338447570801, + "loss": 2.8563, + "nll_loss": 2.827171564102173, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1050831526517868, + "rewards/margins": 0.39115068316459656, + "rewards/rejected": -0.49623382091522217, + "step": 855 + }, + { + "epoch": 0.5325038880248834, + "grad_norm": 0.523665189743042, + "learning_rate": 2.86e-05, + "log_odds_chosen": 5.883430004119873, + "log_odds_ratio": -0.3124341666698456, + "logits/chosen": 0.18846668303012848, + "logits/rejected": 0.35798412561416626, + "logps/chosen": -0.9777987003326416, + "logps/rejected": -6.380369663238525, + "loss": 3.1017, + "nll_loss": 3.070420026779175, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09777987003326416, + "rewards/margins": 0.5402571558952332, + "rewards/rejected": -0.6380370259284973, + "step": 856 + }, + { + "epoch": 0.5331259720062208, + "grad_norm": 0.4752536714076996, + "learning_rate": 2.8575000000000003e-05, + "log_odds_chosen": 4.861764907836914, + "log_odds_ratio": -0.13584004342556, + "logits/chosen": 0.29713207483291626, + "logits/rejected": 0.5332407355308533, + "logps/chosen": -1.4573677778244019, + "logps/rejected": -6.071244239807129, + "loss": 3.228, + "nll_loss": 3.2143874168395996, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14573678374290466, + "rewards/margins": 0.46138763427734375, + "rewards/rejected": -0.6071244478225708, + "step": 857 + }, + { + "epoch": 0.5337480559875584, + "grad_norm": 0.3933459222316742, + "learning_rate": 2.855e-05, + "log_odds_chosen": 5.766116142272949, + "log_odds_ratio": -0.2710520327091217, + "logits/chosen": 0.2104707956314087, + "logits/rejected": 0.3261168599128723, + "logps/chosen": -0.9775829315185547, + "logps/rejected": -6.416999816894531, + "loss": 3.2929, + "nll_loss": 3.2657809257507324, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09775829315185547, + "rewards/margins": 0.5439417362213135, + "rewards/rejected": -0.6416999697685242, + "step": 858 + }, + { + "epoch": 0.5343701399688958, + "grad_norm": 0.5813342332839966, + "learning_rate": 2.8525000000000002e-05, + "log_odds_chosen": 4.442496299743652, + "log_odds_ratio": -0.17784219980239868, + "logits/chosen": 0.24876517057418823, + "logits/rejected": 0.58121657371521, + "logps/chosen": -1.5985615253448486, + "logps/rejected": -5.860179901123047, + "loss": 3.1737, + "nll_loss": 3.155898094177246, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1598561555147171, + "rewards/margins": 0.42616188526153564, + "rewards/rejected": -0.5860180258750916, + "step": 859 + }, + { + "epoch": 0.5349922239502333, + "grad_norm": 0.4372238218784332, + "learning_rate": 2.8499999999999998e-05, + "log_odds_chosen": 5.981117248535156, + "log_odds_ratio": -0.1759411096572876, + "logits/chosen": 0.1712474822998047, + "logits/rejected": 0.4489181637763977, + "logps/chosen": -1.0640121698379517, + "logps/rejected": -6.628488540649414, + "loss": 3.1263, + "nll_loss": 3.1087098121643066, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.106401227414608, + "rewards/margins": 0.5564476251602173, + "rewards/rejected": -0.6628488302230835, + "step": 860 + }, + { + "epoch": 0.5356143079315707, + "grad_norm": 0.5723143815994263, + "learning_rate": 2.8475e-05, + "log_odds_chosen": 3.2769057750701904, + "log_odds_ratio": -0.3183494210243225, + "logits/chosen": 0.15005794167518616, + "logits/rejected": 0.4674699604511261, + "logps/chosen": -1.1343958377838135, + "logps/rejected": -4.155270576477051, + "loss": 2.7241, + "nll_loss": 2.6922266483306885, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11343959718942642, + "rewards/margins": 0.3020874857902527, + "rewards/rejected": -0.4155270755290985, + "step": 861 + }, + { + "epoch": 0.5362363919129083, + "grad_norm": 0.3801248073577881, + "learning_rate": 2.845e-05, + "log_odds_chosen": 5.401886940002441, + "log_odds_ratio": -0.1325007528066635, + "logits/chosen": 0.2797131836414337, + "logits/rejected": 0.7725595235824585, + "logps/chosen": -1.2554233074188232, + "logps/rejected": -6.341432094573975, + "loss": 3.4174, + "nll_loss": 3.4041857719421387, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12554234266281128, + "rewards/margins": 0.5086008906364441, + "rewards/rejected": -0.6341432332992554, + "step": 862 + }, + { + "epoch": 0.5368584758942457, + "grad_norm": 0.45833510160446167, + "learning_rate": 2.8425000000000003e-05, + "log_odds_chosen": 7.4684834480285645, + "log_odds_ratio": -0.2111528515815735, + "logits/chosen": 0.3720274269580841, + "logits/rejected": 1.0443079471588135, + "logps/chosen": -1.074413776397705, + "logps/rejected": -8.263851165771484, + "loss": 3.4695, + "nll_loss": 3.448345184326172, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10744138062000275, + "rewards/margins": 0.7189437747001648, + "rewards/rejected": -0.8263851404190063, + "step": 863 + }, + { + "epoch": 0.5374805598755832, + "grad_norm": 0.5950579047203064, + "learning_rate": 2.84e-05, + "log_odds_chosen": 6.653730392456055, + "log_odds_ratio": -0.2213490754365921, + "logits/chosen": 0.1977083534002304, + "logits/rejected": 0.6113790273666382, + "logps/chosen": -1.0264054536819458, + "logps/rejected": -7.271251678466797, + "loss": 2.6901, + "nll_loss": 2.667936086654663, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1026405543088913, + "rewards/margins": 0.624484658241272, + "rewards/rejected": -0.7271252274513245, + "step": 864 + }, + { + "epoch": 0.5381026438569206, + "grad_norm": 0.5189838409423828, + "learning_rate": 2.8375000000000002e-05, + "log_odds_chosen": 4.811558246612549, + "log_odds_ratio": -0.20530752837657928, + "logits/chosen": 0.29856374859809875, + "logits/rejected": 0.7025430202484131, + "logps/chosen": -1.3187425136566162, + "logps/rejected": -5.89030122756958, + "loss": 3.147, + "nll_loss": 3.1264843940734863, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13187426328659058, + "rewards/margins": 0.45715591311454773, + "rewards/rejected": -0.5890301465988159, + "step": 865 + }, + { + "epoch": 0.5387247278382582, + "grad_norm": 0.5219860076904297, + "learning_rate": 2.8349999999999998e-05, + "log_odds_chosen": 10.06355094909668, + "log_odds_ratio": -0.06661475449800491, + "logits/chosen": 0.16469168663024902, + "logits/rejected": 0.5855368971824646, + "logps/chosen": -0.9745451807975769, + "logps/rejected": -10.555267333984375, + "loss": 3.0044, + "nll_loss": 2.997767448425293, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09745452553033829, + "rewards/margins": 0.9580721855163574, + "rewards/rejected": -1.0555267333984375, + "step": 866 + }, + { + "epoch": 0.5393468118195957, + "grad_norm": 0.40929824113845825, + "learning_rate": 2.8325e-05, + "log_odds_chosen": 11.310032844543457, + "log_odds_ratio": -0.1569271832704544, + "logits/chosen": 0.28702273964881897, + "logits/rejected": 0.9445208311080933, + "logps/chosen": -1.1780142784118652, + "logps/rejected": -12.161791801452637, + "loss": 3.249, + "nll_loss": 3.2333545684814453, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11780144274234772, + "rewards/margins": 1.0983777046203613, + "rewards/rejected": -1.2161791324615479, + "step": 867 + }, + { + "epoch": 0.5399688958009331, + "grad_norm": 0.5718206763267517, + "learning_rate": 2.83e-05, + "log_odds_chosen": 9.48852252960205, + "log_odds_ratio": -0.23974888026714325, + "logits/chosen": 0.20256325602531433, + "logits/rejected": 0.7612900733947754, + "logps/chosen": -1.1071330308914185, + "logps/rejected": -10.291450500488281, + "loss": 2.7763, + "nll_loss": 2.752324104309082, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11071330308914185, + "rewards/margins": 0.91843181848526, + "rewards/rejected": -1.0291451215744019, + "step": 868 + }, + { + "epoch": 0.5405909797822706, + "grad_norm": 0.5120429396629333, + "learning_rate": 2.8275000000000003e-05, + "log_odds_chosen": 7.586127281188965, + "log_odds_ratio": -0.3463227152824402, + "logits/chosen": 0.20187543332576752, + "logits/rejected": 0.6514379382133484, + "logps/chosen": -1.0916359424591064, + "logps/rejected": -8.361661911010742, + "loss": 3.122, + "nll_loss": 3.087322235107422, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10916360467672348, + "rewards/margins": 0.7270025014877319, + "rewards/rejected": -0.8361661434173584, + "step": 869 + }, + { + "epoch": 0.5412130637636081, + "grad_norm": 0.5298638939857483, + "learning_rate": 2.825e-05, + "log_odds_chosen": 10.231219291687012, + "log_odds_ratio": -0.11733568459749222, + "logits/chosen": 0.16062955558300018, + "logits/rejected": 0.6653136610984802, + "logps/chosen": -1.0218132734298706, + "logps/rejected": -10.77658748626709, + "loss": 2.9773, + "nll_loss": 2.9655981063842773, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1021813303232193, + "rewards/margins": 0.9754774570465088, + "rewards/rejected": -1.0776586532592773, + "step": 870 + }, + { + "epoch": 0.5418351477449456, + "grad_norm": 0.629223644733429, + "learning_rate": 2.8225e-05, + "log_odds_chosen": 12.822935104370117, + "log_odds_ratio": -0.16422484815120697, + "logits/chosen": 0.18336540460586548, + "logits/rejected": 1.1963590383529663, + "logps/chosen": -1.1354261636734009, + "logps/rejected": -13.514695167541504, + "loss": 2.4868, + "nll_loss": 2.4704039096832275, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11354262381792068, + "rewards/margins": 1.2379268407821655, + "rewards/rejected": -1.3514695167541504, + "step": 871 + }, + { + "epoch": 0.542457231726283, + "grad_norm": 1.4472790956497192, + "learning_rate": 2.8199999999999998e-05, + "log_odds_chosen": 15.860309600830078, + "log_odds_ratio": -0.14200741052627563, + "logits/chosen": 0.1716628074645996, + "logits/rejected": 0.962594211101532, + "logps/chosen": -1.1267319917678833, + "logps/rejected": -16.692829132080078, + "loss": 2.6449, + "nll_loss": 2.630686044692993, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11267320811748505, + "rewards/margins": 1.5566097497940063, + "rewards/rejected": -1.6692829132080078, + "step": 872 + }, + { + "epoch": 0.5430793157076206, + "grad_norm": 0.4921049475669861, + "learning_rate": 2.8175e-05, + "log_odds_chosen": 16.460073471069336, + "log_odds_ratio": -0.06028711423277855, + "logits/chosen": 0.33112722635269165, + "logits/rejected": 1.0481317043304443, + "logps/chosen": -1.0758942365646362, + "logps/rejected": -16.93272590637207, + "loss": 3.3084, + "nll_loss": 3.30238676071167, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10758942365646362, + "rewards/margins": 1.5856831073760986, + "rewards/rejected": -1.693272590637207, + "step": 873 + }, + { + "epoch": 0.543701399688958, + "grad_norm": 0.47205042839050293, + "learning_rate": 2.815e-05, + "log_odds_chosen": 10.865059852600098, + "log_odds_ratio": -0.231788769364357, + "logits/chosen": 0.28823232650756836, + "logits/rejected": 1.0439519882202148, + "logps/chosen": -1.0718377828598022, + "logps/rejected": -11.658307075500488, + "loss": 3.0113, + "nll_loss": 2.988163948059082, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1071837842464447, + "rewards/margins": 1.0586470365524292, + "rewards/rejected": -1.1658308506011963, + "step": 874 + }, + { + "epoch": 0.5443234836702955, + "grad_norm": 0.370347797870636, + "learning_rate": 2.8125000000000003e-05, + "log_odds_chosen": 10.927844047546387, + "log_odds_ratio": -0.1950116902589798, + "logits/chosen": 0.17625781893730164, + "logits/rejected": 0.5121840238571167, + "logps/chosen": -0.975417971611023, + "logps/rejected": -11.16118049621582, + "loss": 3.09, + "nll_loss": 3.0705158710479736, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09754179418087006, + "rewards/margins": 1.0185761451721191, + "rewards/rejected": -1.1161179542541504, + "step": 875 + }, + { + "epoch": 0.5449455676516329, + "grad_norm": 0.4597725570201874, + "learning_rate": 2.8100000000000005e-05, + "log_odds_chosen": 9.441105842590332, + "log_odds_ratio": -0.18627074360847473, + "logits/chosen": 0.2629444897174835, + "logits/rejected": 0.7703899145126343, + "logps/chosen": -1.3209762573242188, + "logps/rejected": -10.505077362060547, + "loss": 3.265, + "nll_loss": 3.2463271617889404, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13209763169288635, + "rewards/margins": 0.9184101223945618, + "rewards/rejected": -1.0505077838897705, + "step": 876 + }, + { + "epoch": 0.5455676516329705, + "grad_norm": 0.3739025890827179, + "learning_rate": 2.8075e-05, + "log_odds_chosen": 11.734682083129883, + "log_odds_ratio": -0.07055188715457916, + "logits/chosen": 0.22402915358543396, + "logits/rejected": 0.826654314994812, + "logps/chosen": -1.6761119365692139, + "logps/rejected": -13.128270149230957, + "loss": 3.2389, + "nll_loss": 3.2317991256713867, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16761119663715363, + "rewards/margins": 1.1452157497406006, + "rewards/rejected": -1.3128271102905273, + "step": 877 + }, + { + "epoch": 0.546189735614308, + "grad_norm": 0.4357503354549408, + "learning_rate": 2.8050000000000004e-05, + "log_odds_chosen": 18.032630920410156, + "log_odds_ratio": -6.927158392500132e-05, + "logits/chosen": 0.4070179760456085, + "logits/rejected": 1.3222649097442627, + "logps/chosen": -1.4783521890640259, + "logps/rejected": -19.151775360107422, + "loss": 3.4047, + "nll_loss": 3.404698133468628, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14783522486686707, + "rewards/margins": 1.7673425674438477, + "rewards/rejected": -1.915177822113037, + "step": 878 + }, + { + "epoch": 0.5468118195956454, + "grad_norm": 0.4408838748931885, + "learning_rate": 2.8025e-05, + "log_odds_chosen": 10.542194366455078, + "log_odds_ratio": -0.28352150321006775, + "logits/chosen": 0.25072023272514343, + "logits/rejected": 1.1159553527832031, + "logps/chosen": -1.1630507707595825, + "logps/rejected": -11.427528381347656, + "loss": 2.7491, + "nll_loss": 2.7207608222961426, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11630507558584213, + "rewards/margins": 1.0264477729797363, + "rewards/rejected": -1.1427528858184814, + "step": 879 + }, + { + "epoch": 0.5474339035769828, + "grad_norm": 0.5349311828613281, + "learning_rate": 2.8000000000000003e-05, + "log_odds_chosen": 14.291757583618164, + "log_odds_ratio": -0.251430481672287, + "logits/chosen": 0.32711905241012573, + "logits/rejected": 0.9382522702217102, + "logps/chosen": -1.1399996280670166, + "logps/rejected": -15.159124374389648, + "loss": 2.9631, + "nll_loss": 2.937985420227051, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11399996280670166, + "rewards/margins": 1.4019126892089844, + "rewards/rejected": -1.5159125328063965, + "step": 880 + }, + { + "epoch": 0.5480559875583204, + "grad_norm": 0.4650278091430664, + "learning_rate": 2.7975000000000002e-05, + "log_odds_chosen": 14.79272747039795, + "log_odds_ratio": -0.06423240154981613, + "logits/chosen": 0.3779009282588959, + "logits/rejected": 1.0615414381027222, + "logps/chosen": -1.2400293350219727, + "logps/rejected": -15.624247550964355, + "loss": 3.1322, + "nll_loss": 3.1258246898651123, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12400293350219727, + "rewards/margins": 1.4384218454360962, + "rewards/rejected": -1.562424898147583, + "step": 881 + }, + { + "epoch": 0.5486780715396579, + "grad_norm": 0.4733143746852875, + "learning_rate": 2.7950000000000005e-05, + "log_odds_chosen": 6.083606719970703, + "log_odds_ratio": -0.33826759457588196, + "logits/chosen": 0.2951495349407196, + "logits/rejected": 0.5478895902633667, + "logps/chosen": -1.0794161558151245, + "logps/rejected": -6.8474345207214355, + "loss": 2.9234, + "nll_loss": 2.889528751373291, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10794162005186081, + "rewards/margins": 0.5768018364906311, + "rewards/rejected": -0.6847434639930725, + "step": 882 + }, + { + "epoch": 0.5493001555209953, + "grad_norm": 0.42593225836753845, + "learning_rate": 2.7925e-05, + "log_odds_chosen": 12.911933898925781, + "log_odds_ratio": -0.3391897976398468, + "logits/chosen": 0.31484758853912354, + "logits/rejected": 1.1003917455673218, + "logps/chosen": -1.1400558948516846, + "logps/rejected": -13.681197166442871, + "loss": 2.7707, + "nll_loss": 2.7367684841156006, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11400558799505234, + "rewards/margins": 1.2541141510009766, + "rewards/rejected": -1.368119716644287, + "step": 883 + }, + { + "epoch": 0.5499222395023328, + "grad_norm": 0.44411978125572205, + "learning_rate": 2.7900000000000004e-05, + "log_odds_chosen": 15.06024169921875, + "log_odds_ratio": -0.13867679238319397, + "logits/chosen": 0.2806948125362396, + "logits/rejected": 1.4216599464416504, + "logps/chosen": -1.1676044464111328, + "logps/rejected": -15.850728034973145, + "loss": 2.7267, + "nll_loss": 2.7128076553344727, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11676044762134552, + "rewards/margins": 1.468312382698059, + "rewards/rejected": -1.5850728750228882, + "step": 884 + }, + { + "epoch": 0.5505443234836703, + "grad_norm": 1.44214928150177, + "learning_rate": 2.7875e-05, + "log_odds_chosen": 12.520509719848633, + "log_odds_ratio": -0.18546557426452637, + "logits/chosen": 0.16061872243881226, + "logits/rejected": 0.9799090027809143, + "logps/chosen": -1.3326960802078247, + "logps/rejected": -13.587608337402344, + "loss": 2.9357, + "nll_loss": 2.917177677154541, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13326960802078247, + "rewards/margins": 1.2254914045333862, + "rewards/rejected": -1.3587608337402344, + "step": 885 + }, + { + "epoch": 0.5511664074650078, + "grad_norm": 0.4010365605354309, + "learning_rate": 2.7850000000000003e-05, + "log_odds_chosen": 17.20555877685547, + "log_odds_ratio": -8.836646884446964e-06, + "logits/chosen": 0.32361698150634766, + "logits/rejected": 1.3668720722198486, + "logps/chosen": -1.1861801147460938, + "logps/rejected": -17.992464065551758, + "loss": 3.2253, + "nll_loss": 3.2252705097198486, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11861801147460938, + "rewards/margins": 1.6806282997131348, + "rewards/rejected": -1.7992461919784546, + "step": 886 + }, + { + "epoch": 0.5517884914463452, + "grad_norm": 0.5873351693153381, + "learning_rate": 2.7825000000000002e-05, + "log_odds_chosen": 8.508808135986328, + "log_odds_ratio": -0.3058502972126007, + "logits/chosen": 0.2773277759552002, + "logits/rejected": 0.7884806394577026, + "logps/chosen": -1.3308159112930298, + "logps/rejected": -9.67809009552002, + "loss": 2.7651, + "nll_loss": 2.7345235347747803, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1330816000699997, + "rewards/margins": 0.83472740650177, + "rewards/rejected": -0.9678090214729309, + "step": 887 + }, + { + "epoch": 0.5524105754276827, + "grad_norm": 22.026321411132812, + "learning_rate": 2.7800000000000005e-05, + "log_odds_chosen": 13.509963035583496, + "log_odds_ratio": -0.002110434928908944, + "logits/chosen": 0.3400443196296692, + "logits/rejected": 1.213066577911377, + "logps/chosen": -1.1196520328521729, + "logps/rejected": -13.974618911743164, + "loss": 2.9615, + "nll_loss": 2.961289882659912, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11196519434452057, + "rewards/margins": 1.2854965925216675, + "rewards/rejected": -1.3974618911743164, + "step": 888 + }, + { + "epoch": 0.5530326594090202, + "grad_norm": 3.287785291671753, + "learning_rate": 2.7775e-05, + "log_odds_chosen": 10.978760719299316, + "log_odds_ratio": -0.5777610540390015, + "logits/chosen": 0.35481563210487366, + "logits/rejected": 0.9291172027587891, + "logps/chosen": -2.050435781478882, + "logps/rejected": -12.973872184753418, + "loss": 3.2977, + "nll_loss": 3.2398900985717773, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.20504358410835266, + "rewards/margins": 1.0923435688018799, + "rewards/rejected": -1.2973871231079102, + "step": 889 + }, + { + "epoch": 0.5536547433903577, + "grad_norm": 0.43079885840415955, + "learning_rate": 2.7750000000000004e-05, + "log_odds_chosen": 13.530800819396973, + "log_odds_ratio": -0.01729394681751728, + "logits/chosen": 0.22219036519527435, + "logits/rejected": 0.8625215888023376, + "logps/chosen": -1.6603683233261108, + "logps/rejected": -14.933816909790039, + "loss": 3.3577, + "nll_loss": 3.3560125827789307, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16603684425354004, + "rewards/margins": 1.3273450136184692, + "rewards/rejected": -1.4933817386627197, + "step": 890 + }, + { + "epoch": 0.5542768273716951, + "grad_norm": 0.4820476770401001, + "learning_rate": 2.7725e-05, + "log_odds_chosen": 7.27010440826416, + "log_odds_ratio": -0.21835504472255707, + "logits/chosen": 0.2366558313369751, + "logits/rejected": 0.615390419960022, + "logps/chosen": -1.1921238899230957, + "logps/rejected": -8.159709930419922, + "loss": 3.1871, + "nll_loss": 3.1652331352233887, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11921238899230957, + "rewards/margins": 0.6967586278915405, + "rewards/rejected": -0.8159710168838501, + "step": 891 + }, + { + "epoch": 0.5548989113530327, + "grad_norm": 0.4553963840007782, + "learning_rate": 2.7700000000000002e-05, + "log_odds_chosen": 9.985837936401367, + "log_odds_ratio": -0.2571437954902649, + "logits/chosen": 0.23121318221092224, + "logits/rejected": 0.8296804428100586, + "logps/chosen": -1.2417988777160645, + "logps/rejected": -11.013862609863281, + "loss": 2.8727, + "nll_loss": 2.846977710723877, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1241798847913742, + "rewards/margins": 0.977206289768219, + "rewards/rejected": -1.1013861894607544, + "step": 892 + }, + { + "epoch": 0.5555209953343702, + "grad_norm": 0.5255298018455505, + "learning_rate": 2.7675000000000002e-05, + "log_odds_chosen": 13.86887264251709, + "log_odds_ratio": -0.19372327625751495, + "logits/chosen": 0.30388227105140686, + "logits/rejected": 1.4697858095169067, + "logps/chosen": -1.3105946779251099, + "logps/rejected": -14.92329216003418, + "loss": 2.4972, + "nll_loss": 2.47778058052063, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.131059467792511, + "rewards/margins": 1.3612698316574097, + "rewards/rejected": -1.4923293590545654, + "step": 893 + }, + { + "epoch": 0.5561430793157076, + "grad_norm": 0.5121700763702393, + "learning_rate": 2.7650000000000005e-05, + "log_odds_chosen": 3.2167305946350098, + "log_odds_ratio": -0.32042694091796875, + "logits/chosen": 0.25697678327560425, + "logits/rejected": 0.38786739110946655, + "logps/chosen": -1.0599229335784912, + "logps/rejected": -3.9423530101776123, + "loss": 2.8183, + "nll_loss": 2.786297082901001, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10599227994680405, + "rewards/margins": 0.28824299573898315, + "rewards/rejected": -0.3942352831363678, + "step": 894 + }, + { + "epoch": 0.5567651632970451, + "grad_norm": 5.089456558227539, + "learning_rate": 2.7625e-05, + "log_odds_chosen": 8.74205493927002, + "log_odds_ratio": -0.15869277715682983, + "logits/chosen": 0.32882875204086304, + "logits/rejected": 1.0555585622787476, + "logps/chosen": -1.5479052066802979, + "logps/rejected": -9.963598251342773, + "loss": 2.7022, + "nll_loss": 2.686356782913208, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15479052066802979, + "rewards/margins": 0.8415693044662476, + "rewards/rejected": -0.9963598847389221, + "step": 895 + }, + { + "epoch": 0.5573872472783826, + "grad_norm": 0.523443341255188, + "learning_rate": 2.7600000000000003e-05, + "log_odds_chosen": 15.575971603393555, + "log_odds_ratio": -0.08751137554645538, + "logits/chosen": 0.38994699716567993, + "logits/rejected": 1.6484946012496948, + "logps/chosen": -1.126055121421814, + "logps/rejected": -16.346158981323242, + "loss": 2.8432, + "nll_loss": 2.834432363510132, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1126055121421814, + "rewards/margins": 1.5220104455947876, + "rewards/rejected": -1.6346158981323242, + "step": 896 + }, + { + "epoch": 0.5580093312597201, + "grad_norm": 0.43732067942619324, + "learning_rate": 2.7575e-05, + "log_odds_chosen": 12.99947452545166, + "log_odds_ratio": -0.22496089339256287, + "logits/chosen": 0.3272874653339386, + "logits/rejected": 0.942588210105896, + "logps/chosen": -1.0477204322814941, + "logps/rejected": -13.669449806213379, + "loss": 3.2135, + "nll_loss": 3.191025495529175, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10477204620838165, + "rewards/margins": 1.2621729373931885, + "rewards/rejected": -1.3669450283050537, + "step": 897 + }, + { + "epoch": 0.5586314152410575, + "grad_norm": 0.6751424670219421, + "learning_rate": 2.7550000000000002e-05, + "log_odds_chosen": 8.845113754272461, + "log_odds_ratio": -0.1459217518568039, + "logits/chosen": 0.12097790837287903, + "logits/rejected": 0.7129504084587097, + "logps/chosen": -0.9098332524299622, + "logps/rejected": -9.02649974822998, + "loss": 2.4552, + "nll_loss": 2.440615653991699, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0909833312034607, + "rewards/margins": 0.81166672706604, + "rewards/rejected": -0.9026500582695007, + "step": 898 + }, + { + "epoch": 0.559253499222395, + "grad_norm": 0.5536679029464722, + "learning_rate": 2.7525e-05, + "log_odds_chosen": 10.608327865600586, + "log_odds_ratio": -0.01897694170475006, + "logits/chosen": 0.22658635675907135, + "logits/rejected": 0.7577848434448242, + "logps/chosen": -1.0816148519515991, + "logps/rejected": -11.110265731811523, + "loss": 2.8688, + "nll_loss": 2.866927146911621, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10816149413585663, + "rewards/margins": 1.0028650760650635, + "rewards/rejected": -1.111026644706726, + "step": 899 + }, + { + "epoch": 0.5598755832037325, + "grad_norm": 0.43533676862716675, + "learning_rate": 2.7500000000000004e-05, + "log_odds_chosen": 7.546494483947754, + "log_odds_ratio": -0.1429726481437683, + "logits/chosen": 0.20976758003234863, + "logits/rejected": 0.4699644446372986, + "logps/chosen": -1.0662420988082886, + "logps/rejected": -8.181591033935547, + "loss": 3.1895, + "nll_loss": 3.1751976013183594, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10662420839071274, + "rewards/margins": 0.7115349769592285, + "rewards/rejected": -0.8181591629981995, + "step": 900 + }, + { + "epoch": 0.56049766718507, + "grad_norm": 0.5101000666618347, + "learning_rate": 2.7475e-05, + "log_odds_chosen": 9.346063613891602, + "log_odds_ratio": -0.16607795655727386, + "logits/chosen": 0.22395896911621094, + "logits/rejected": 0.6142349243164062, + "logps/chosen": -1.1865246295928955, + "logps/rejected": -9.847481727600098, + "loss": 3.1556, + "nll_loss": 3.1389565467834473, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11865246295928955, + "rewards/margins": 0.8660956621170044, + "rewards/rejected": -0.9847480654716492, + "step": 901 + }, + { + "epoch": 0.5611197511664074, + "grad_norm": 0.44190743565559387, + "learning_rate": 2.7450000000000003e-05, + "log_odds_chosen": 7.044838905334473, + "log_odds_ratio": -0.14595672488212585, + "logits/chosen": 0.1050267368555069, + "logits/rejected": 0.3443432152271271, + "logps/chosen": -1.0475255250930786, + "logps/rejected": -7.505648612976074, + "loss": 3.0451, + "nll_loss": 3.030492067337036, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1047525554895401, + "rewards/margins": 0.6458122730255127, + "rewards/rejected": -0.7505648136138916, + "step": 902 + }, + { + "epoch": 0.5617418351477449, + "grad_norm": 0.3499910235404968, + "learning_rate": 2.7425e-05, + "log_odds_chosen": 7.8855743408203125, + "log_odds_ratio": -0.2606319189071655, + "logits/chosen": 0.29135677218437195, + "logits/rejected": 0.577876091003418, + "logps/chosen": -0.9390688538551331, + "logps/rejected": -8.381467819213867, + "loss": 3.7401, + "nll_loss": 3.7140378952026367, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09390688687562943, + "rewards/margins": 0.7442399263381958, + "rewards/rejected": -0.8381468057632446, + "step": 903 + }, + { + "epoch": 0.5623639191290825, + "grad_norm": 0.5649048089981079, + "learning_rate": 2.7400000000000002e-05, + "log_odds_chosen": 8.967251777648926, + "log_odds_ratio": -0.03768826648592949, + "logits/chosen": 0.2890816330909729, + "logits/rejected": 0.7596768140792847, + "logps/chosen": -0.9239016175270081, + "logps/rejected": -9.305893898010254, + "loss": 3.2545, + "nll_loss": 3.2506906986236572, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09239016473293304, + "rewards/margins": 0.838199257850647, + "rewards/rejected": -0.9305894374847412, + "step": 904 + }, + { + "epoch": 0.5629860031104199, + "grad_norm": 0.45023804903030396, + "learning_rate": 2.7375e-05, + "log_odds_chosen": 6.16334342956543, + "log_odds_ratio": -0.2813335359096527, + "logits/chosen": 0.16055354475975037, + "logits/rejected": 0.49607011675834656, + "logps/chosen": -1.1517943143844604, + "logps/rejected": -6.91542911529541, + "loss": 2.9652, + "nll_loss": 2.9370510578155518, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11517943441867828, + "rewards/margins": 0.5763634443283081, + "rewards/rejected": -0.6915428638458252, + "step": 905 + }, + { + "epoch": 0.5636080870917574, + "grad_norm": 0.4151860475540161, + "learning_rate": 2.7350000000000004e-05, + "log_odds_chosen": 5.84874153137207, + "log_odds_ratio": -0.30869588255882263, + "logits/chosen": 0.17200124263763428, + "logits/rejected": 0.48454806208610535, + "logps/chosen": -1.2069495916366577, + "logps/rejected": -6.622710227966309, + "loss": 3.2825, + "nll_loss": 3.2516021728515625, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12069497257471085, + "rewards/margins": 0.5415760278701782, + "rewards/rejected": -0.6622710227966309, + "step": 906 + }, + { + "epoch": 0.5642301710730949, + "grad_norm": 0.3901362717151642, + "learning_rate": 2.7325e-05, + "log_odds_chosen": 8.91002368927002, + "log_odds_ratio": -0.09111232310533524, + "logits/chosen": 0.22789613902568817, + "logits/rejected": 0.7879712581634521, + "logps/chosen": -1.3444089889526367, + "logps/rejected": -9.875727653503418, + "loss": 3.2262, + "nll_loss": 3.2170474529266357, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13444089889526367, + "rewards/margins": 0.8531317710876465, + "rewards/rejected": -0.9875727295875549, + "step": 907 + }, + { + "epoch": 0.5648522550544324, + "grad_norm": 0.4596397280693054, + "learning_rate": 2.7300000000000003e-05, + "log_odds_chosen": 6.89910888671875, + "log_odds_ratio": -0.14412494003772736, + "logits/chosen": 0.2593741714954376, + "logits/rejected": 0.8184172511100769, + "logps/chosen": -1.1923887729644775, + "logps/rejected": -7.755105018615723, + "loss": 2.9366, + "nll_loss": 2.9221863746643066, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11923889070749283, + "rewards/margins": 0.6562716960906982, + "rewards/rejected": -0.7755105495452881, + "step": 908 + }, + { + "epoch": 0.5654743390357698, + "grad_norm": 0.5999178290367126, + "learning_rate": 2.7275e-05, + "log_odds_chosen": 11.099242210388184, + "log_odds_ratio": -0.17223107814788818, + "logits/chosen": 0.16461758315563202, + "logits/rejected": 0.345947265625, + "logps/chosen": -1.1326367855072021, + "logps/rejected": -11.848851203918457, + "loss": 3.0254, + "nll_loss": 3.008146047592163, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11326368153095245, + "rewards/margins": 1.0716214179992676, + "rewards/rejected": -1.1848851442337036, + "step": 909 + }, + { + "epoch": 0.5660964230171073, + "grad_norm": 0.37448394298553467, + "learning_rate": 2.725e-05, + "log_odds_chosen": 5.122501850128174, + "log_odds_ratio": -0.26253485679626465, + "logits/chosen": 0.1458125114440918, + "logits/rejected": 0.38186800479888916, + "logps/chosen": -0.9692260026931763, + "logps/rejected": -5.624997138977051, + "loss": 2.946, + "nll_loss": 2.9197349548339844, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09692259877920151, + "rewards/margins": 0.4655771255493164, + "rewards/rejected": -0.5624997019767761, + "step": 910 + }, + { + "epoch": 0.5667185069984448, + "grad_norm": 0.46044209599494934, + "learning_rate": 2.7225e-05, + "log_odds_chosen": 6.9234795570373535, + "log_odds_ratio": -0.21885745227336884, + "logits/chosen": 0.09006873518228531, + "logits/rejected": 0.40548866987228394, + "logps/chosen": -0.9413871169090271, + "logps/rejected": -7.406796455383301, + "loss": 2.6326, + "nll_loss": 2.6107070446014404, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09413871169090271, + "rewards/margins": 0.6465408802032471, + "rewards/rejected": -0.7406796216964722, + "step": 911 + }, + { + "epoch": 0.5673405909797823, + "grad_norm": 0.48035767674446106, + "learning_rate": 2.7200000000000004e-05, + "log_odds_chosen": 9.26119613647461, + "log_odds_ratio": -0.17187556624412537, + "logits/chosen": 0.13499784469604492, + "logits/rejected": 0.6166836619377136, + "logps/chosen": -1.251449704170227, + "logps/rejected": -10.169729232788086, + "loss": 2.6754, + "nll_loss": 2.658219814300537, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12514497339725494, + "rewards/margins": 0.8918280005455017, + "rewards/rejected": -1.0169728994369507, + "step": 912 + }, + { + "epoch": 0.5679626749611197, + "grad_norm": 0.47506648302078247, + "learning_rate": 2.7175e-05, + "log_odds_chosen": 10.130144119262695, + "log_odds_ratio": -0.04994361102581024, + "logits/chosen": 0.17378772795200348, + "logits/rejected": 0.629872739315033, + "logps/chosen": -0.772241473197937, + "logps/rejected": -10.029999732971191, + "loss": 2.8842, + "nll_loss": 2.8792366981506348, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07722415030002594, + "rewards/margins": 0.9257758855819702, + "rewards/rejected": -1.003000020980835, + "step": 913 + }, + { + "epoch": 0.5685847589424572, + "grad_norm": 0.48764681816101074, + "learning_rate": 2.7150000000000003e-05, + "log_odds_chosen": 5.916272163391113, + "log_odds_ratio": -0.25303128361701965, + "logits/chosen": 0.1644158661365509, + "logits/rejected": 0.30059051513671875, + "logps/chosen": -1.1256078481674194, + "logps/rejected": -6.709453105926514, + "loss": 3.0249, + "nll_loss": 2.9995951652526855, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11256079375743866, + "rewards/margins": 0.5583845376968384, + "rewards/rejected": -0.6709453463554382, + "step": 914 + }, + { + "epoch": 0.5692068429237948, + "grad_norm": 0.5090453028678894, + "learning_rate": 2.7125000000000002e-05, + "log_odds_chosen": 9.453378677368164, + "log_odds_ratio": -0.11234141886234283, + "logits/chosen": 0.2241164743900299, + "logits/rejected": 0.9943174719810486, + "logps/chosen": -1.0483677387237549, + "logps/rejected": -9.902132987976074, + "loss": 2.8647, + "nll_loss": 2.8534181118011475, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10483676195144653, + "rewards/margins": 0.8853765726089478, + "rewards/rejected": -0.9902133941650391, + "step": 915 + }, + { + "epoch": 0.5698289269051322, + "grad_norm": 0.3839872181415558, + "learning_rate": 2.7100000000000005e-05, + "log_odds_chosen": 12.735099792480469, + "log_odds_ratio": -0.07314764708280563, + "logits/chosen": 0.3284153938293457, + "logits/rejected": 1.1309449672698975, + "logps/chosen": -0.8413907289505005, + "logps/rejected": -12.652066230773926, + "loss": 3.1403, + "nll_loss": 3.132990837097168, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08413907140493393, + "rewards/margins": 1.1810675859451294, + "rewards/rejected": -1.2652066946029663, + "step": 916 + }, + { + "epoch": 0.5704510108864697, + "grad_norm": 0.7262453436851501, + "learning_rate": 2.7075e-05, + "log_odds_chosen": 10.773283958435059, + "log_odds_ratio": -0.17366476356983185, + "logits/chosen": 0.27300554513931274, + "logits/rejected": 0.6795448064804077, + "logps/chosen": -1.1927194595336914, + "logps/rejected": -11.570327758789062, + "loss": 2.9818, + "nll_loss": 2.964430570602417, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11927194148302078, + "rewards/margins": 1.037760853767395, + "rewards/rejected": -1.15703284740448, + "step": 917 + }, + { + "epoch": 0.5710730948678071, + "grad_norm": 0.4268437922000885, + "learning_rate": 2.7050000000000004e-05, + "log_odds_chosen": 12.669416427612305, + "log_odds_ratio": -0.18929001688957214, + "logits/chosen": 0.23210585117340088, + "logits/rejected": 0.7328687906265259, + "logps/chosen": -0.9498953223228455, + "logps/rejected": -13.114092826843262, + "loss": 3.6412, + "nll_loss": 3.622225761413574, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09498953074216843, + "rewards/margins": 1.2164198160171509, + "rewards/rejected": -1.3114093542099, + "step": 918 + }, + { + "epoch": 0.5716951788491447, + "grad_norm": 0.4433814585208893, + "learning_rate": 2.7025e-05, + "log_odds_chosen": 10.184727668762207, + "log_odds_ratio": -0.025781212374567986, + "logits/chosen": 0.17551684379577637, + "logits/rejected": 1.0827550888061523, + "logps/chosen": -1.2214159965515137, + "logps/rejected": -11.030315399169922, + "loss": 2.7723, + "nll_loss": 2.769714832305908, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12214161455631256, + "rewards/margins": 0.9808899760246277, + "rewards/rejected": -1.1030315160751343, + "step": 919 + }, + { + "epoch": 0.5723172628304821, + "grad_norm": 0.48769259452819824, + "learning_rate": 2.7000000000000002e-05, + "log_odds_chosen": 9.125656127929688, + "log_odds_ratio": -0.26961836218833923, + "logits/chosen": 0.1965629905462265, + "logits/rejected": 0.3319164514541626, + "logps/chosen": -1.3702173233032227, + "logps/rejected": -10.158577919006348, + "loss": 3.5997, + "nll_loss": 3.5727407932281494, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1370217353105545, + "rewards/margins": 0.8788360357284546, + "rewards/rejected": -1.0158576965332031, + "step": 920 + }, + { + "epoch": 0.5729393468118196, + "grad_norm": 0.48434844613075256, + "learning_rate": 2.6975000000000002e-05, + "log_odds_chosen": 13.56881332397461, + "log_odds_ratio": -0.0026917767245322466, + "logits/chosen": 0.2141491025686264, + "logits/rejected": 1.3020198345184326, + "logps/chosen": -1.3336665630340576, + "logps/rejected": -14.552571296691895, + "loss": 2.7108, + "nll_loss": 2.7105112075805664, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1333666741847992, + "rewards/margins": 1.3218905925750732, + "rewards/rejected": -1.4552571773529053, + "step": 921 + }, + { + "epoch": 0.573561430793157, + "grad_norm": 0.5728157162666321, + "learning_rate": 2.6950000000000005e-05, + "log_odds_chosen": 10.488691329956055, + "log_odds_ratio": -0.23412871360778809, + "logits/chosen": 0.2155309021472931, + "logits/rejected": 0.6558471322059631, + "logps/chosen": -1.1757211685180664, + "logps/rejected": -11.354267120361328, + "loss": 3.3212, + "nll_loss": 3.2977969646453857, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.117572121322155, + "rewards/margins": 1.0178545713424683, + "rewards/rejected": -1.135426640510559, + "step": 922 + }, + { + "epoch": 0.5741835147744946, + "grad_norm": 0.472119003534317, + "learning_rate": 2.6925e-05, + "log_odds_chosen": 8.171377182006836, + "log_odds_ratio": -0.22905880212783813, + "logits/chosen": 0.1736389398574829, + "logits/rejected": 0.8877742290496826, + "logps/chosen": -1.1164681911468506, + "logps/rejected": -8.963059425354004, + "loss": 2.8084, + "nll_loss": 2.785543203353882, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11164682358503342, + "rewards/margins": 0.7846590876579285, + "rewards/rejected": -0.8963059186935425, + "step": 923 + }, + { + "epoch": 0.574805598755832, + "grad_norm": 1.098946213722229, + "learning_rate": 2.6900000000000003e-05, + "log_odds_chosen": 8.532747268676758, + "log_odds_ratio": -0.16291804611682892, + "logits/chosen": 0.26898065209388733, + "logits/rejected": 0.7751161456108093, + "logps/chosen": -1.2293330430984497, + "logps/rejected": -9.393357276916504, + "loss": 3.3784, + "nll_loss": 3.3621580600738525, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12293331325054169, + "rewards/margins": 0.8164024353027344, + "rewards/rejected": -0.9393357634544373, + "step": 924 + }, + { + "epoch": 0.5754276827371695, + "grad_norm": 0.661605715751648, + "learning_rate": 2.6875e-05, + "log_odds_chosen": 16.35256576538086, + "log_odds_ratio": -0.1963188201189041, + "logits/chosen": 0.1324017345905304, + "logits/rejected": 0.8213398456573486, + "logps/chosen": -1.7915053367614746, + "logps/rejected": -17.91004180908203, + "loss": 3.0766, + "nll_loss": 3.0570147037506104, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1791505366563797, + "rewards/margins": 1.611853837966919, + "rewards/rejected": -1.7910041809082031, + "step": 925 + }, + { + "epoch": 0.576049766718507, + "grad_norm": 0.3944947123527527, + "learning_rate": 2.6850000000000002e-05, + "log_odds_chosen": 19.438589096069336, + "log_odds_ratio": -0.09257218986749649, + "logits/chosen": 0.11398239433765411, + "logits/rejected": 0.9922657608985901, + "logps/chosen": -0.9387992024421692, + "logps/rejected": -19.83526039123535, + "loss": 2.8709, + "nll_loss": 2.8616538047790527, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09387992322444916, + "rewards/margins": 1.889646291732788, + "rewards/rejected": -1.9835262298583984, + "step": 926 + }, + { + "epoch": 0.5766718506998445, + "grad_norm": 0.3849738538265228, + "learning_rate": 2.6825e-05, + "log_odds_chosen": 16.61699867248535, + "log_odds_ratio": -0.03179116174578667, + "logits/chosen": 0.2330043613910675, + "logits/rejected": 1.0248134136199951, + "logps/chosen": -1.3797637224197388, + "logps/rejected": -17.69917106628418, + "loss": 3.4806, + "nll_loss": 3.4774537086486816, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13797636330127716, + "rewards/margins": 1.6319406032562256, + "rewards/rejected": -1.7699170112609863, + "step": 927 + }, + { + "epoch": 0.577293934681182, + "grad_norm": 0.5158620476722717, + "learning_rate": 2.6800000000000004e-05, + "log_odds_chosen": 20.5090274810791, + "log_odds_ratio": -0.0825430378317833, + "logits/chosen": 0.11498251557350159, + "logits/rejected": 0.9717357754707336, + "logps/chosen": -0.903712272644043, + "logps/rejected": -20.81587028503418, + "loss": 2.7439, + "nll_loss": 2.735637664794922, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09037123620510101, + "rewards/margins": 1.9912158250808716, + "rewards/rejected": -2.081587314605713, + "step": 928 + }, + { + "epoch": 0.5779160186625194, + "grad_norm": 0.4215962886810303, + "learning_rate": 2.6775e-05, + "log_odds_chosen": 22.170780181884766, + "log_odds_ratio": 0.0, + "logits/chosen": 0.3199015259742737, + "logits/rejected": 1.854710340499878, + "logps/chosen": -1.3474838733673096, + "logps/rejected": -23.19628143310547, + "loss": 3.4444, + "nll_loss": 3.444409132003784, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13474836945533752, + "rewards/margins": 2.184879779815674, + "rewards/rejected": -2.3196280002593994, + "step": 929 + }, + { + "epoch": 0.578538102643857, + "grad_norm": 0.7233343720436096, + "learning_rate": 2.6750000000000003e-05, + "log_odds_chosen": 13.190216064453125, + "log_odds_ratio": -0.1239352896809578, + "logits/chosen": -0.007729051634669304, + "logits/rejected": 0.6695624589920044, + "logps/chosen": -1.3063056468963623, + "logps/rejected": -14.166091918945312, + "loss": 2.7662, + "nll_loss": 2.7538347244262695, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13063055276870728, + "rewards/margins": 1.2859786748886108, + "rewards/rejected": -1.416609287261963, + "step": 930 + }, + { + "epoch": 0.5791601866251944, + "grad_norm": 0.5069626569747925, + "learning_rate": 2.6725e-05, + "log_odds_chosen": 10.079081535339355, + "log_odds_ratio": -0.2044445276260376, + "logits/chosen": 0.21780771017074585, + "logits/rejected": 0.6870952844619751, + "logps/chosen": -1.3345181941986084, + "logps/rejected": -11.053346633911133, + "loss": 3.3541, + "nll_loss": 3.333698272705078, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13345181941986084, + "rewards/margins": 0.9718827605247498, + "rewards/rejected": -1.1053345203399658, + "step": 931 + }, + { + "epoch": 0.5797822706065319, + "grad_norm": 0.7521727681159973, + "learning_rate": 2.6700000000000002e-05, + "log_odds_chosen": 10.476853370666504, + "log_odds_ratio": -0.2420932799577713, + "logits/chosen": 0.17987507581710815, + "logits/rejected": 0.8727846741676331, + "logps/chosen": -1.2537364959716797, + "logps/rejected": -11.50350570678711, + "loss": 3.1104, + "nll_loss": 3.0861494541168213, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12537366151809692, + "rewards/margins": 1.0249770879745483, + "rewards/rejected": -1.1503506898880005, + "step": 932 + }, + { + "epoch": 0.5804043545878693, + "grad_norm": 0.5356715321540833, + "learning_rate": 2.6675e-05, + "log_odds_chosen": 14.187935829162598, + "log_odds_ratio": -0.24278056621551514, + "logits/chosen": 0.17469918727874756, + "logits/rejected": 1.1708053350448608, + "logps/chosen": -1.0131219625473022, + "logps/rejected": -14.83353328704834, + "loss": 2.2884, + "nll_loss": 2.26407527923584, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10131219774484634, + "rewards/margins": 1.3820412158966064, + "rewards/rejected": -1.4833533763885498, + "step": 933 + }, + { + "epoch": 0.5810264385692069, + "grad_norm": 0.5571019649505615, + "learning_rate": 2.6650000000000004e-05, + "log_odds_chosen": 9.943033218383789, + "log_odds_ratio": -0.26776123046875, + "logits/chosen": 0.265704870223999, + "logits/rejected": 0.9763750433921814, + "logps/chosen": -1.1288625001907349, + "logps/rejected": -10.797094345092773, + "loss": 3.5617, + "nll_loss": 3.534961700439453, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11288625001907349, + "rewards/margins": 0.966823160648346, + "rewards/rejected": -1.0797094106674194, + "step": 934 + }, + { + "epoch": 0.5816485225505443, + "grad_norm": 0.45613187551498413, + "learning_rate": 2.6625e-05, + "log_odds_chosen": 8.923871040344238, + "log_odds_ratio": -0.25769421458244324, + "logits/chosen": 0.07586418092250824, + "logits/rejected": 0.34221017360687256, + "logps/chosen": -1.0070983171463013, + "logps/rejected": -9.35234260559082, + "loss": 3.1174, + "nll_loss": 3.0916061401367188, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10070984065532684, + "rewards/margins": 0.8345244526863098, + "rewards/rejected": -0.9352343082427979, + "step": 935 + }, + { + "epoch": 0.5822706065318818, + "grad_norm": 0.44240835309028625, + "learning_rate": 2.6600000000000003e-05, + "log_odds_chosen": 20.093387603759766, + "log_odds_ratio": -0.11705049872398376, + "logits/chosen": 0.2263740599155426, + "logits/rejected": 1.405846357345581, + "logps/chosen": -1.076794147491455, + "logps/rejected": -20.625396728515625, + "loss": 2.8483, + "nll_loss": 2.8366036415100098, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10767941921949387, + "rewards/margins": 1.9548602104187012, + "rewards/rejected": -2.062539577484131, + "step": 936 + }, + { + "epoch": 0.5828926905132192, + "grad_norm": 0.39377760887145996, + "learning_rate": 2.6575e-05, + "log_odds_chosen": 14.986661911010742, + "log_odds_ratio": -0.27674102783203125, + "logits/chosen": 0.1652100682258606, + "logits/rejected": 0.9994176626205444, + "logps/chosen": -1.0255934000015259, + "logps/rejected": -15.587301254272461, + "loss": 2.8296, + "nll_loss": 2.801974296569824, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10255934298038483, + "rewards/margins": 1.4561707973480225, + "rewards/rejected": -1.558730125427246, + "step": 937 + }, + { + "epoch": 0.5835147744945568, + "grad_norm": 4.184680938720703, + "learning_rate": 2.655e-05, + "log_odds_chosen": 11.676606178283691, + "log_odds_ratio": -0.12282869964838028, + "logits/chosen": 0.21672847867012024, + "logits/rejected": 0.7535300254821777, + "logps/chosen": -1.5664196014404297, + "logps/rejected": -12.69096851348877, + "loss": 3.4853, + "nll_loss": 3.4730653762817383, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15664197504520416, + "rewards/margins": 1.112454891204834, + "rewards/rejected": -1.269096851348877, + "step": 938 + }, + { + "epoch": 0.5841368584758942, + "grad_norm": 0.5880439877510071, + "learning_rate": 2.6525e-05, + "log_odds_chosen": 10.81521224975586, + "log_odds_ratio": -0.47919604182243347, + "logits/chosen": 0.1771206557750702, + "logits/rejected": 0.8535869121551514, + "logps/chosen": -1.2247461080551147, + "logps/rejected": -11.720307350158691, + "loss": 2.8229, + "nll_loss": 2.774963855743408, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12247461825609207, + "rewards/margins": 1.0495561361312866, + "rewards/rejected": -1.1720308065414429, + "step": 939 + }, + { + "epoch": 0.5847589424572317, + "grad_norm": 2.263173818588257, + "learning_rate": 2.6500000000000004e-05, + "log_odds_chosen": 14.880687713623047, + "log_odds_ratio": -0.04369397461414337, + "logits/chosen": 0.17089928686618805, + "logits/rejected": 0.7093451023101807, + "logps/chosen": -1.288264274597168, + "logps/rejected": -15.811230659484863, + "loss": 3.2519, + "nll_loss": 3.247570276260376, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12882643938064575, + "rewards/margins": 1.4522968530654907, + "rewards/rejected": -1.5811231136322021, + "step": 940 + }, + { + "epoch": 0.5853810264385692, + "grad_norm": 0.6679919362068176, + "learning_rate": 2.6475e-05, + "log_odds_chosen": 13.88052749633789, + "log_odds_ratio": -0.17339672148227692, + "logits/chosen": 0.2502378821372986, + "logits/rejected": 1.128913402557373, + "logps/chosen": -1.2840638160705566, + "logps/rejected": -14.904745101928711, + "loss": 2.6963, + "nll_loss": 2.67893648147583, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1284063756465912, + "rewards/margins": 1.3620681762695312, + "rewards/rejected": -1.4904745817184448, + "step": 941 + }, + { + "epoch": 0.5860031104199067, + "grad_norm": 1.2314687967300415, + "learning_rate": 2.6450000000000003e-05, + "log_odds_chosen": 18.171859741210938, + "log_odds_ratio": -0.1047465056180954, + "logits/chosen": 0.36809712648391724, + "logits/rejected": 1.2636301517486572, + "logps/chosen": -1.0214661359786987, + "logps/rejected": -18.68871307373047, + "loss": 3.5026, + "nll_loss": 3.492130756378174, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10214661061763763, + "rewards/margins": 1.7667248249053955, + "rewards/rejected": -1.8688714504241943, + "step": 942 + }, + { + "epoch": 0.5866251944012442, + "grad_norm": 0.5520533919334412, + "learning_rate": 2.6425e-05, + "log_odds_chosen": 9.532363891601562, + "log_odds_ratio": -0.033491168171167374, + "logits/chosen": 0.3422502279281616, + "logits/rejected": 0.8718456029891968, + "logps/chosen": -1.18587327003479, + "logps/rejected": -10.302650451660156, + "loss": 3.2041, + "nll_loss": 3.20070743560791, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11858732998371124, + "rewards/margins": 0.9116777181625366, + "rewards/rejected": -1.0302650928497314, + "step": 943 + }, + { + "epoch": 0.5872472783825816, + "grad_norm": 0.5288044810295105, + "learning_rate": 2.64e-05, + "log_odds_chosen": 15.668525695800781, + "log_odds_ratio": -0.27726009488105774, + "logits/chosen": 0.2326730489730835, + "logits/rejected": 0.898745059967041, + "logps/chosen": -1.0137486457824707, + "logps/rejected": -16.33362579345703, + "loss": 2.6635, + "nll_loss": 2.6358118057250977, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10137484967708588, + "rewards/margins": 1.5319877862930298, + "rewards/rejected": -1.6333625316619873, + "step": 944 + }, + { + "epoch": 0.5878693623639192, + "grad_norm": 0.46301695704460144, + "learning_rate": 2.6375e-05, + "log_odds_chosen": 17.152551651000977, + "log_odds_ratio": -0.07121972739696503, + "logits/chosen": 0.4616335928440094, + "logits/rejected": 1.5514743328094482, + "logps/chosen": -1.638116478919983, + "logps/rejected": -18.526134490966797, + "loss": 3.2308, + "nll_loss": 3.22369647026062, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16381165385246277, + "rewards/margins": 1.6888020038604736, + "rewards/rejected": -1.8526135683059692, + "step": 945 + }, + { + "epoch": 0.5884914463452566, + "grad_norm": 0.4805068075656891, + "learning_rate": 2.6350000000000004e-05, + "log_odds_chosen": 10.249034881591797, + "log_odds_ratio": -0.19067715108394623, + "logits/chosen": 0.2742692232131958, + "logits/rejected": 0.5140730142593384, + "logps/chosen": -1.0531798601150513, + "logps/rejected": -10.836760520935059, + "loss": 2.7838, + "nll_loss": 2.764707565307617, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10531798750162125, + "rewards/margins": 0.9783580899238586, + "rewards/rejected": -1.0836760997772217, + "step": 946 + }, + { + "epoch": 0.5891135303265941, + "grad_norm": 0.4442595839500427, + "learning_rate": 2.6325e-05, + "log_odds_chosen": 16.763885498046875, + "log_odds_ratio": -0.14290139079093933, + "logits/chosen": 0.3244515061378479, + "logits/rejected": 1.0541630983352661, + "logps/chosen": -0.9952157735824585, + "logps/rejected": -17.375211715698242, + "loss": 3.0868, + "nll_loss": 3.072525978088379, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09952157735824585, + "rewards/margins": 1.6379995346069336, + "rewards/rejected": -1.7375211715698242, + "step": 947 + }, + { + "epoch": 0.5897356143079315, + "grad_norm": 0.3361284136772156, + "learning_rate": 2.6300000000000002e-05, + "log_odds_chosen": 19.245708465576172, + "log_odds_ratio": -0.07403115928173065, + "logits/chosen": 0.31128478050231934, + "logits/rejected": 0.9080635905265808, + "logps/chosen": -0.8605557680130005, + "logps/rejected": -19.487812042236328, + "loss": 3.7087, + "nll_loss": 3.7012686729431152, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08605558425188065, + "rewards/margins": 1.8627257347106934, + "rewards/rejected": -1.9487813711166382, + "step": 948 + }, + { + "epoch": 0.5903576982892691, + "grad_norm": 0.5872551798820496, + "learning_rate": 2.6275e-05, + "log_odds_chosen": 8.34841251373291, + "log_odds_ratio": -0.3960355222225189, + "logits/chosen": 0.1396239548921585, + "logits/rejected": 0.6412820219993591, + "logps/chosen": -1.0153343677520752, + "logps/rejected": -9.043679237365723, + "loss": 2.4457, + "nll_loss": 2.4060676097869873, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.101533442735672, + "rewards/margins": 0.8028345108032227, + "rewards/rejected": -0.9043679237365723, + "step": 949 + }, + { + "epoch": 0.5909797822706065, + "grad_norm": 0.5597155094146729, + "learning_rate": 2.625e-05, + "log_odds_chosen": 17.4451847076416, + "log_odds_ratio": -0.0725177526473999, + "logits/chosen": 0.110418900847435, + "logits/rejected": 0.8833154439926147, + "logps/chosen": -0.8898366093635559, + "logps/rejected": -17.72234535217285, + "loss": 2.631, + "nll_loss": 2.623771905899048, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08898366987705231, + "rewards/margins": 1.683250904083252, + "rewards/rejected": -1.7722344398498535, + "step": 950 + }, + { + "epoch": 0.591601866251944, + "grad_norm": 0.7085988521575928, + "learning_rate": 2.6225e-05, + "log_odds_chosen": 8.395122528076172, + "log_odds_ratio": -0.36476755142211914, + "logits/chosen": 0.26892557740211487, + "logits/rejected": 0.6090400218963623, + "logps/chosen": -1.258937954902649, + "logps/rejected": -9.453425407409668, + "loss": 3.015, + "nll_loss": 2.9785232543945312, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12589380145072937, + "rewards/margins": 0.8194488286972046, + "rewards/rejected": -0.9453425407409668, + "step": 951 + }, + { + "epoch": 0.5922239502332814, + "grad_norm": 0.7648904323577881, + "learning_rate": 2.6200000000000003e-05, + "log_odds_chosen": 8.385675430297852, + "log_odds_ratio": -0.21214787662029266, + "logits/chosen": 0.18494370579719543, + "logits/rejected": 0.8852918148040771, + "logps/chosen": -1.3512593507766724, + "logps/rejected": -9.508600234985352, + "loss": 2.5537, + "nll_loss": 2.532467842102051, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13512593507766724, + "rewards/margins": 0.8157341480255127, + "rewards/rejected": -0.9508601427078247, + "step": 952 + }, + { + "epoch": 0.592846034214619, + "grad_norm": 0.4409056305885315, + "learning_rate": 2.6175e-05, + "log_odds_chosen": 19.01350212097168, + "log_odds_ratio": -7.912763976491988e-06, + "logits/chosen": 0.24696475267410278, + "logits/rejected": 1.9544768333435059, + "logps/chosen": -1.246018648147583, + "logps/rejected": -19.90366554260254, + "loss": 2.4758, + "nll_loss": 2.4758312702178955, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12460187077522278, + "rewards/margins": 1.8657647371292114, + "rewards/rejected": -1.9903665781021118, + "step": 953 + }, + { + "epoch": 0.5934681181959565, + "grad_norm": 0.5151073336601257, + "learning_rate": 2.6150000000000002e-05, + "log_odds_chosen": 10.778078079223633, + "log_odds_ratio": -0.29022905230522156, + "logits/chosen": 0.13743232190608978, + "logits/rejected": 0.6315884590148926, + "logps/chosen": -1.1233041286468506, + "logps/rejected": -11.37158203125, + "loss": 2.9005, + "nll_loss": 2.871509313583374, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11233042180538177, + "rewards/margins": 1.0248278379440308, + "rewards/rejected": -1.1371582746505737, + "step": 954 + }, + { + "epoch": 0.5940902021772939, + "grad_norm": 0.3522619307041168, + "learning_rate": 2.6124999999999998e-05, + "log_odds_chosen": 11.523155212402344, + "log_odds_ratio": -0.2107788622379303, + "logits/chosen": 0.3925516903400421, + "logits/rejected": 1.0518916845321655, + "logps/chosen": -1.0370612144470215, + "logps/rejected": -12.201189994812012, + "loss": 3.6713, + "nll_loss": 3.650195360183716, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10370612144470215, + "rewards/margins": 1.116412878036499, + "rewards/rejected": -1.2201189994812012, + "step": 955 + }, + { + "epoch": 0.5947122861586314, + "grad_norm": 0.5082216262817383, + "learning_rate": 2.61e-05, + "log_odds_chosen": 16.058555603027344, + "log_odds_ratio": -0.06356058269739151, + "logits/chosen": 0.17834536731243134, + "logits/rejected": 0.8007286190986633, + "logps/chosen": -1.3070101737976074, + "logps/rejected": -17.062992095947266, + "loss": 3.2413, + "nll_loss": 3.234963893890381, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13070102035999298, + "rewards/margins": 1.5755982398986816, + "rewards/rejected": -1.7062993049621582, + "step": 956 + }, + { + "epoch": 0.5953343701399689, + "grad_norm": 0.5702728629112244, + "learning_rate": 2.6075e-05, + "log_odds_chosen": 24.80580711364746, + "log_odds_ratio": -0.021674897521734238, + "logits/chosen": 0.2285652458667755, + "logits/rejected": 1.1431223154067993, + "logps/chosen": -1.0820637941360474, + "logps/rejected": -25.419315338134766, + "loss": 2.9305, + "nll_loss": 2.9282896518707275, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1082063838839531, + "rewards/margins": 2.433724880218506, + "rewards/rejected": -2.541931390762329, + "step": 957 + }, + { + "epoch": 0.5959564541213064, + "grad_norm": 1.0141100883483887, + "learning_rate": 2.6050000000000003e-05, + "log_odds_chosen": 16.26883316040039, + "log_odds_ratio": -0.3170120120048523, + "logits/chosen": 0.3057654798030853, + "logits/rejected": 1.2227212190628052, + "logps/chosen": -1.0155651569366455, + "logps/rejected": -16.994293212890625, + "loss": 2.2965, + "nll_loss": 2.264829158782959, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10155650973320007, + "rewards/margins": 1.5978727340698242, + "rewards/rejected": -1.6994292736053467, + "step": 958 + }, + { + "epoch": 0.5965785381026438, + "grad_norm": 0.489462673664093, + "learning_rate": 2.6025e-05, + "log_odds_chosen": 19.681560516357422, + "log_odds_ratio": -0.023856129497289658, + "logits/chosen": 0.2779539227485657, + "logits/rejected": 1.4224390983581543, + "logps/chosen": -1.0721073150634766, + "logps/rejected": -20.223628997802734, + "loss": 2.8642, + "nll_loss": 2.861823558807373, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10721074789762497, + "rewards/margins": 1.9151520729064941, + "rewards/rejected": -2.0223629474639893, + "step": 959 + }, + { + "epoch": 0.5972006220839814, + "grad_norm": 0.5690145492553711, + "learning_rate": 2.6000000000000002e-05, + "log_odds_chosen": 16.70014190673828, + "log_odds_ratio": -0.10952571779489517, + "logits/chosen": 0.26871487498283386, + "logits/rejected": 1.045303463935852, + "logps/chosen": -1.7916289567947388, + "logps/rejected": -18.297130584716797, + "loss": 2.8808, + "nll_loss": 2.869889736175537, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1791629046201706, + "rewards/margins": 1.650550127029419, + "rewards/rejected": -1.8297131061553955, + "step": 960 + }, + { + "epoch": 0.5978227060653188, + "grad_norm": 0.37688279151916504, + "learning_rate": 2.5974999999999998e-05, + "log_odds_chosen": 11.515750885009766, + "log_odds_ratio": -0.12808285653591156, + "logits/chosen": 0.4044106602668762, + "logits/rejected": 1.100311517715454, + "logps/chosen": -1.228320837020874, + "logps/rejected": -12.13768482208252, + "loss": 3.5135, + "nll_loss": 3.500669002532959, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12283208966255188, + "rewards/margins": 1.0909364223480225, + "rewards/rejected": -1.213768482208252, + "step": 961 + }, + { + "epoch": 0.5984447900466563, + "grad_norm": 0.5325865745544434, + "learning_rate": 2.595e-05, + "log_odds_chosen": 21.426223754882812, + "log_odds_ratio": -0.13188225030899048, + "logits/chosen": 0.30785199999809265, + "logits/rejected": 1.3725229501724243, + "logps/chosen": -1.0754218101501465, + "logps/rejected": -21.913288116455078, + "loss": 2.9752, + "nll_loss": 2.9620540142059326, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10754218697547913, + "rewards/margins": 2.0837864875793457, + "rewards/rejected": -2.191328525543213, + "step": 962 + }, + { + "epoch": 0.5990668740279937, + "grad_norm": 0.5490668416023254, + "learning_rate": 2.5925e-05, + "log_odds_chosen": 9.045169830322266, + "log_odds_ratio": -0.29519692063331604, + "logits/chosen": 0.2381560206413269, + "logits/rejected": 0.8757392764091492, + "logps/chosen": -1.1203395128250122, + "logps/rejected": -9.771276473999023, + "loss": 2.7598, + "nll_loss": 2.730231761932373, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11203394830226898, + "rewards/margins": 0.8650937676429749, + "rewards/rejected": -0.977127730846405, + "step": 963 + }, + { + "epoch": 0.5996889580093313, + "grad_norm": 0.5232042074203491, + "learning_rate": 2.5900000000000003e-05, + "log_odds_chosen": 16.675994873046875, + "log_odds_ratio": -0.07509205490350723, + "logits/chosen": 0.24147437512874603, + "logits/rejected": 1.0761547088623047, + "logps/chosen": -0.8001296520233154, + "logps/rejected": -16.835155487060547, + "loss": 3.0022, + "nll_loss": 2.9946541786193848, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0800129622220993, + "rewards/margins": 1.6035027503967285, + "rewards/rejected": -1.6835157871246338, + "step": 964 + }, + { + "epoch": 0.6003110419906688, + "grad_norm": 0.36291661858558655, + "learning_rate": 2.5875e-05, + "log_odds_chosen": 19.8973445892334, + "log_odds_ratio": -0.022971786558628082, + "logits/chosen": 0.07175733894109726, + "logits/rejected": 0.8581687211990356, + "logps/chosen": -0.9344284534454346, + "logps/rejected": -20.3033390045166, + "loss": 2.6604, + "nll_loss": 2.658095359802246, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09344284236431122, + "rewards/margins": 1.9368910789489746, + "rewards/rejected": -2.030333995819092, + "step": 965 + }, + { + "epoch": 0.6009331259720062, + "grad_norm": 0.8470042943954468, + "learning_rate": 2.585e-05, + "log_odds_chosen": 16.980083465576172, + "log_odds_ratio": -0.2519052028656006, + "logits/chosen": 0.22023792564868927, + "logits/rejected": 1.116905689239502, + "logps/chosen": -1.0762019157409668, + "logps/rejected": -17.686002731323242, + "loss": 3.0433, + "nll_loss": 3.0181312561035156, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10762019455432892, + "rewards/margins": 1.6609801054000854, + "rewards/rejected": -1.7686002254486084, + "step": 966 + }, + { + "epoch": 0.6015552099533437, + "grad_norm": 0.6046644449234009, + "learning_rate": 2.5824999999999998e-05, + "log_odds_chosen": 12.115790367126465, + "log_odds_ratio": -0.22894859313964844, + "logits/chosen": 0.21607916057109833, + "logits/rejected": 0.9050865173339844, + "logps/chosen": -1.0518709421157837, + "logps/rejected": -12.8179292678833, + "loss": 2.7021, + "nll_loss": 2.6792078018188477, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10518710315227509, + "rewards/margins": 1.1766059398651123, + "rewards/rejected": -1.2817929983139038, + "step": 967 + }, + { + "epoch": 0.6021772939346812, + "grad_norm": 0.5602324604988098, + "learning_rate": 2.58e-05, + "log_odds_chosen": 16.60848617553711, + "log_odds_ratio": -0.16328565776348114, + "logits/chosen": 0.2715965807437897, + "logits/rejected": 1.2223488092422485, + "logps/chosen": -1.2314372062683105, + "logps/rejected": -17.56035804748535, + "loss": 3.1499, + "nll_loss": 3.1335926055908203, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12314371764659882, + "rewards/margins": 1.63289213180542, + "rewards/rejected": -1.7560360431671143, + "step": 968 + }, + { + "epoch": 0.6027993779160187, + "grad_norm": 0.499213308095932, + "learning_rate": 2.5775e-05, + "log_odds_chosen": 9.463824272155762, + "log_odds_ratio": -0.2594992518424988, + "logits/chosen": 0.13768236339092255, + "logits/rejected": 0.3776164650917053, + "logps/chosen": -1.2077715396881104, + "logps/rejected": -10.380818367004395, + "loss": 3.2398, + "nll_loss": 3.2138986587524414, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12077715992927551, + "rewards/margins": 0.9173046350479126, + "rewards/rejected": -1.0380817651748657, + "step": 969 + }, + { + "epoch": 0.6034214618973561, + "grad_norm": 0.44655492901802063, + "learning_rate": 2.5750000000000002e-05, + "log_odds_chosen": 19.190210342407227, + "log_odds_ratio": -0.10930891335010529, + "logits/chosen": 0.1695220023393631, + "logits/rejected": 1.1123127937316895, + "logps/chosen": -1.1357357501983643, + "logps/rejected": -19.973918914794922, + "loss": 2.8969, + "nll_loss": 2.885981321334839, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1135735735297203, + "rewards/margins": 1.8838183879852295, + "rewards/rejected": -1.997391939163208, + "step": 970 + }, + { + "epoch": 0.6040435458786936, + "grad_norm": 0.539529025554657, + "learning_rate": 2.5725e-05, + "log_odds_chosen": 18.83353042602539, + "log_odds_ratio": -0.09447011351585388, + "logits/chosen": 0.15006932616233826, + "logits/rejected": 0.7248711585998535, + "logps/chosen": -1.026084303855896, + "logps/rejected": -19.30118179321289, + "loss": 3.159, + "nll_loss": 3.1495561599731445, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10260843485593796, + "rewards/margins": 1.827509880065918, + "rewards/rejected": -1.9301183223724365, + "step": 971 + }, + { + "epoch": 0.6046656298600311, + "grad_norm": 5.969109058380127, + "learning_rate": 2.57e-05, + "log_odds_chosen": 14.402713775634766, + "log_odds_ratio": -0.3919801414012909, + "logits/chosen": 0.18785786628723145, + "logits/rejected": 0.6937410235404968, + "logps/chosen": -1.815807819366455, + "logps/rejected": -15.869197845458984, + "loss": 3.7518, + "nll_loss": 3.712613821029663, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1815807819366455, + "rewards/margins": 1.4053387641906738, + "rewards/rejected": -1.5869196653366089, + "step": 972 + }, + { + "epoch": 0.6052877138413686, + "grad_norm": 0.48145902156829834, + "learning_rate": 2.5675e-05, + "log_odds_chosen": 18.73621940612793, + "log_odds_ratio": -0.11193342506885529, + "logits/chosen": 0.20852616429328918, + "logits/rejected": 1.1668469905853271, + "logps/chosen": -1.0225214958190918, + "logps/rejected": -19.24210548400879, + "loss": 2.9252, + "nll_loss": 2.9140021800994873, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10225215554237366, + "rewards/margins": 1.8219584226608276, + "rewards/rejected": -1.924210548400879, + "step": 973 + }, + { + "epoch": 0.605909797822706, + "grad_norm": 0.3980204463005066, + "learning_rate": 2.5650000000000003e-05, + "log_odds_chosen": 11.081676483154297, + "log_odds_ratio": -0.08179325610399246, + "logits/chosen": 0.26339149475097656, + "logits/rejected": 0.9369683861732483, + "logps/chosen": -0.9269077777862549, + "logps/rejected": -11.361970901489258, + "loss": 3.488, + "nll_loss": 3.479806900024414, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09269078075885773, + "rewards/margins": 1.043506383895874, + "rewards/rejected": -1.1361970901489258, + "step": 974 + }, + { + "epoch": 0.6065318818040435, + "grad_norm": 0.5839635729789734, + "learning_rate": 2.5625e-05, + "log_odds_chosen": 13.500631332397461, + "log_odds_ratio": -0.0822688415646553, + "logits/chosen": 0.28204748034477234, + "logits/rejected": 1.2430696487426758, + "logps/chosen": -1.311071753501892, + "logps/rejected": -14.472478866577148, + "loss": 3.6005, + "nll_loss": 3.59232234954834, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1311071664094925, + "rewards/margins": 1.3161406517028809, + "rewards/rejected": -1.4472477436065674, + "step": 975 + }, + { + "epoch": 0.6071539657853811, + "grad_norm": 0.44382327795028687, + "learning_rate": 2.5600000000000002e-05, + "log_odds_chosen": 18.092775344848633, + "log_odds_ratio": -0.11017350852489471, + "logits/chosen": 0.06789172440767288, + "logits/rejected": 0.7348443865776062, + "logps/chosen": -0.9244815111160278, + "logps/rejected": -18.366724014282227, + "loss": 3.2554, + "nll_loss": 3.2444264888763428, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0924481526017189, + "rewards/margins": 1.744224190711975, + "rewards/rejected": -1.8366724252700806, + "step": 976 + }, + { + "epoch": 0.6077760497667185, + "grad_norm": 0.6069784164428711, + "learning_rate": 2.5574999999999998e-05, + "log_odds_chosen": 10.627729415893555, + "log_odds_ratio": -0.12660586833953857, + "logits/chosen": 0.0689058005809784, + "logits/rejected": 0.5917812585830688, + "logps/chosen": -0.9631859064102173, + "logps/rejected": -11.07681655883789, + "loss": 2.7384, + "nll_loss": 2.7257091999053955, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09631859511137009, + "rewards/margins": 1.0113630294799805, + "rewards/rejected": -1.1076816320419312, + "step": 977 + }, + { + "epoch": 0.608398133748056, + "grad_norm": 0.45217904448509216, + "learning_rate": 2.555e-05, + "log_odds_chosen": 15.359528541564941, + "log_odds_ratio": -0.21051090955734253, + "logits/chosen": 0.065369613468647, + "logits/rejected": 0.8507799506187439, + "logps/chosen": -1.0331388711929321, + "logps/rejected": -15.877238273620605, + "loss": 2.6868, + "nll_loss": 2.6657238006591797, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10331389307975769, + "rewards/margins": 1.4844098091125488, + "rewards/rejected": -1.587723731994629, + "step": 978 + }, + { + "epoch": 0.6090202177293935, + "grad_norm": 0.5865066051483154, + "learning_rate": 2.5525e-05, + "log_odds_chosen": 17.618770599365234, + "log_odds_ratio": -0.10912881791591644, + "logits/chosen": 0.110966756939888, + "logits/rejected": 1.0559561252593994, + "logps/chosen": -1.1738271713256836, + "logps/rejected": -18.374874114990234, + "loss": 2.8958, + "nll_loss": 2.8849360942840576, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1173827201128006, + "rewards/margins": 1.7201048135757446, + "rewards/rejected": -1.8374874591827393, + "step": 979 + }, + { + "epoch": 0.609642301710731, + "grad_norm": 0.4005695879459381, + "learning_rate": 2.5500000000000003e-05, + "log_odds_chosen": 15.089963912963867, + "log_odds_ratio": -0.10706076771020889, + "logits/chosen": 0.19660604000091553, + "logits/rejected": 0.7421777844429016, + "logps/chosen": -1.6507558822631836, + "logps/rejected": -16.388992309570312, + "loss": 3.6645, + "nll_loss": 3.6537725925445557, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16507560014724731, + "rewards/margins": 1.4738237857818604, + "rewards/rejected": -1.638899326324463, + "step": 980 + }, + { + "epoch": 0.6102643856920684, + "grad_norm": 0.49785563349723816, + "learning_rate": 2.5475e-05, + "log_odds_chosen": 11.623516082763672, + "log_odds_ratio": -0.26292070746421814, + "logits/chosen": 0.11350183188915253, + "logits/rejected": 0.8784139752388, + "logps/chosen": -1.347922682762146, + "logps/rejected": -12.797706604003906, + "loss": 2.894, + "nll_loss": 2.867668390274048, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1347922682762146, + "rewards/margins": 1.144978404045105, + "rewards/rejected": -1.2797706127166748, + "step": 981 + }, + { + "epoch": 0.6108864696734059, + "grad_norm": 0.4649457037448883, + "learning_rate": 2.5450000000000002e-05, + "log_odds_chosen": 20.90005874633789, + "log_odds_ratio": -8.940698137394065e-08, + "logits/chosen": 0.21456483006477356, + "logits/rejected": 1.568105697631836, + "logps/chosen": -1.1345700025558472, + "logps/rejected": -21.52691650390625, + "loss": 3.2944, + "nll_loss": 3.2943665981292725, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11345699429512024, + "rewards/margins": 2.0392348766326904, + "rewards/rejected": -2.1526918411254883, + "step": 982 + }, + { + "epoch": 0.6115085536547434, + "grad_norm": 0.5988860130310059, + "learning_rate": 2.5424999999999998e-05, + "log_odds_chosen": 10.66419792175293, + "log_odds_ratio": -0.3058769106864929, + "logits/chosen": 0.05050152167677879, + "logits/rejected": 0.2465941309928894, + "logps/chosen": -0.8942004442214966, + "logps/rejected": -11.015356063842773, + "loss": 3.0743, + "nll_loss": 3.0436720848083496, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08942003548145294, + "rewards/margins": 1.012115716934204, + "rewards/rejected": -1.101535677909851, + "step": 983 + }, + { + "epoch": 0.6121306376360809, + "grad_norm": 0.6072388887405396, + "learning_rate": 2.54e-05, + "log_odds_chosen": 16.32722282409668, + "log_odds_ratio": -0.11603593826293945, + "logits/chosen": 0.12268135696649551, + "logits/rejected": 1.0790374279022217, + "logps/chosen": -1.3971768617630005, + "logps/rejected": -17.381330490112305, + "loss": 2.7476, + "nll_loss": 2.735957384109497, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1397176831960678, + "rewards/margins": 1.5984153747558594, + "rewards/rejected": -1.7381330728530884, + "step": 984 + }, + { + "epoch": 0.6127527216174183, + "grad_norm": 0.4394105076789856, + "learning_rate": 2.5375e-05, + "log_odds_chosen": 20.815372467041016, + "log_odds_ratio": -0.11913128942251205, + "logits/chosen": 0.2849823832511902, + "logits/rejected": 1.7651139497756958, + "logps/chosen": -1.3435211181640625, + "logps/rejected": -21.902217864990234, + "loss": 3.1595, + "nll_loss": 3.1476199626922607, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13435211777687073, + "rewards/margins": 2.0558695793151855, + "rewards/rejected": -2.1902217864990234, + "step": 985 + }, + { + "epoch": 0.6133748055987558, + "grad_norm": 0.4176173210144043, + "learning_rate": 2.5350000000000003e-05, + "log_odds_chosen": 20.118194580078125, + "log_odds_ratio": -0.041579604148864746, + "logits/chosen": 0.2213955968618393, + "logits/rejected": 1.2255231142044067, + "logps/chosen": -1.022838830947876, + "logps/rejected": -20.671480178833008, + "loss": 3.4104, + "nll_loss": 3.406257390975952, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10228388756513596, + "rewards/margins": 1.9648641347885132, + "rewards/rejected": -2.067148208618164, + "step": 986 + }, + { + "epoch": 0.6139968895800934, + "grad_norm": 0.5290238261222839, + "learning_rate": 2.5325e-05, + "log_odds_chosen": 15.454238891601562, + "log_odds_ratio": -0.09638696908950806, + "logits/chosen": 0.2696585953235626, + "logits/rejected": 1.1312285661697388, + "logps/chosen": -1.2690268754959106, + "logps/rejected": -16.407495498657227, + "loss": 3.1839, + "nll_loss": 3.174257755279541, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12690268456935883, + "rewards/margins": 1.51384699344635, + "rewards/rejected": -1.6407495737075806, + "step": 987 + }, + { + "epoch": 0.6146189735614308, + "grad_norm": 0.44811832904815674, + "learning_rate": 2.5300000000000002e-05, + "log_odds_chosen": 18.196022033691406, + "log_odds_ratio": -0.0027530835941433907, + "logits/chosen": 0.306151807308197, + "logits/rejected": 1.4259248971939087, + "logps/chosen": -1.3465137481689453, + "logps/rejected": -19.213729858398438, + "loss": 3.4642, + "nll_loss": 3.463940143585205, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13465136289596558, + "rewards/margins": 1.7867215871810913, + "rewards/rejected": -1.9213731288909912, + "step": 988 + }, + { + "epoch": 0.6152410575427683, + "grad_norm": 0.5820187330245972, + "learning_rate": 2.5274999999999998e-05, + "log_odds_chosen": 13.781078338623047, + "log_odds_ratio": -0.19668269157409668, + "logits/chosen": -0.0342557318508625, + "logits/rejected": 0.30851006507873535, + "logps/chosen": -0.8840237855911255, + "logps/rejected": -14.013053894042969, + "loss": 2.5317, + "nll_loss": 2.5119993686676025, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08840238302946091, + "rewards/margins": 1.3129030466079712, + "rewards/rejected": -1.4013054370880127, + "step": 989 + }, + { + "epoch": 0.6158631415241057, + "grad_norm": 0.5693361163139343, + "learning_rate": 2.525e-05, + "log_odds_chosen": 9.703099250793457, + "log_odds_ratio": -0.2681022882461548, + "logits/chosen": 0.00022461498156189919, + "logits/rejected": 0.3866625428199768, + "logps/chosen": -1.0230138301849365, + "logps/rejected": -10.349815368652344, + "loss": 2.8994, + "nll_loss": 2.8725709915161133, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10230138897895813, + "rewards/margins": 0.9326801300048828, + "rewards/rejected": -1.0349814891815186, + "step": 990 + }, + { + "epoch": 0.6164852255054433, + "grad_norm": 0.5749667882919312, + "learning_rate": 2.5225e-05, + "log_odds_chosen": 10.241193771362305, + "log_odds_ratio": -0.3188455104827881, + "logits/chosen": 0.07506673783063889, + "logits/rejected": 0.46898865699768066, + "logps/chosen": -1.2021185159683228, + "logps/rejected": -11.231522560119629, + "loss": 2.8334, + "nll_loss": 2.8015153408050537, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12021185457706451, + "rewards/margins": 1.0029405355453491, + "rewards/rejected": -1.1231523752212524, + "step": 991 + }, + { + "epoch": 0.6171073094867807, + "grad_norm": 0.6073411107063293, + "learning_rate": 2.5200000000000003e-05, + "log_odds_chosen": 11.625310897827148, + "log_odds_ratio": -0.19626720249652863, + "logits/chosen": 0.2134128212928772, + "logits/rejected": 0.8630800247192383, + "logps/chosen": -0.7396443486213684, + "logps/rejected": -11.635083198547363, + "loss": 3.168, + "nll_loss": 3.148395538330078, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0739644318819046, + "rewards/margins": 1.0895440578460693, + "rewards/rejected": -1.163508415222168, + "step": 992 + }, + { + "epoch": 0.6177293934681182, + "grad_norm": 0.5637009143829346, + "learning_rate": 2.5175e-05, + "log_odds_chosen": 8.45470142364502, + "log_odds_ratio": -0.38158631324768066, + "logits/chosen": 0.02333865687251091, + "logits/rejected": 0.4010867476463318, + "logps/chosen": -1.1170642375946045, + "logps/rejected": -9.288888931274414, + "loss": 2.8206, + "nll_loss": 2.7824807167053223, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11170642077922821, + "rewards/margins": 0.8171824216842651, + "rewards/rejected": -0.9288888573646545, + "step": 993 + }, + { + "epoch": 0.6183514774494556, + "grad_norm": 0.5053258538246155, + "learning_rate": 2.515e-05, + "log_odds_chosen": 21.61244773864746, + "log_odds_ratio": -0.11922389268875122, + "logits/chosen": 0.281334787607193, + "logits/rejected": 1.7507303953170776, + "logps/chosen": -1.2235077619552612, + "logps/rejected": -22.507673263549805, + "loss": 2.9003, + "nll_loss": 2.88834810256958, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1223507821559906, + "rewards/margins": 2.1284167766571045, + "rewards/rejected": -2.250767469406128, + "step": 994 + }, + { + "epoch": 0.6189735614307932, + "grad_norm": 0.4218065142631531, + "learning_rate": 2.5124999999999997e-05, + "log_odds_chosen": 13.609810829162598, + "log_odds_ratio": -0.17529192566871643, + "logits/chosen": 0.20690587162971497, + "logits/rejected": 0.9557357430458069, + "logps/chosen": -1.2656549215316772, + "logps/rejected": -14.553084373474121, + "loss": 3.604, + "nll_loss": 3.5864503383636475, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12656548619270325, + "rewards/margins": 1.3287428617477417, + "rewards/rejected": -1.455308437347412, + "step": 995 + }, + { + "epoch": 0.6195956454121306, + "grad_norm": 0.6388038396835327, + "learning_rate": 2.51e-05, + "log_odds_chosen": 13.46223258972168, + "log_odds_ratio": -0.28370407223701477, + "logits/chosen": 0.19488823413848877, + "logits/rejected": 0.7138241529464722, + "logps/chosen": -1.0961161851882935, + "logps/rejected": -14.22940444946289, + "loss": 3.1086, + "nll_loss": 3.0802266597747803, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1096116155385971, + "rewards/margins": 1.3133288621902466, + "rewards/rejected": -1.4229404926300049, + "step": 996 + }, + { + "epoch": 0.6202177293934681, + "grad_norm": 0.49505603313446045, + "learning_rate": 2.5075e-05, + "log_odds_chosen": 9.735605239868164, + "log_odds_ratio": -0.21508948504924774, + "logits/chosen": 0.14490671455860138, + "logits/rejected": 0.45840179920196533, + "logps/chosen": -1.259823203086853, + "logps/rejected": -10.751209259033203, + "loss": 3.2765, + "nll_loss": 3.254956007003784, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12598231434822083, + "rewards/margins": 0.9491385221481323, + "rewards/rejected": -1.0751209259033203, + "step": 997 + }, + { + "epoch": 0.6208398133748056, + "grad_norm": 0.5221970677375793, + "learning_rate": 2.5050000000000002e-05, + "log_odds_chosen": 17.351388931274414, + "log_odds_ratio": -0.0899883359670639, + "logits/chosen": 0.22275736927986145, + "logits/rejected": 0.9023792743682861, + "logps/chosen": -1.104635238647461, + "logps/rejected": -18.011194229125977, + "loss": 3.4768, + "nll_loss": 3.467817783355713, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11046352982521057, + "rewards/margins": 1.690656065940857, + "rewards/rejected": -1.8011195659637451, + "step": 998 + }, + { + "epoch": 0.6214618973561431, + "grad_norm": 0.5278105735778809, + "learning_rate": 2.5025e-05, + "log_odds_chosen": 19.447450637817383, + "log_odds_ratio": -0.1714281141757965, + "logits/chosen": 0.2100244164466858, + "logits/rejected": 1.0468909740447998, + "logps/chosen": -0.9524563550949097, + "logps/rejected": -19.773094177246094, + "loss": 3.0782, + "nll_loss": 3.06109619140625, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09524564445018768, + "rewards/margins": 1.882063865661621, + "rewards/rejected": -1.9773094654083252, + "step": 999 + }, + { + "epoch": 0.6220839813374806, + "grad_norm": 0.7394985556602478, + "learning_rate": 2.5e-05, + "log_odds_chosen": 24.217151641845703, + "log_odds_ratio": -0.11016383767127991, + "logits/chosen": 0.09682357311248779, + "logits/rejected": 1.188527226448059, + "logps/chosen": -1.3372222185134888, + "logps/rejected": -25.23487091064453, + "loss": 2.2574, + "nll_loss": 2.2463905811309814, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1337222158908844, + "rewards/margins": 2.3897650241851807, + "rewards/rejected": -2.5234873294830322, + "step": 1000 + }, + { + "epoch": 0.622706065318818, + "grad_norm": 10.445813179016113, + "learning_rate": 2.4975e-05, + "log_odds_chosen": 25.370601654052734, + "log_odds_ratio": -0.0863175019621849, + "logits/chosen": 0.23290672898292542, + "logits/rejected": 1.5314041376113892, + "logps/chosen": -1.8342373371124268, + "logps/rejected": -26.98197364807129, + "loss": 2.9581, + "nll_loss": 2.949495315551758, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1834237426519394, + "rewards/margins": 2.514773368835449, + "rewards/rejected": -2.69819712638855, + "step": 1001 + }, + { + "epoch": 0.6233281493001556, + "grad_norm": 0.9837327599525452, + "learning_rate": 2.495e-05, + "log_odds_chosen": 26.120695114135742, + "log_odds_ratio": -8.239331509685144e-05, + "logits/chosen": 0.21516145765781403, + "logits/rejected": 1.3835011720657349, + "logps/chosen": -1.1667652130126953, + "logps/rejected": -26.76409912109375, + "loss": 3.2298, + "nll_loss": 3.2298390865325928, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11667650938034058, + "rewards/margins": 2.5597333908081055, + "rewards/rejected": -2.6764097213745117, + "step": 1002 + }, + { + "epoch": 0.623950233281493, + "grad_norm": 0.5452992916107178, + "learning_rate": 2.4925000000000003e-05, + "log_odds_chosen": 11.288724899291992, + "log_odds_ratio": -0.3280940353870392, + "logits/chosen": 0.02780914306640625, + "logits/rejected": 0.698295533657074, + "logps/chosen": -1.4289721250534058, + "logps/rejected": -12.558980941772461, + "loss": 2.5772, + "nll_loss": 2.54435658454895, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14289721846580505, + "rewards/margins": 1.1130008697509766, + "rewards/rejected": -1.2558979988098145, + "step": 1003 + }, + { + "epoch": 0.6245723172628305, + "grad_norm": 1.2315927743911743, + "learning_rate": 2.4900000000000002e-05, + "log_odds_chosen": 21.15532684326172, + "log_odds_ratio": -0.2190973460674286, + "logits/chosen": 0.11318109184503555, + "logits/rejected": 1.183733582496643, + "logps/chosen": -1.5021014213562012, + "logps/rejected": -22.3739070892334, + "loss": 2.6072, + "nll_loss": 2.5852890014648438, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1502101570367813, + "rewards/margins": 2.0871806144714355, + "rewards/rejected": -2.2373907566070557, + "step": 1004 + }, + { + "epoch": 0.6251944012441679, + "grad_norm": 0.5702396631240845, + "learning_rate": 2.4875e-05, + "log_odds_chosen": 18.833229064941406, + "log_odds_ratio": -0.07586246728897095, + "logits/chosen": -0.06201568990945816, + "logits/rejected": 1.269275426864624, + "logps/chosen": -1.2848140001296997, + "logps/rejected": -19.835512161254883, + "loss": 2.0326, + "nll_loss": 2.0250072479248047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1284814029932022, + "rewards/margins": 1.855069875717163, + "rewards/rejected": -1.9835513830184937, + "step": 1005 + }, + { + "epoch": 0.6258164852255055, + "grad_norm": 0.5604910850524902, + "learning_rate": 2.485e-05, + "log_odds_chosen": 12.637853622436523, + "log_odds_ratio": -0.08352010697126389, + "logits/chosen": 0.0846133753657341, + "logits/rejected": 0.6342355608940125, + "logps/chosen": -0.9381386637687683, + "logps/rejected": -12.998655319213867, + "loss": 2.849, + "nll_loss": 2.8406927585601807, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09381386637687683, + "rewards/margins": 1.2060517072677612, + "rewards/rejected": -1.299865484237671, + "step": 1006 + }, + { + "epoch": 0.6264385692068429, + "grad_norm": 0.42290839552879333, + "learning_rate": 2.4825e-05, + "log_odds_chosen": 11.835851669311523, + "log_odds_ratio": -0.1617979258298874, + "logits/chosen": 0.10653581470251083, + "logits/rejected": 0.5529526472091675, + "logps/chosen": -1.1142702102661133, + "logps/rejected": -12.558614730834961, + "loss": 3.1891, + "nll_loss": 3.1729464530944824, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11142702400684357, + "rewards/margins": 1.1444344520568848, + "rewards/rejected": -1.255861520767212, + "step": 1007 + }, + { + "epoch": 0.6270606531881804, + "grad_norm": 0.48082074522972107, + "learning_rate": 2.48e-05, + "log_odds_chosen": 20.665428161621094, + "log_odds_ratio": -0.0005173269310034811, + "logits/chosen": 0.2348843216896057, + "logits/rejected": 1.3875151872634888, + "logps/chosen": -1.0887088775634766, + "logps/rejected": -21.299148559570312, + "loss": 3.3537, + "nll_loss": 3.3536720275878906, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10887089371681213, + "rewards/margins": 2.0210437774658203, + "rewards/rejected": -2.1299147605895996, + "step": 1008 + }, + { + "epoch": 0.6276827371695178, + "grad_norm": 0.589474618434906, + "learning_rate": 2.4775000000000003e-05, + "log_odds_chosen": 8.409334182739258, + "log_odds_ratio": -0.2995333969593048, + "logits/chosen": 0.09103868901729584, + "logits/rejected": 0.5081114768981934, + "logps/chosen": -1.3712034225463867, + "logps/rejected": -9.455653190612793, + "loss": 3.2591, + "nll_loss": 3.2291390895843506, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1371203362941742, + "rewards/margins": 0.8084449768066406, + "rewards/rejected": -0.9455653429031372, + "step": 1009 + }, + { + "epoch": 0.6283048211508554, + "grad_norm": 1.1940956115722656, + "learning_rate": 2.4750000000000002e-05, + "log_odds_chosen": 11.112287521362305, + "log_odds_ratio": -0.17824390530586243, + "logits/chosen": 0.030925815925002098, + "logits/rejected": 0.5977746248245239, + "logps/chosen": -1.2858790159225464, + "logps/rejected": -12.050931930541992, + "loss": 3.1955, + "nll_loss": 3.177677869796753, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12858790159225464, + "rewards/margins": 1.0765053033828735, + "rewards/rejected": -1.2050931453704834, + "step": 1010 + }, + { + "epoch": 0.6289269051321928, + "grad_norm": 0.40679919719696045, + "learning_rate": 2.4725e-05, + "log_odds_chosen": 19.506404876708984, + "log_odds_ratio": -0.21070463955402374, + "logits/chosen": 0.08191069960594177, + "logits/rejected": 0.7757222652435303, + "logps/chosen": -0.983881950378418, + "logps/rejected": -20.12225341796875, + "loss": 2.9743, + "nll_loss": 2.953184127807617, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0983881950378418, + "rewards/margins": 1.9138373136520386, + "rewards/rejected": -2.012225389480591, + "step": 1011 + }, + { + "epoch": 0.6295489891135303, + "grad_norm": 1.9736859798431396, + "learning_rate": 2.47e-05, + "log_odds_chosen": 12.312723159790039, + "log_odds_ratio": -0.2725464701652527, + "logits/chosen": 0.09264761954545975, + "logits/rejected": 0.7018003463745117, + "logps/chosen": -1.1806060075759888, + "logps/rejected": -13.192741394042969, + "loss": 2.6436, + "nll_loss": 2.6163501739501953, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11806060373783112, + "rewards/margins": 1.2012135982513428, + "rewards/rejected": -1.3192741870880127, + "step": 1012 + }, + { + "epoch": 0.6301710730948679, + "grad_norm": 0.5497341156005859, + "learning_rate": 2.4675e-05, + "log_odds_chosen": 12.800125122070312, + "log_odds_ratio": -0.133247509598732, + "logits/chosen": 0.07118834555149078, + "logits/rejected": 1.0009067058563232, + "logps/chosen": -1.0593130588531494, + "logps/rejected": -13.403684616088867, + "loss": 2.8056, + "nll_loss": 2.792227029800415, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10593131929636002, + "rewards/margins": 1.2344372272491455, + "rewards/rejected": -1.3403685092926025, + "step": 1013 + }, + { + "epoch": 0.6307931570762053, + "grad_norm": 0.4955105185508728, + "learning_rate": 2.465e-05, + "log_odds_chosen": 15.762319564819336, + "log_odds_ratio": -0.16057822108268738, + "logits/chosen": 0.27589911222457886, + "logits/rejected": 1.0252506732940674, + "logps/chosen": -1.2522268295288086, + "logps/rejected": -16.704978942871094, + "loss": 3.5937, + "nll_loss": 3.5776491165161133, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12522268295288086, + "rewards/margins": 1.545275092124939, + "rewards/rejected": -1.6704977750778198, + "step": 1014 + }, + { + "epoch": 0.6314152410575428, + "grad_norm": 0.4815879464149475, + "learning_rate": 2.4625000000000002e-05, + "log_odds_chosen": 16.962005615234375, + "log_odds_ratio": -0.012883543968200684, + "logits/chosen": 0.05764898285269737, + "logits/rejected": 0.8884307742118835, + "logps/chosen": -1.1525784730911255, + "logps/rejected": -17.727672576904297, + "loss": 2.7473, + "nll_loss": 2.7459635734558105, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11525784432888031, + "rewards/margins": 1.6575093269348145, + "rewards/rejected": -1.7727673053741455, + "step": 1015 + }, + { + "epoch": 0.6320373250388802, + "grad_norm": 0.48324769735336304, + "learning_rate": 2.46e-05, + "log_odds_chosen": 16.998523712158203, + "log_odds_ratio": -0.06444693356752396, + "logits/chosen": 0.09633009135723114, + "logits/rejected": 0.7852844595909119, + "logps/chosen": -1.0850187540054321, + "logps/rejected": -17.69282341003418, + "loss": 2.8924, + "nll_loss": 2.88600492477417, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10850188136100769, + "rewards/margins": 1.660780429840088, + "rewards/rejected": -1.769282341003418, + "step": 1016 + }, + { + "epoch": 0.6326594090202178, + "grad_norm": 0.7034952044487, + "learning_rate": 2.4575e-05, + "log_odds_chosen": 10.237931251525879, + "log_odds_ratio": -0.2111145257949829, + "logits/chosen": 0.10851834714412689, + "logits/rejected": 0.8848839998245239, + "logps/chosen": -1.2556936740875244, + "logps/rejected": -11.173830032348633, + "loss": 2.6084, + "nll_loss": 2.5872819423675537, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12556937336921692, + "rewards/margins": 0.9918137192726135, + "rewards/rejected": -1.1173830032348633, + "step": 1017 + }, + { + "epoch": 0.6332814930015552, + "grad_norm": 0.4607456922531128, + "learning_rate": 2.455e-05, + "log_odds_chosen": 18.348682403564453, + "log_odds_ratio": -0.001183602144010365, + "logits/chosen": 0.18806810677051544, + "logits/rejected": 1.8798189163208008, + "logps/chosen": -1.3022570610046387, + "logps/rejected": -19.317102432250977, + "loss": 3.0541, + "nll_loss": 3.0539772510528564, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13022570312023163, + "rewards/margins": 1.8014845848083496, + "rewards/rejected": -1.9317102432250977, + "step": 1018 + }, + { + "epoch": 0.6339035769828927, + "grad_norm": 0.5187017917633057, + "learning_rate": 2.4525e-05, + "log_odds_chosen": 22.93252944946289, + "log_odds_ratio": -4.4703490686970326e-08, + "logits/chosen": 0.1605634093284607, + "logits/rejected": 1.8251597881317139, + "logps/chosen": -1.1265020370483398, + "logps/rejected": -23.65814971923828, + "loss": 2.4123, + "nll_loss": 2.4122743606567383, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11265021562576294, + "rewards/margins": 2.2531650066375732, + "rewards/rejected": -2.3658151626586914, + "step": 1019 + }, + { + "epoch": 0.6345256609642301, + "grad_norm": 0.5116434097290039, + "learning_rate": 2.45e-05, + "log_odds_chosen": 17.80671501159668, + "log_odds_ratio": -0.2849065065383911, + "logits/chosen": 0.10839949548244476, + "logits/rejected": 1.3892898559570312, + "logps/chosen": -1.1734644174575806, + "logps/rejected": -18.635705947875977, + "loss": 3.038, + "nll_loss": 3.009539842605591, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11734645068645477, + "rewards/margins": 1.7462241649627686, + "rewards/rejected": -1.8635706901550293, + "step": 1020 + }, + { + "epoch": 0.6351477449455677, + "grad_norm": 0.5411016345024109, + "learning_rate": 2.4475000000000002e-05, + "log_odds_chosen": 14.560041427612305, + "log_odds_ratio": -0.23231032490730286, + "logits/chosen": 0.20895469188690186, + "logits/rejected": 1.1282305717468262, + "logps/chosen": -0.9110753536224365, + "logps/rejected": -15.089509963989258, + "loss": 3.0198, + "nll_loss": 2.9965720176696777, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09110753983259201, + "rewards/margins": 1.4178435802459717, + "rewards/rejected": -1.50895094871521, + "step": 1021 + }, + { + "epoch": 0.6357698289269051, + "grad_norm": 0.6156372427940369, + "learning_rate": 2.445e-05, + "log_odds_chosen": 16.500022888183594, + "log_odds_ratio": -0.12201125919818878, + "logits/chosen": 0.19160394370555878, + "logits/rejected": 0.9838314652442932, + "logps/chosen": -1.152182936668396, + "logps/rejected": -17.30032730102539, + "loss": 2.4539, + "nll_loss": 2.441667318344116, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11521830409765244, + "rewards/margins": 1.6148146390914917, + "rewards/rejected": -1.7300329208374023, + "step": 1022 + }, + { + "epoch": 0.6363919129082426, + "grad_norm": 0.437686949968338, + "learning_rate": 2.4425e-05, + "log_odds_chosen": 9.71164608001709, + "log_odds_ratio": -0.1439571976661682, + "logits/chosen": 0.051395781338214874, + "logits/rejected": 0.7666381001472473, + "logps/chosen": -1.4583333730697632, + "logps/rejected": -10.898263931274414, + "loss": 2.44, + "nll_loss": 2.4255826473236084, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1458333432674408, + "rewards/margins": 0.9439931511878967, + "rewards/rejected": -1.0898264646530151, + "step": 1023 + }, + { + "epoch": 0.63701399688958, + "grad_norm": 0.42805302143096924, + "learning_rate": 2.44e-05, + "log_odds_chosen": 16.866641998291016, + "log_odds_ratio": -0.11649006605148315, + "logits/chosen": 0.17741787433624268, + "logits/rejected": 1.0520137548446655, + "logps/chosen": -1.4884114265441895, + "logps/rejected": -18.07642364501953, + "loss": 3.1272, + "nll_loss": 3.1155271530151367, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14884114265441895, + "rewards/margins": 1.6588011980056763, + "rewards/rejected": -1.8076422214508057, + "step": 1024 + }, + { + "epoch": 0.6376360808709176, + "grad_norm": 0.4942615330219269, + "learning_rate": 2.4375e-05, + "log_odds_chosen": 20.236421585083008, + "log_odds_ratio": -0.0003126816882286221, + "logits/chosen": 0.3794230818748474, + "logits/rejected": 1.705965518951416, + "logps/chosen": -1.119978427886963, + "logps/rejected": -20.884624481201172, + "loss": 3.1617, + "nll_loss": 3.1616899967193604, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11199785023927689, + "rewards/margins": 1.9764647483825684, + "rewards/rejected": -2.0884625911712646, + "step": 1025 + }, + { + "epoch": 0.6382581648522551, + "grad_norm": 0.5315221548080444, + "learning_rate": 2.435e-05, + "log_odds_chosen": 17.677621841430664, + "log_odds_ratio": -0.1995067596435547, + "logits/chosen": 0.1513439416885376, + "logits/rejected": 1.3360419273376465, + "logps/chosen": -1.2765010595321655, + "logps/rejected": -18.678483963012695, + "loss": 2.5653, + "nll_loss": 2.545377016067505, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12765009701251984, + "rewards/margins": 1.7401981353759766, + "rewards/rejected": -1.8678483963012695, + "step": 1026 + }, + { + "epoch": 0.6388802488335925, + "grad_norm": 1.1557024717330933, + "learning_rate": 2.4325000000000002e-05, + "log_odds_chosen": 19.117094039916992, + "log_odds_ratio": -0.08928176760673523, + "logits/chosen": 0.2427656352519989, + "logits/rejected": 1.4793113470077515, + "logps/chosen": -1.402268409729004, + "logps/rejected": -20.209617614746094, + "loss": 2.912, + "nll_loss": 2.9031004905700684, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1402268409729004, + "rewards/margins": 1.880735158920288, + "rewards/rejected": -2.0209617614746094, + "step": 1027 + }, + { + "epoch": 0.63950233281493, + "grad_norm": 0.6319509744644165, + "learning_rate": 2.43e-05, + "log_odds_chosen": 21.61825942993164, + "log_odds_ratio": -0.09945891797542572, + "logits/chosen": 0.10278205573558807, + "logits/rejected": 0.8583885431289673, + "logps/chosen": -1.3808908462524414, + "logps/rejected": -22.713088989257812, + "loss": 2.7508, + "nll_loss": 2.7408299446105957, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13808909058570862, + "rewards/margins": 2.1332197189331055, + "rewards/rejected": -2.2713088989257812, + "step": 1028 + }, + { + "epoch": 0.6401244167962675, + "grad_norm": 0.4444740116596222, + "learning_rate": 2.4275e-05, + "log_odds_chosen": 20.923830032348633, + "log_odds_ratio": -0.1124124825000763, + "logits/chosen": 0.09198861569166183, + "logits/rejected": 0.6280720829963684, + "logps/chosen": -1.0244150161743164, + "logps/rejected": -21.289560317993164, + "loss": 2.8599, + "nll_loss": 2.848630905151367, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10244148969650269, + "rewards/margins": 2.0265142917633057, + "rewards/rejected": -2.1289560794830322, + "step": 1029 + }, + { + "epoch": 0.640746500777605, + "grad_norm": 0.4391319751739502, + "learning_rate": 2.425e-05, + "log_odds_chosen": 17.82830810546875, + "log_odds_ratio": -0.20137743651866913, + "logits/chosen": 0.4636791944503784, + "logits/rejected": 1.4730623960494995, + "logps/chosen": -1.35006582736969, + "logps/rejected": -18.960269927978516, + "loss": 3.6925, + "nll_loss": 3.672400951385498, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13500657677650452, + "rewards/margins": 1.761020541191101, + "rewards/rejected": -1.8960270881652832, + "step": 1030 + }, + { + "epoch": 0.6413685847589424, + "grad_norm": 5.313420295715332, + "learning_rate": 2.4225e-05, + "log_odds_chosen": 16.11229705810547, + "log_odds_ratio": -0.06506342440843582, + "logits/chosen": 0.2195087969303131, + "logits/rejected": 0.9584240317344666, + "logps/chosen": -2.420592784881592, + "logps/rejected": -18.16111183166504, + "loss": 3.0274, + "nll_loss": 3.020932912826538, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.24205929040908813, + "rewards/margins": 1.574051856994629, + "rewards/rejected": -1.8161113262176514, + "step": 1031 + }, + { + "epoch": 0.64199066874028, + "grad_norm": 0.40007975697517395, + "learning_rate": 2.4200000000000002e-05, + "log_odds_chosen": 15.192806243896484, + "log_odds_ratio": -0.3525593876838684, + "logits/chosen": 0.2354370653629303, + "logits/rejected": 1.27947199344635, + "logps/chosen": -1.2368344068527222, + "logps/rejected": -16.266971588134766, + "loss": 3.1313, + "nll_loss": 3.096068859100342, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12368344515562057, + "rewards/margins": 1.5030137300491333, + "rewards/rejected": -1.6266971826553345, + "step": 1032 + }, + { + "epoch": 0.6426127527216174, + "grad_norm": 0.5552942156791687, + "learning_rate": 2.4175e-05, + "log_odds_chosen": 15.988584518432617, + "log_odds_ratio": -0.07936682552099228, + "logits/chosen": 0.10833962261676788, + "logits/rejected": 0.8231009244918823, + "logps/chosen": -0.965819239616394, + "logps/rejected": -16.373872756958008, + "loss": 2.7753, + "nll_loss": 2.7673282623291016, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09658192843198776, + "rewards/margins": 1.5408053398132324, + "rewards/rejected": -1.6373872756958008, + "step": 1033 + }, + { + "epoch": 0.6432348367029549, + "grad_norm": 0.4293906092643738, + "learning_rate": 2.415e-05, + "log_odds_chosen": 26.196657180786133, + "log_odds_ratio": -0.0685395896434784, + "logits/chosen": 0.19044487178325653, + "logits/rejected": 1.0367000102996826, + "logps/chosen": -0.9686414003372192, + "logps/rejected": -26.55857276916504, + "loss": 3.0572, + "nll_loss": 3.050372838973999, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09686414152383804, + "rewards/margins": 2.558993101119995, + "rewards/rejected": -2.6558570861816406, + "step": 1034 + }, + { + "epoch": 0.6438569206842923, + "grad_norm": 0.5134528279304504, + "learning_rate": 2.4125e-05, + "log_odds_chosen": 22.512725830078125, + "log_odds_ratio": -1.673509723332245e-05, + "logits/chosen": 0.03580861911177635, + "logits/rejected": 0.7297523021697998, + "logps/chosen": -1.1147419214248657, + "logps/rejected": -23.143142700195312, + "loss": 2.7376, + "nll_loss": 2.7376210689544678, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11147420108318329, + "rewards/margins": 2.2028400897979736, + "rewards/rejected": -2.314314365386963, + "step": 1035 + }, + { + "epoch": 0.6444790046656299, + "grad_norm": 3.430917263031006, + "learning_rate": 2.41e-05, + "log_odds_chosen": 16.318185806274414, + "log_odds_ratio": -0.0678626224398613, + "logits/chosen": 0.272255003452301, + "logits/rejected": 1.316011905670166, + "logps/chosen": -1.4429469108581543, + "logps/rejected": -17.469158172607422, + "loss": 3.3379, + "nll_loss": 3.3311076164245605, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14429470896720886, + "rewards/margins": 1.6026211977005005, + "rewards/rejected": -1.7469159364700317, + "step": 1036 + }, + { + "epoch": 0.6451010886469674, + "grad_norm": 0.4014487564563751, + "learning_rate": 2.4075e-05, + "log_odds_chosen": 10.295687675476074, + "log_odds_ratio": -0.055723220109939575, + "logits/chosen": 0.09633732587099075, + "logits/rejected": 0.6330247521400452, + "logps/chosen": -1.3109146356582642, + "logps/rejected": -11.317893981933594, + "loss": 3.0171, + "nll_loss": 3.0115628242492676, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13109146058559418, + "rewards/margins": 1.0006979703903198, + "rewards/rejected": -1.1317894458770752, + "step": 1037 + }, + { + "epoch": 0.6457231726283048, + "grad_norm": 0.5607584118843079, + "learning_rate": 2.4050000000000002e-05, + "log_odds_chosen": 18.80658721923828, + "log_odds_ratio": -0.023404287174344063, + "logits/chosen": 0.39442214369773865, + "logits/rejected": 1.445420265197754, + "logps/chosen": -1.073487639427185, + "logps/rejected": -19.431781768798828, + "loss": 2.8651, + "nll_loss": 2.8627424240112305, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10734876245260239, + "rewards/margins": 1.8358293771743774, + "rewards/rejected": -1.9431781768798828, + "step": 1038 + }, + { + "epoch": 0.6463452566096423, + "grad_norm": 0.41965922713279724, + "learning_rate": 2.4025e-05, + "log_odds_chosen": 15.010441780090332, + "log_odds_ratio": -0.049356162548065186, + "logits/chosen": 0.258677214384079, + "logits/rejected": 1.031904935836792, + "logps/chosen": -1.0338720083236694, + "logps/rejected": -15.611568450927734, + "loss": 3.118, + "nll_loss": 3.1130309104919434, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10338720679283142, + "rewards/margins": 1.4577698707580566, + "rewards/rejected": -1.561156988143921, + "step": 1039 + }, + { + "epoch": 0.6469673405909798, + "grad_norm": 0.36550575494766235, + "learning_rate": 2.4e-05, + "log_odds_chosen": 14.48094367980957, + "log_odds_ratio": -0.1260167956352234, + "logits/chosen": 0.1283513307571411, + "logits/rejected": 0.8361437320709229, + "logps/chosen": -1.0468213558197021, + "logps/rejected": -14.914956092834473, + "loss": 3.0304, + "nll_loss": 3.0178399085998535, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10468213260173798, + "rewards/margins": 1.3868136405944824, + "rewards/rejected": -1.4914956092834473, + "step": 1040 + }, + { + "epoch": 0.6475894245723173, + "grad_norm": 1.547513484954834, + "learning_rate": 2.3975e-05, + "log_odds_chosen": 12.045820236206055, + "log_odds_ratio": -0.22973600029945374, + "logits/chosen": 0.1489129662513733, + "logits/rejected": 0.9729058146476746, + "logps/chosen": -1.2251245975494385, + "logps/rejected": -12.936586380004883, + "loss": 2.5435, + "nll_loss": 2.5204789638519287, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12251245975494385, + "rewards/margins": 1.1711461544036865, + "rewards/rejected": -1.2936586141586304, + "step": 1041 + }, + { + "epoch": 0.6482115085536547, + "grad_norm": 0.3640393912792206, + "learning_rate": 2.395e-05, + "log_odds_chosen": 13.0018310546875, + "log_odds_ratio": -0.23523327708244324, + "logits/chosen": 0.10855109244585037, + "logits/rejected": 0.6623581647872925, + "logps/chosen": -1.285058617591858, + "logps/rejected": -14.037286758422852, + "loss": 3.0383, + "nll_loss": 3.014784812927246, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1285058706998825, + "rewards/margins": 1.2752227783203125, + "rewards/rejected": -1.403728723526001, + "step": 1042 + }, + { + "epoch": 0.6488335925349922, + "grad_norm": 0.4505966007709503, + "learning_rate": 2.3925e-05, + "log_odds_chosen": 7.586543083190918, + "log_odds_ratio": -0.278242290019989, + "logits/chosen": 0.1186991035938263, + "logits/rejected": 0.504472017288208, + "logps/chosen": -1.3483988046646118, + "logps/rejected": -8.706100463867188, + "loss": 3.0296, + "nll_loss": 3.0018014907836914, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13483989238739014, + "rewards/margins": 0.7357701659202576, + "rewards/rejected": -0.8706101179122925, + "step": 1043 + }, + { + "epoch": 0.6494556765163297, + "grad_norm": 0.5245290994644165, + "learning_rate": 2.39e-05, + "log_odds_chosen": 8.640314102172852, + "log_odds_ratio": -0.21840831637382507, + "logits/chosen": 0.23793521523475647, + "logits/rejected": 0.5504704713821411, + "logps/chosen": -1.2520349025726318, + "logps/rejected": -9.354804992675781, + "loss": 3.5628, + "nll_loss": 3.540966749191284, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12520349025726318, + "rewards/margins": 0.8102771043777466, + "rewards/rejected": -0.9354805946350098, + "step": 1044 + }, + { + "epoch": 0.6500777604976672, + "grad_norm": 0.5833095908164978, + "learning_rate": 2.3875e-05, + "log_odds_chosen": 10.210502624511719, + "log_odds_ratio": -0.17007872462272644, + "logits/chosen": 0.12730933725833893, + "logits/rejected": 0.5066798329353333, + "logps/chosen": -0.9844697713851929, + "logps/rejected": -10.70468807220459, + "loss": 2.9488, + "nll_loss": 2.9318201541900635, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09844697266817093, + "rewards/margins": 0.9720218777656555, + "rewards/rejected": -1.0704689025878906, + "step": 1045 + }, + { + "epoch": 0.6506998444790046, + "grad_norm": 0.47387048602104187, + "learning_rate": 2.385e-05, + "log_odds_chosen": 8.573151588439941, + "log_odds_ratio": -0.2872176766395569, + "logits/chosen": 0.2543594539165497, + "logits/rejected": 0.5832099914550781, + "logps/chosen": -1.0247888565063477, + "logps/rejected": -9.087888717651367, + "loss": 3.2127, + "nll_loss": 3.184014081954956, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10247889161109924, + "rewards/margins": 0.8063100576400757, + "rewards/rejected": -0.9087889194488525, + "step": 1046 + }, + { + "epoch": 0.6513219284603421, + "grad_norm": 0.4664608836174011, + "learning_rate": 2.3825e-05, + "log_odds_chosen": 7.170677661895752, + "log_odds_ratio": -0.2467111349105835, + "logits/chosen": -0.025047319009900093, + "logits/rejected": 0.4170008897781372, + "logps/chosen": -1.1048283576965332, + "logps/rejected": -7.730550765991211, + "loss": 2.3306, + "nll_loss": 2.305936098098755, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1104828268289566, + "rewards/margins": 0.6625722646713257, + "rewards/rejected": -0.7730551362037659, + "step": 1047 + }, + { + "epoch": 0.6519440124416797, + "grad_norm": 0.5314801931381226, + "learning_rate": 2.38e-05, + "log_odds_chosen": 6.302850246429443, + "log_odds_ratio": -0.18075837194919586, + "logits/chosen": 0.0019352678209543228, + "logits/rejected": 0.21269835531711578, + "logps/chosen": -0.8562182784080505, + "logps/rejected": -6.463325023651123, + "loss": 2.9775, + "nll_loss": 2.959415912628174, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08562182635068893, + "rewards/margins": 0.5607106685638428, + "rewards/rejected": -0.6463325023651123, + "step": 1048 + }, + { + "epoch": 0.6525660964230171, + "grad_norm": 0.5723354816436768, + "learning_rate": 2.3775e-05, + "log_odds_chosen": 6.959724426269531, + "log_odds_ratio": -0.1822098344564438, + "logits/chosen": 0.12127923220396042, + "logits/rejected": 0.36715346574783325, + "logps/chosen": -1.1644635200500488, + "logps/rejected": -7.830971717834473, + "loss": 3.0659, + "nll_loss": 3.0477237701416016, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.116446353495121, + "rewards/margins": 0.6666508913040161, + "rewards/rejected": -0.7830972075462341, + "step": 1049 + }, + { + "epoch": 0.6531881804043546, + "grad_norm": 0.48077571392059326, + "learning_rate": 2.375e-05, + "log_odds_chosen": 10.392155647277832, + "log_odds_ratio": -0.18274815380573273, + "logits/chosen": 0.2417854368686676, + "logits/rejected": 0.8186101913452148, + "logps/chosen": -1.1042566299438477, + "logps/rejected": -11.141618728637695, + "loss": 3.3063, + "nll_loss": 3.288038492202759, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11042565852403641, + "rewards/margins": 1.0037362575531006, + "rewards/rejected": -1.1141619682312012, + "step": 1050 + }, + { + "epoch": 0.6538102643856921, + "grad_norm": 0.46275290846824646, + "learning_rate": 2.3725e-05, + "log_odds_chosen": 8.993810653686523, + "log_odds_ratio": -0.13959504663944244, + "logits/chosen": 0.1534716784954071, + "logits/rejected": 0.7214460372924805, + "logps/chosen": -1.1377089023590088, + "logps/rejected": -9.824823379516602, + "loss": 3.2004, + "nll_loss": 3.186446189880371, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11377088725566864, + "rewards/margins": 0.8687114119529724, + "rewards/rejected": -0.9824823141098022, + "step": 1051 + }, + { + "epoch": 0.6544323483670296, + "grad_norm": 0.4433314800262451, + "learning_rate": 2.37e-05, + "log_odds_chosen": 10.695637702941895, + "log_odds_ratio": -0.15089482069015503, + "logits/chosen": 0.24725496768951416, + "logits/rejected": 0.9210346341133118, + "logps/chosen": -1.2222862243652344, + "logps/rejected": -11.55354118347168, + "loss": 3.217, + "nll_loss": 3.201942205429077, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12222862243652344, + "rewards/margins": 1.0331255197525024, + "rewards/rejected": -1.1553540229797363, + "step": 1052 + }, + { + "epoch": 0.655054432348367, + "grad_norm": 0.41843631863594055, + "learning_rate": 2.3675e-05, + "log_odds_chosen": 15.736959457397461, + "log_odds_ratio": -0.0017072423361241817, + "logits/chosen": 0.06725797802209854, + "logits/rejected": 0.6552459001541138, + "logps/chosen": -1.5394830703735352, + "logps/rejected": -16.988080978393555, + "loss": 3.3547, + "nll_loss": 3.3545265197753906, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15394830703735352, + "rewards/margins": 1.5448598861694336, + "rewards/rejected": -1.698808193206787, + "step": 1053 + }, + { + "epoch": 0.6556765163297045, + "grad_norm": 0.506420373916626, + "learning_rate": 2.365e-05, + "log_odds_chosen": 9.227112770080566, + "log_odds_ratio": -0.047787003219127655, + "logits/chosen": 0.0533127635717392, + "logits/rejected": 0.6818662285804749, + "logps/chosen": -1.3065093755722046, + "logps/rejected": -10.112043380737305, + "loss": 2.7755, + "nll_loss": 2.7706892490386963, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13065093755722046, + "rewards/margins": 0.8805534839630127, + "rewards/rejected": -1.0112043619155884, + "step": 1054 + }, + { + "epoch": 0.656298600311042, + "grad_norm": 0.4978622794151306, + "learning_rate": 2.3624999999999998e-05, + "log_odds_chosen": 12.25826358795166, + "log_odds_ratio": -0.1236209124326706, + "logits/chosen": 0.18020248413085938, + "logits/rejected": 1.1149808168411255, + "logps/chosen": -1.2221086025238037, + "logps/rejected": -13.194238662719727, + "loss": 2.4563, + "nll_loss": 2.443939447402954, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12221086025238037, + "rewards/margins": 1.197213053703308, + "rewards/rejected": -1.3194239139556885, + "step": 1055 + }, + { + "epoch": 0.6569206842923795, + "grad_norm": 0.6258676648139954, + "learning_rate": 2.36e-05, + "log_odds_chosen": 10.127604484558105, + "log_odds_ratio": -0.15316568315029144, + "logits/chosen": 0.2081230878829956, + "logits/rejected": 0.5622198581695557, + "logps/chosen": -1.128497838973999, + "logps/rejected": -10.898340225219727, + "loss": 3.3251, + "nll_loss": 3.309760093688965, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11284978687763214, + "rewards/margins": 0.9769842624664307, + "rewards/rejected": -1.0898339748382568, + "step": 1056 + }, + { + "epoch": 0.6575427682737169, + "grad_norm": 0.5358809232711792, + "learning_rate": 2.3575e-05, + "log_odds_chosen": 14.776838302612305, + "log_odds_ratio": -0.12466295063495636, + "logits/chosen": 0.3303581774234772, + "logits/rejected": 0.9545040130615234, + "logps/chosen": -0.9283601641654968, + "logps/rejected": -15.194375991821289, + "loss": 3.6474, + "nll_loss": 3.6349401473999023, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09283602982759476, + "rewards/margins": 1.4266016483306885, + "rewards/rejected": -1.519437551498413, + "step": 1057 + }, + { + "epoch": 0.6581648522550544, + "grad_norm": 0.5612006783485413, + "learning_rate": 2.355e-05, + "log_odds_chosen": 13.543533325195312, + "log_odds_ratio": -0.1741914451122284, + "logits/chosen": 0.3145029842853546, + "logits/rejected": 1.0952863693237305, + "logps/chosen": -0.9952583909034729, + "logps/rejected": -14.09577465057373, + "loss": 3.4287, + "nll_loss": 3.4113099575042725, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09952583909034729, + "rewards/margins": 1.310051679611206, + "rewards/rejected": -1.409577488899231, + "step": 1058 + }, + { + "epoch": 0.658786936236392, + "grad_norm": 0.5212019085884094, + "learning_rate": 2.3525e-05, + "log_odds_chosen": 11.775590896606445, + "log_odds_ratio": -0.08714871853590012, + "logits/chosen": 0.17237700521945953, + "logits/rejected": 1.0523440837860107, + "logps/chosen": -0.8339164853096008, + "logps/rejected": -11.932096481323242, + "loss": 2.8133, + "nll_loss": 2.8046300411224365, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08339165151119232, + "rewards/margins": 1.1098179817199707, + "rewards/rejected": -1.1932096481323242, + "step": 1059 + }, + { + "epoch": 0.6594090202177294, + "grad_norm": 0.6565141677856445, + "learning_rate": 2.35e-05, + "log_odds_chosen": 11.25783920288086, + "log_odds_ratio": -0.26100558042526245, + "logits/chosen": 0.18183985352516174, + "logits/rejected": 0.8650422096252441, + "logps/chosen": -0.9057870507240295, + "logps/rejected": -11.680928230285645, + "loss": 2.5247, + "nll_loss": 2.498648166656494, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09057871252298355, + "rewards/margins": 1.0775140523910522, + "rewards/rejected": -1.1680928468704224, + "step": 1060 + }, + { + "epoch": 0.6600311041990669, + "grad_norm": 0.4263257384300232, + "learning_rate": 2.3475e-05, + "log_odds_chosen": 17.359153747558594, + "log_odds_ratio": -0.09745100885629654, + "logits/chosen": 0.18176640570163727, + "logits/rejected": 1.2584339380264282, + "logps/chosen": -1.2209086418151855, + "logps/rejected": -18.21963119506836, + "loss": 3.1567, + "nll_loss": 3.146958112716675, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12209086120128632, + "rewards/margins": 1.6998722553253174, + "rewards/rejected": -1.8219630718231201, + "step": 1061 + }, + { + "epoch": 0.6606531881804043, + "grad_norm": 0.41249606013298035, + "learning_rate": 2.345e-05, + "log_odds_chosen": 18.928997039794922, + "log_odds_ratio": -0.001390795805491507, + "logits/chosen": 0.19376930594444275, + "logits/rejected": 1.2592663764953613, + "logps/chosen": -0.9024685621261597, + "logps/rejected": -19.096473693847656, + "loss": 2.7631, + "nll_loss": 2.7629570960998535, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09024685621261597, + "rewards/margins": 1.8194003105163574, + "rewards/rejected": -1.9096472263336182, + "step": 1062 + }, + { + "epoch": 0.6612752721617419, + "grad_norm": 0.4502386152744293, + "learning_rate": 2.3425000000000004e-05, + "log_odds_chosen": 12.114904403686523, + "log_odds_ratio": -0.1213536337018013, + "logits/chosen": 0.24823647737503052, + "logits/rejected": 1.1324775218963623, + "logps/chosen": -1.1689949035644531, + "logps/rejected": -12.937753677368164, + "loss": 3.2664, + "nll_loss": 3.254295587539673, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1168995052576065, + "rewards/margins": 1.1768758296966553, + "rewards/rejected": -1.2937753200531006, + "step": 1063 + }, + { + "epoch": 0.6618973561430793, + "grad_norm": 0.5948436856269836, + "learning_rate": 2.3400000000000003e-05, + "log_odds_chosen": 9.675779342651367, + "log_odds_ratio": -0.15184316039085388, + "logits/chosen": 0.07025956362485886, + "logits/rejected": 0.536703884601593, + "logps/chosen": -1.4402323961257935, + "logps/rejected": -10.905509948730469, + "loss": 2.9341, + "nll_loss": 2.9189584255218506, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14402325451374054, + "rewards/margins": 0.946527898311615, + "rewards/rejected": -1.0905512571334839, + "step": 1064 + }, + { + "epoch": 0.6625194401244168, + "grad_norm": 1.291369080543518, + "learning_rate": 2.3375000000000002e-05, + "log_odds_chosen": 15.299062728881836, + "log_odds_ratio": -0.10372968018054962, + "logits/chosen": 0.14694362878799438, + "logits/rejected": 1.0618005990982056, + "logps/chosen": -1.1700985431671143, + "logps/rejected": -16.039615631103516, + "loss": 2.9016, + "nll_loss": 2.8912243843078613, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11700985580682755, + "rewards/margins": 1.4869518280029297, + "rewards/rejected": -1.603961706161499, + "step": 1065 + }, + { + "epoch": 0.6631415241057543, + "grad_norm": 0.41034582257270813, + "learning_rate": 2.3350000000000002e-05, + "log_odds_chosen": 10.85059642791748, + "log_odds_ratio": -0.2492515593767166, + "logits/chosen": 0.3745499849319458, + "logits/rejected": 0.9561654329299927, + "logps/chosen": -1.2551486492156982, + "logps/rejected": -11.893285751342773, + "loss": 3.7877, + "nll_loss": 3.762781858444214, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12551486492156982, + "rewards/margins": 1.0638136863708496, + "rewards/rejected": -1.1893285512924194, + "step": 1066 + }, + { + "epoch": 0.6637636080870918, + "grad_norm": 0.42354220151901245, + "learning_rate": 2.3325e-05, + "log_odds_chosen": 17.378068923950195, + "log_odds_ratio": -0.13644790649414062, + "logits/chosen": 0.24174179136753082, + "logits/rejected": 1.284676432609558, + "logps/chosen": -1.274251937866211, + "logps/rejected": -18.352718353271484, + "loss": 3.082, + "nll_loss": 3.0684027671813965, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1274251937866211, + "rewards/margins": 1.7078466415405273, + "rewards/rejected": -1.8352718353271484, + "step": 1067 + }, + { + "epoch": 0.6643856920684292, + "grad_norm": 1.2665324211120605, + "learning_rate": 2.3300000000000004e-05, + "log_odds_chosen": 11.014419555664062, + "log_odds_ratio": -0.06745558232069016, + "logits/chosen": 0.19637992978096008, + "logits/rejected": 0.9851882457733154, + "logps/chosen": -1.110511302947998, + "logps/rejected": -11.721153259277344, + "loss": 3.3051, + "nll_loss": 3.298313617706299, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11105112731456757, + "rewards/margins": 1.0610642433166504, + "rewards/rejected": -1.1721153259277344, + "step": 1068 + }, + { + "epoch": 0.6650077760497667, + "grad_norm": 0.42871132493019104, + "learning_rate": 2.3275000000000003e-05, + "log_odds_chosen": 19.26378631591797, + "log_odds_ratio": -0.00020241702441126108, + "logits/chosen": 0.28503745794296265, + "logits/rejected": 1.4887564182281494, + "logps/chosen": -1.0176734924316406, + "logps/rejected": -19.808406829833984, + "loss": 3.1755, + "nll_loss": 3.175529718399048, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10176734626293182, + "rewards/margins": 1.8790735006332397, + "rewards/rejected": -1.980840802192688, + "step": 1069 + }, + { + "epoch": 0.6656298600311042, + "grad_norm": 1.2943010330200195, + "learning_rate": 2.3250000000000003e-05, + "log_odds_chosen": 11.460484504699707, + "log_odds_ratio": -0.30275437235832214, + "logits/chosen": 0.11974619328975677, + "logits/rejected": 0.6861406564712524, + "logps/chosen": -0.9381018280982971, + "logps/rejected": -11.976990699768066, + "loss": 2.6535, + "nll_loss": 2.623176097869873, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09381018579006195, + "rewards/margins": 1.103888988494873, + "rewards/rejected": -1.1976991891860962, + "step": 1070 + }, + { + "epoch": 0.6662519440124417, + "grad_norm": 4.658538818359375, + "learning_rate": 2.3225000000000002e-05, + "log_odds_chosen": 14.916189193725586, + "log_odds_ratio": -0.1716422736644745, + "logits/chosen": 0.19798047840595245, + "logits/rejected": 0.8838070631027222, + "logps/chosen": -1.6107474565505981, + "logps/rejected": -15.790814399719238, + "loss": 3.1172, + "nll_loss": 3.100071907043457, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16107475757598877, + "rewards/margins": 1.4180067777633667, + "rewards/rejected": -1.5790815353393555, + "step": 1071 + }, + { + "epoch": 0.6668740279937792, + "grad_norm": 0.5105116963386536, + "learning_rate": 2.32e-05, + "log_odds_chosen": 14.650772094726562, + "log_odds_ratio": -0.007171155884861946, + "logits/chosen": 0.15524886548519135, + "logits/rejected": 1.0009796619415283, + "logps/chosen": -1.2726460695266724, + "logps/rejected": -15.550126075744629, + "loss": 2.8389, + "nll_loss": 2.838151454925537, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1272645890712738, + "rewards/margins": 1.4277479648590088, + "rewards/rejected": -1.5550127029418945, + "step": 1072 + }, + { + "epoch": 0.6674961119751166, + "grad_norm": 0.6901617050170898, + "learning_rate": 2.3175e-05, + "log_odds_chosen": 7.709420680999756, + "log_odds_ratio": -0.05205727741122246, + "logits/chosen": 0.14403298497200012, + "logits/rejected": 0.5285943746566772, + "logps/chosen": -1.155900001525879, + "logps/rejected": -8.388169288635254, + "loss": 3.0981, + "nll_loss": 3.092885971069336, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11559000611305237, + "rewards/margins": 0.7232269048690796, + "rewards/rejected": -0.8388169407844543, + "step": 1073 + }, + { + "epoch": 0.6681181959564542, + "grad_norm": 0.5226415991783142, + "learning_rate": 2.3150000000000004e-05, + "log_odds_chosen": 14.16978645324707, + "log_odds_ratio": -0.10162307322025299, + "logits/chosen": 0.1284743696451187, + "logits/rejected": 0.8283305168151855, + "logps/chosen": -1.5021108388900757, + "logps/rejected": -15.423286437988281, + "loss": 3.2585, + "nll_loss": 3.248311996459961, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15021109580993652, + "rewards/margins": 1.3921175003051758, + "rewards/rejected": -1.5423285961151123, + "step": 1074 + }, + { + "epoch": 0.6687402799377916, + "grad_norm": 1.8207690715789795, + "learning_rate": 2.3125000000000003e-05, + "log_odds_chosen": 16.029922485351562, + "log_odds_ratio": -5.97071266383864e-05, + "logits/chosen": 0.06656906008720398, + "logits/rejected": 0.8557884097099304, + "logps/chosen": -1.1395089626312256, + "logps/rejected": -16.5898494720459, + "loss": 2.7598, + "nll_loss": 2.7598206996917725, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11395090073347092, + "rewards/margins": 1.5450341701507568, + "rewards/rejected": -1.6589850187301636, + "step": 1075 + }, + { + "epoch": 0.6693623639191291, + "grad_norm": 0.6128720045089722, + "learning_rate": 2.3100000000000002e-05, + "log_odds_chosen": 7.803811550140381, + "log_odds_ratio": -0.16541123390197754, + "logits/chosen": 0.04209950566291809, + "logits/rejected": 0.5215015411376953, + "logps/chosen": -1.3678196668624878, + "logps/rejected": -8.899198532104492, + "loss": 2.797, + "nll_loss": 2.780494213104248, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1367819756269455, + "rewards/margins": 0.7531379461288452, + "rewards/rejected": -0.8899198174476624, + "step": 1076 + }, + { + "epoch": 0.6699844479004665, + "grad_norm": 0.533178448677063, + "learning_rate": 2.3075000000000002e-05, + "log_odds_chosen": 5.934687614440918, + "log_odds_ratio": -0.3675815463066101, + "logits/chosen": 0.07074688374996185, + "logits/rejected": 0.4116901159286499, + "logps/chosen": -1.1007202863693237, + "logps/rejected": -6.782163619995117, + "loss": 2.5115, + "nll_loss": 2.4747588634490967, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11007203161716461, + "rewards/margins": 0.5681442618370056, + "rewards/rejected": -0.6782163381576538, + "step": 1077 + }, + { + "epoch": 0.6706065318818041, + "grad_norm": 0.5132092833518982, + "learning_rate": 2.305e-05, + "log_odds_chosen": 11.052836418151855, + "log_odds_ratio": -0.2220580130815506, + "logits/chosen": 0.11277395486831665, + "logits/rejected": 0.8609932661056519, + "logps/chosen": -1.1415894031524658, + "logps/rejected": -11.936271667480469, + "loss": 2.4863, + "nll_loss": 2.4641189575195312, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11415895819664001, + "rewards/margins": 1.0794682502746582, + "rewards/rejected": -1.1936272382736206, + "step": 1078 + }, + { + "epoch": 0.6712286158631415, + "grad_norm": 0.5379019379615784, + "learning_rate": 2.3025e-05, + "log_odds_chosen": 12.296998977661133, + "log_odds_ratio": -0.10660477727651596, + "logits/chosen": 0.14019262790679932, + "logits/rejected": 0.8425007462501526, + "logps/chosen": -1.435052514076233, + "logps/rejected": -13.453621864318848, + "loss": 3.4839, + "nll_loss": 3.473271369934082, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14350526034832, + "rewards/margins": 1.2018568515777588, + "rewards/rejected": -1.3453621864318848, + "step": 1079 + }, + { + "epoch": 0.671850699844479, + "grad_norm": 0.46937939524650574, + "learning_rate": 2.3000000000000003e-05, + "log_odds_chosen": 12.263298034667969, + "log_odds_ratio": -0.24854573607444763, + "logits/chosen": 0.14811545610427856, + "logits/rejected": 0.7356073260307312, + "logps/chosen": -1.1790632009506226, + "logps/rejected": -13.17032527923584, + "loss": 2.9886, + "nll_loss": 2.963712215423584, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11790631711483002, + "rewards/margins": 1.1991263628005981, + "rewards/rejected": -1.3170325756072998, + "step": 1080 + }, + { + "epoch": 0.6724727838258164, + "grad_norm": 0.48714011907577515, + "learning_rate": 2.2975000000000003e-05, + "log_odds_chosen": 5.6596174240112305, + "log_odds_ratio": -0.5009754300117493, + "logits/chosen": 0.10506222397089005, + "logits/rejected": 0.18131209909915924, + "logps/chosen": -1.3187544345855713, + "logps/rejected": -6.899771690368652, + "loss": 3.4112, + "nll_loss": 3.361135959625244, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1318754404783249, + "rewards/margins": 0.5581017732620239, + "rewards/rejected": -0.6899771690368652, + "step": 1081 + }, + { + "epoch": 0.673094867807154, + "grad_norm": 0.5086039900779724, + "learning_rate": 2.2950000000000002e-05, + "log_odds_chosen": 10.907814979553223, + "log_odds_ratio": -0.08033996820449829, + "logits/chosen": 0.12853284180164337, + "logits/rejected": 0.823364794254303, + "logps/chosen": -1.0794905424118042, + "logps/rejected": -11.466302871704102, + "loss": 2.8303, + "nll_loss": 2.8222546577453613, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10794906318187714, + "rewards/margins": 1.038681149482727, + "rewards/rejected": -1.1466301679611206, + "step": 1082 + }, + { + "epoch": 0.6737169517884914, + "grad_norm": 0.4580300748348236, + "learning_rate": 2.2925e-05, + "log_odds_chosen": 16.086458206176758, + "log_odds_ratio": -0.00020257926371414214, + "logits/chosen": 0.13432970643043518, + "logits/rejected": 1.132068395614624, + "logps/chosen": -1.136250376701355, + "logps/rejected": -16.81014633178711, + "loss": 2.7228, + "nll_loss": 2.722776412963867, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11362504959106445, + "rewards/margins": 1.567389726638794, + "rewards/rejected": -1.6810145378112793, + "step": 1083 + }, + { + "epoch": 0.6743390357698289, + "grad_norm": 1.2523407936096191, + "learning_rate": 2.29e-05, + "log_odds_chosen": 10.811274528503418, + "log_odds_ratio": -0.3882692754268646, + "logits/chosen": 0.09717198461294174, + "logits/rejected": 0.5859137773513794, + "logps/chosen": -1.5438910722732544, + "logps/rejected": -12.124390602111816, + "loss": 2.8007, + "nll_loss": 2.7619214057922363, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15438911318778992, + "rewards/margins": 1.0580499172210693, + "rewards/rejected": -1.2124391794204712, + "step": 1084 + }, + { + "epoch": 0.6749611197511665, + "grad_norm": 0.5664082765579224, + "learning_rate": 2.2875e-05, + "log_odds_chosen": 12.99592113494873, + "log_odds_ratio": -0.15935920178890228, + "logits/chosen": 0.15744078159332275, + "logits/rejected": 0.6450719237327576, + "logps/chosen": -0.8572717905044556, + "logps/rejected": -13.253767013549805, + "loss": 3.2876, + "nll_loss": 3.2716193199157715, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08572718501091003, + "rewards/margins": 1.2396494150161743, + "rewards/rejected": -1.3253766298294067, + "step": 1085 + }, + { + "epoch": 0.6755832037325039, + "grad_norm": 0.8332890272140503, + "learning_rate": 2.2850000000000003e-05, + "log_odds_chosen": 9.230619430541992, + "log_odds_ratio": -0.4146743416786194, + "logits/chosen": 0.136587455868721, + "logits/rejected": 0.7445818781852722, + "logps/chosen": -1.5004432201385498, + "logps/rejected": -10.546527862548828, + "loss": 3.4476, + "nll_loss": 3.4061646461486816, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15004432201385498, + "rewards/margins": 0.9046083688735962, + "rewards/rejected": -1.0546526908874512, + "step": 1086 + }, + { + "epoch": 0.6762052877138414, + "grad_norm": 0.5611684322357178, + "learning_rate": 2.2825000000000003e-05, + "log_odds_chosen": 7.212390899658203, + "log_odds_ratio": -0.253339022397995, + "logits/chosen": 0.09394458681344986, + "logits/rejected": 0.35125231742858887, + "logps/chosen": -0.6430901288986206, + "logps/rejected": -7.047637939453125, + "loss": 2.771, + "nll_loss": 2.745687961578369, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0643090084195137, + "rewards/margins": 0.6404548287391663, + "rewards/rejected": -0.7047637701034546, + "step": 1087 + }, + { + "epoch": 0.6768273716951788, + "grad_norm": 0.5705518126487732, + "learning_rate": 2.2800000000000002e-05, + "log_odds_chosen": 10.540750503540039, + "log_odds_ratio": -0.1235579401254654, + "logits/chosen": 0.221440851688385, + "logits/rejected": 0.7182592153549194, + "logps/chosen": -1.1119794845581055, + "logps/rejected": -11.2672119140625, + "loss": 3.233, + "nll_loss": 3.2206497192382812, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11119795590639114, + "rewards/margins": 1.0155231952667236, + "rewards/rejected": -1.1267211437225342, + "step": 1088 + }, + { + "epoch": 0.6774494556765164, + "grad_norm": 0.4467851221561432, + "learning_rate": 2.2775e-05, + "log_odds_chosen": 16.156049728393555, + "log_odds_ratio": -0.2083779126405716, + "logits/chosen": 0.13699352741241455, + "logits/rejected": 0.7858900427818298, + "logps/chosen": -1.272729754447937, + "logps/rejected": -17.19986343383789, + "loss": 2.9677, + "nll_loss": 2.946847915649414, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12727296352386475, + "rewards/margins": 1.592713475227356, + "rewards/rejected": -1.7199864387512207, + "step": 1089 + }, + { + "epoch": 0.6780715396578538, + "grad_norm": 0.5542477369308472, + "learning_rate": 2.275e-05, + "log_odds_chosen": 5.15977144241333, + "log_odds_ratio": -0.3942872881889343, + "logits/chosen": 0.15950456261634827, + "logits/rejected": 0.2518439292907715, + "logps/chosen": -1.4990637302398682, + "logps/rejected": -6.542140483856201, + "loss": 3.1346, + "nll_loss": 3.0951952934265137, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.14990636706352234, + "rewards/margins": 0.5043076872825623, + "rewards/rejected": -0.6542140245437622, + "step": 1090 + }, + { + "epoch": 0.6786936236391913, + "grad_norm": 0.5533297061920166, + "learning_rate": 2.2725000000000003e-05, + "log_odds_chosen": 10.516497611999512, + "log_odds_ratio": -0.18116550147533417, + "logits/chosen": 0.15582889318466187, + "logits/rejected": 0.7493836879730225, + "logps/chosen": -1.2090470790863037, + "logps/rejected": -11.461071014404297, + "loss": 3.2506, + "nll_loss": 3.232510805130005, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12090469896793365, + "rewards/margins": 1.0252025127410889, + "rewards/rejected": -1.1461071968078613, + "step": 1091 + }, + { + "epoch": 0.6793157076205287, + "grad_norm": 0.44804054498672485, + "learning_rate": 2.2700000000000003e-05, + "log_odds_chosen": 9.345958709716797, + "log_odds_ratio": -0.0998968705534935, + "logits/chosen": 0.10394829511642456, + "logits/rejected": 0.22612932324409485, + "logps/chosen": -0.6652541160583496, + "logps/rejected": -9.150422096252441, + "loss": 3.0124, + "nll_loss": 3.0023632049560547, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0665254145860672, + "rewards/margins": 0.8485168218612671, + "rewards/rejected": -0.9150421619415283, + "step": 1092 + }, + { + "epoch": 0.6799377916018663, + "grad_norm": 0.45218420028686523, + "learning_rate": 2.2675000000000002e-05, + "log_odds_chosen": 11.6668062210083, + "log_odds_ratio": -0.08176952600479126, + "logits/chosen": 0.3411112129688263, + "logits/rejected": 1.1545166969299316, + "logps/chosen": -0.9080636501312256, + "logps/rejected": -11.78917407989502, + "loss": 3.5003, + "nll_loss": 3.492164134979248, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09080636501312256, + "rewards/margins": 1.088111162185669, + "rewards/rejected": -1.1789175271987915, + "step": 1093 + }, + { + "epoch": 0.6805598755832037, + "grad_norm": 0.6241004467010498, + "learning_rate": 2.265e-05, + "log_odds_chosen": 6.734440803527832, + "log_odds_ratio": -0.2348792403936386, + "logits/chosen": 0.11519190669059753, + "logits/rejected": 0.5573137998580933, + "logps/chosen": -1.2351096868515015, + "logps/rejected": -7.5556793212890625, + "loss": 2.7045, + "nll_loss": 2.681022882461548, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12351097166538239, + "rewards/margins": 0.6320569515228271, + "rewards/rejected": -0.7555679082870483, + "step": 1094 + }, + { + "epoch": 0.6811819595645412, + "grad_norm": 2.355020761489868, + "learning_rate": 2.2625e-05, + "log_odds_chosen": 13.27316665649414, + "log_odds_ratio": -0.13957035541534424, + "logits/chosen": 0.24990928173065186, + "logits/rejected": 0.8357008099555969, + "logps/chosen": -1.2236608266830444, + "logps/rejected": -14.109601020812988, + "loss": 3.6854, + "nll_loss": 3.6714515686035156, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12236608564853668, + "rewards/margins": 1.2885940074920654, + "rewards/rejected": -1.410960078239441, + "step": 1095 + }, + { + "epoch": 0.6818040435458786, + "grad_norm": 0.43016043305397034, + "learning_rate": 2.26e-05, + "log_odds_chosen": 13.364396095275879, + "log_odds_ratio": -0.10089391469955444, + "logits/chosen": 0.12420229613780975, + "logits/rejected": 0.6156033277511597, + "logps/chosen": -1.0751311779022217, + "logps/rejected": -13.820520401000977, + "loss": 3.2673, + "nll_loss": 3.2571640014648438, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10751312971115112, + "rewards/margins": 1.2745389938354492, + "rewards/rejected": -1.3820520639419556, + "step": 1096 + }, + { + "epoch": 0.6824261275272162, + "grad_norm": 0.7112170457839966, + "learning_rate": 2.2575000000000003e-05, + "log_odds_chosen": 15.50126838684082, + "log_odds_ratio": -0.17970341444015503, + "logits/chosen": 0.13599060475826263, + "logits/rejected": 0.8270803689956665, + "logps/chosen": -1.0619562864303589, + "logps/rejected": -16.088138580322266, + "loss": 2.6784, + "nll_loss": 2.6603970527648926, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10619563609361649, + "rewards/margins": 1.5026183128356934, + "rewards/rejected": -1.6088138818740845, + "step": 1097 + }, + { + "epoch": 0.6830482115085537, + "grad_norm": 0.5061715841293335, + "learning_rate": 2.2550000000000003e-05, + "log_odds_chosen": 13.113398551940918, + "log_odds_ratio": -0.12759830057621002, + "logits/chosen": 0.16755160689353943, + "logits/rejected": 0.8419879674911499, + "logps/chosen": -1.0674017667770386, + "logps/rejected": -13.659807205200195, + "loss": 2.6941, + "nll_loss": 2.681312322616577, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10674017667770386, + "rewards/margins": 1.259240746498108, + "rewards/rejected": -1.365980863571167, + "step": 1098 + }, + { + "epoch": 0.6836702954898911, + "grad_norm": 0.4506334960460663, + "learning_rate": 2.2525000000000002e-05, + "log_odds_chosen": 11.249850273132324, + "log_odds_ratio": -0.1481650024652481, + "logits/chosen": 0.16708631813526154, + "logits/rejected": 0.6881266832351685, + "logps/chosen": -1.1782821416854858, + "logps/rejected": -12.08417797088623, + "loss": 3.056, + "nll_loss": 3.041147232055664, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11782821267843246, + "rewards/margins": 1.0905896425247192, + "rewards/rejected": -1.2084178924560547, + "step": 1099 + }, + { + "epoch": 0.6842923794712286, + "grad_norm": 0.47914910316467285, + "learning_rate": 2.25e-05, + "log_odds_chosen": 13.498701095581055, + "log_odds_ratio": -0.05041101947426796, + "logits/chosen": 0.2943468689918518, + "logits/rejected": 1.0841618776321411, + "logps/chosen": -1.1848316192626953, + "logps/rejected": -14.254444122314453, + "loss": 3.3055, + "nll_loss": 3.3004813194274902, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11848315596580505, + "rewards/margins": 1.3069612979888916, + "rewards/rejected": -1.4254443645477295, + "step": 1100 + }, + { + "epoch": 0.6849144634525661, + "grad_norm": 0.46730777621269226, + "learning_rate": 2.2475e-05, + "log_odds_chosen": 13.177154541015625, + "log_odds_ratio": -0.20132030546665192, + "logits/chosen": 0.19598224759101868, + "logits/rejected": 0.6804916262626648, + "logps/chosen": -1.008966088294983, + "logps/rejected": -13.777807235717773, + "loss": 3.1359, + "nll_loss": 3.1157898902893066, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10089661180973053, + "rewards/margins": 1.2768840789794922, + "rewards/rejected": -1.3777806758880615, + "step": 1101 + }, + { + "epoch": 0.6855365474339036, + "grad_norm": 0.5543920397758484, + "learning_rate": 2.245e-05, + "log_odds_chosen": 13.08626651763916, + "log_odds_ratio": -0.08596739172935486, + "logits/chosen": 0.12830127775669098, + "logits/rejected": 0.7522597312927246, + "logps/chosen": -1.0769314765930176, + "logps/rejected": -13.698116302490234, + "loss": 2.7265, + "nll_loss": 2.7178595066070557, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1076931580901146, + "rewards/margins": 1.2621186971664429, + "rewards/rejected": -1.369811773300171, + "step": 1102 + }, + { + "epoch": 0.686158631415241, + "grad_norm": 0.5758435130119324, + "learning_rate": 2.2425000000000003e-05, + "log_odds_chosen": 14.17343521118164, + "log_odds_ratio": -0.10952825099229813, + "logits/chosen": 0.2742041349411011, + "logits/rejected": 1.0324076414108276, + "logps/chosen": -1.4365853071212769, + "logps/rejected": -15.37224006652832, + "loss": 2.8356, + "nll_loss": 2.8246562480926514, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14365853369235992, + "rewards/margins": 1.3935655355453491, + "rewards/rejected": -1.5372240543365479, + "step": 1103 + }, + { + "epoch": 0.6867807153965786, + "grad_norm": 0.36500388383865356, + "learning_rate": 2.2400000000000002e-05, + "log_odds_chosen": 16.786182403564453, + "log_odds_ratio": -0.06503584235906601, + "logits/chosen": 0.3020685911178589, + "logits/rejected": 1.4873595237731934, + "logps/chosen": -1.0615830421447754, + "logps/rejected": -17.41973876953125, + "loss": 3.3075, + "nll_loss": 3.3009724617004395, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1061583086848259, + "rewards/margins": 1.6358156204223633, + "rewards/rejected": -1.741973876953125, + "step": 1104 + }, + { + "epoch": 0.687402799377916, + "grad_norm": 0.49695879220962524, + "learning_rate": 2.2375000000000002e-05, + "log_odds_chosen": 20.125638961791992, + "log_odds_ratio": -6.066138666938059e-05, + "logits/chosen": 0.2225799411535263, + "logits/rejected": 1.438215970993042, + "logps/chosen": -1.3540451526641846, + "logps/rejected": -21.168962478637695, + "loss": 3.1225, + "nll_loss": 3.1224632263183594, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1354045271873474, + "rewards/margins": 1.9814918041229248, + "rewards/rejected": -2.116896390914917, + "step": 1105 + }, + { + "epoch": 0.6880248833592535, + "grad_norm": 0.6216310262680054, + "learning_rate": 2.235e-05, + "log_odds_chosen": 15.467144012451172, + "log_odds_ratio": -0.0264641921967268, + "logits/chosen": 0.22354130446910858, + "logits/rejected": 0.8883647918701172, + "logps/chosen": -0.950381338596344, + "logps/rejected": -15.762426376342773, + "loss": 3.0147, + "nll_loss": 3.012061834335327, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09503813087940216, + "rewards/margins": 1.481204628944397, + "rewards/rejected": -1.5762428045272827, + "step": 1106 + }, + { + "epoch": 0.6886469673405909, + "grad_norm": 0.6510394811630249, + "learning_rate": 2.2325e-05, + "log_odds_chosen": 15.790122032165527, + "log_odds_ratio": -0.0470377579331398, + "logits/chosen": 0.07415720820426941, + "logits/rejected": 0.7188281416893005, + "logps/chosen": -1.2903048992156982, + "logps/rejected": -16.766942977905273, + "loss": 2.9234, + "nll_loss": 2.918691635131836, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1290304958820343, + "rewards/margins": 1.547663927078247, + "rewards/rejected": -1.676694393157959, + "step": 1107 + }, + { + "epoch": 0.6892690513219285, + "grad_norm": 0.7006060481071472, + "learning_rate": 2.23e-05, + "log_odds_chosen": 14.18627643585205, + "log_odds_ratio": -0.14766886830329895, + "logits/chosen": 0.2061242163181305, + "logits/rejected": 0.6048423051834106, + "logps/chosen": -1.2005126476287842, + "logps/rejected": -15.043365478515625, + "loss": 3.36, + "nll_loss": 3.345226526260376, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12005127221345901, + "rewards/margins": 1.3842853307724, + "rewards/rejected": -1.5043364763259888, + "step": 1108 + }, + { + "epoch": 0.689891135303266, + "grad_norm": 11.92190933227539, + "learning_rate": 2.2275000000000003e-05, + "log_odds_chosen": 17.27753448486328, + "log_odds_ratio": -0.09403269737958908, + "logits/chosen": 0.3899608850479126, + "logits/rejected": 1.604209303855896, + "logps/chosen": -1.9780341386795044, + "logps/rejected": -18.90896224975586, + "loss": 3.4092, + "nll_loss": 3.3998172283172607, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.19780340790748596, + "rewards/margins": 1.6930928230285645, + "rewards/rejected": -1.890896201133728, + "step": 1109 + }, + { + "epoch": 0.6905132192846034, + "grad_norm": 0.5367807745933533, + "learning_rate": 2.2250000000000002e-05, + "log_odds_chosen": 11.975065231323242, + "log_odds_ratio": -0.1697680503129959, + "logits/chosen": 0.1529502123594284, + "logits/rejected": 1.010935664176941, + "logps/chosen": -1.0631258487701416, + "logps/rejected": -12.682013511657715, + "loss": 2.5578, + "nll_loss": 2.5407814979553223, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1063125804066658, + "rewards/margins": 1.1618887186050415, + "rewards/rejected": -1.2682013511657715, + "step": 1110 + }, + { + "epoch": 0.6911353032659409, + "grad_norm": 0.5301330089569092, + "learning_rate": 2.2225e-05, + "log_odds_chosen": 10.99447250366211, + "log_odds_ratio": -0.19437550008296967, + "logits/chosen": 0.015375887975096703, + "logits/rejected": 0.5180283188819885, + "logps/chosen": -0.95285564661026, + "logps/rejected": -11.372751235961914, + "loss": 2.3729, + "nll_loss": 2.3534395694732666, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.095285564661026, + "rewards/margins": 1.0419895648956299, + "rewards/rejected": -1.1372750997543335, + "step": 1111 + }, + { + "epoch": 0.6917573872472784, + "grad_norm": 0.5872764587402344, + "learning_rate": 2.22e-05, + "log_odds_chosen": 17.98602294921875, + "log_odds_ratio": -0.09409792721271515, + "logits/chosen": 0.24407103657722473, + "logits/rejected": 1.2107287645339966, + "logps/chosen": -0.9928297996520996, + "logps/rejected": -18.3500919342041, + "loss": 2.6225, + "nll_loss": 2.613097667694092, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09928297251462936, + "rewards/margins": 1.735726237297058, + "rewards/rejected": -1.8350093364715576, + "step": 1112 + }, + { + "epoch": 0.6923794712286159, + "grad_norm": 0.5280458331108093, + "learning_rate": 2.2175e-05, + "log_odds_chosen": 9.216509819030762, + "log_odds_ratio": -0.2553660571575165, + "logits/chosen": 0.21463128924369812, + "logits/rejected": 0.5493931174278259, + "logps/chosen": -1.0679031610488892, + "logps/rejected": -9.879371643066406, + "loss": 3.2027, + "nll_loss": 3.1772048473358154, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10679031163454056, + "rewards/margins": 0.8811469078063965, + "rewards/rejected": -0.9879372119903564, + "step": 1113 + }, + { + "epoch": 0.6930015552099533, + "grad_norm": 0.44100382924079895, + "learning_rate": 2.215e-05, + "log_odds_chosen": 14.408587455749512, + "log_odds_ratio": -0.0010577059583738446, + "logits/chosen": 0.05236963927745819, + "logits/rejected": 0.5131943225860596, + "logps/chosen": -1.3580536842346191, + "logps/rejected": -15.343432426452637, + "loss": 3.184, + "nll_loss": 3.1839027404785156, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1358053833246231, + "rewards/margins": 1.3985378742218018, + "rewards/rejected": -1.5343432426452637, + "step": 1114 + }, + { + "epoch": 0.6936236391912908, + "grad_norm": 0.5382852554321289, + "learning_rate": 2.2125000000000002e-05, + "log_odds_chosen": 9.10023307800293, + "log_odds_ratio": -0.30452102422714233, + "logits/chosen": 0.3505043387413025, + "logits/rejected": 1.0762572288513184, + "logps/chosen": -1.0695630311965942, + "logps/rejected": -9.810070037841797, + "loss": 3.3941, + "nll_loss": 3.3636703491210938, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10695630311965942, + "rewards/margins": 0.8740507364273071, + "rewards/rejected": -0.9810069799423218, + "step": 1115 + }, + { + "epoch": 0.6942457231726283, + "grad_norm": 0.44020068645477295, + "learning_rate": 2.2100000000000002e-05, + "log_odds_chosen": 14.810802459716797, + "log_odds_ratio": -0.10710399597883224, + "logits/chosen": 0.11335402727127075, + "logits/rejected": 0.7006421089172363, + "logps/chosen": -1.4732582569122314, + "logps/rejected": -15.886306762695312, + "loss": 2.9517, + "nll_loss": 2.9409916400909424, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1473258137702942, + "rewards/margins": 1.4413048028945923, + "rewards/rejected": -1.5886306762695312, + "step": 1116 + }, + { + "epoch": 0.6948678071539658, + "grad_norm": 0.6491237282752991, + "learning_rate": 2.2075e-05, + "log_odds_chosen": 8.861434936523438, + "log_odds_ratio": -0.09095767885446548, + "logits/chosen": 0.22493115067481995, + "logits/rejected": 0.7060920000076294, + "logps/chosen": -1.0282622575759888, + "logps/rejected": -9.425237655639648, + "loss": 3.1569, + "nll_loss": 3.1477737426757812, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10282622277736664, + "rewards/margins": 0.8396974802017212, + "rewards/rejected": -0.942523717880249, + "step": 1117 + }, + { + "epoch": 0.6954898911353032, + "grad_norm": 0.5764707326889038, + "learning_rate": 2.205e-05, + "log_odds_chosen": 7.087107181549072, + "log_odds_ratio": -0.33189916610717773, + "logits/chosen": 0.14153975248336792, + "logits/rejected": 0.653849720954895, + "logps/chosen": -1.2045460939407349, + "logps/rejected": -8.065446853637695, + "loss": 3.0227, + "nll_loss": 2.9894912242889404, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12045460939407349, + "rewards/margins": 0.6860900521278381, + "rewards/rejected": -0.8065447211265564, + "step": 1118 + }, + { + "epoch": 0.6961119751166407, + "grad_norm": 0.6903092265129089, + "learning_rate": 2.2025e-05, + "log_odds_chosen": 9.988282203674316, + "log_odds_ratio": -0.2689623534679413, + "logits/chosen": 0.18848875164985657, + "logits/rejected": 0.3909173309803009, + "logps/chosen": -1.037498950958252, + "logps/rejected": -10.61701774597168, + "loss": 3.3393, + "nll_loss": 3.3123836517333984, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10374990105628967, + "rewards/margins": 0.9579518437385559, + "rewards/rejected": -1.061701774597168, + "step": 1119 + }, + { + "epoch": 0.6967340590979783, + "grad_norm": 0.5290844440460205, + "learning_rate": 2.2000000000000003e-05, + "log_odds_chosen": 12.99549674987793, + "log_odds_ratio": -0.04361700266599655, + "logits/chosen": 0.2747880816459656, + "logits/rejected": 0.681348443031311, + "logps/chosen": -1.3809425830841064, + "logps/rejected": -13.821148872375488, + "loss": 3.3427, + "nll_loss": 3.3383641242980957, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13809427618980408, + "rewards/margins": 1.244020700454712, + "rewards/rejected": -1.3821148872375488, + "step": 1120 + }, + { + "epoch": 0.6973561430793157, + "grad_norm": 0.4989076256752014, + "learning_rate": 2.1975000000000002e-05, + "log_odds_chosen": 10.855270385742188, + "log_odds_ratio": -0.038061439990997314, + "logits/chosen": 0.10623833537101746, + "logits/rejected": 0.9076218605041504, + "logps/chosen": -1.3005287647247314, + "logps/rejected": -11.753395080566406, + "loss": 2.8766, + "nll_loss": 2.8727827072143555, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1300528645515442, + "rewards/margins": 1.0452866554260254, + "rewards/rejected": -1.1753395795822144, + "step": 1121 + }, + { + "epoch": 0.6979782270606532, + "grad_norm": 0.7603822946548462, + "learning_rate": 2.195e-05, + "log_odds_chosen": 11.2942476272583, + "log_odds_ratio": -0.06391564011573792, + "logits/chosen": 0.22070272266864777, + "logits/rejected": 0.8907159566879272, + "logps/chosen": -1.1149401664733887, + "logps/rejected": -12.02164363861084, + "loss": 2.6533, + "nll_loss": 2.646906852722168, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11149401217699051, + "rewards/margins": 1.0906702280044556, + "rewards/rejected": -1.2021644115447998, + "step": 1122 + }, + { + "epoch": 0.6986003110419907, + "grad_norm": 0.46358850598335266, + "learning_rate": 2.1925e-05, + "log_odds_chosen": 16.277067184448242, + "log_odds_ratio": -0.106183260679245, + "logits/chosen": 0.12363559007644653, + "logits/rejected": 0.7034668326377869, + "logps/chosen": -1.018523931503296, + "logps/rejected": -16.79071807861328, + "loss": 2.6021, + "nll_loss": 2.591519594192505, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1018524020910263, + "rewards/margins": 1.5772193670272827, + "rewards/rejected": -1.6790719032287598, + "step": 1123 + }, + { + "epoch": 0.6992223950233282, + "grad_norm": 0.6551591753959656, + "learning_rate": 2.19e-05, + "log_odds_chosen": 6.512795448303223, + "log_odds_ratio": -0.046365853399038315, + "logits/chosen": 0.25244051218032837, + "logits/rejected": 0.5576794147491455, + "logps/chosen": -1.0957938432693481, + "logps/rejected": -7.111298084259033, + "loss": 2.7942, + "nll_loss": 2.7895545959472656, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10957939177751541, + "rewards/margins": 0.6015503406524658, + "rewards/rejected": -0.7111297249794006, + "step": 1124 + }, + { + "epoch": 0.6998444790046656, + "grad_norm": 0.662030041217804, + "learning_rate": 2.1875e-05, + "log_odds_chosen": 13.827706336975098, + "log_odds_ratio": -0.19535228610038757, + "logits/chosen": 0.3746417462825775, + "logits/rejected": 1.1256225109100342, + "logps/chosen": -1.1726419925689697, + "logps/rejected": -14.544219970703125, + "loss": 2.8034, + "nll_loss": 2.783874273300171, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11726421117782593, + "rewards/margins": 1.3371577262878418, + "rewards/rejected": -1.4544219970703125, + "step": 1125 + }, + { + "epoch": 0.7004665629860031, + "grad_norm": 4.096120357513428, + "learning_rate": 2.1850000000000003e-05, + "log_odds_chosen": 20.036169052124023, + "log_odds_ratio": -0.013790621422231197, + "logits/chosen": 0.20078325271606445, + "logits/rejected": 0.9888613820075989, + "logps/chosen": -0.9814634919166565, + "logps/rejected": -20.45991325378418, + "loss": 1.9901, + "nll_loss": 1.9886776208877563, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09814634919166565, + "rewards/margins": 1.9478448629379272, + "rewards/rejected": -2.0459911823272705, + "step": 1126 + }, + { + "epoch": 0.7010886469673406, + "grad_norm": 0.5734096169471741, + "learning_rate": 2.1825000000000002e-05, + "log_odds_chosen": 13.67259693145752, + "log_odds_ratio": -0.04071996361017227, + "logits/chosen": 0.17778745293617249, + "logits/rejected": 0.9131073951721191, + "logps/chosen": -1.2617847919464111, + "logps/rejected": -14.583654403686523, + "loss": 2.9613, + "nll_loss": 2.9572203159332275, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12617850303649902, + "rewards/margins": 1.3321870565414429, + "rewards/rejected": -1.4583654403686523, + "step": 1127 + }, + { + "epoch": 0.7017107309486781, + "grad_norm": 0.4533398747444153, + "learning_rate": 2.18e-05, + "log_odds_chosen": 10.825969696044922, + "log_odds_ratio": -0.026933560147881508, + "logits/chosen": 0.24595864117145538, + "logits/rejected": 0.6292278170585632, + "logps/chosen": -1.0519578456878662, + "logps/rejected": -11.445229530334473, + "loss": 3.5149, + "nll_loss": 3.512205123901367, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1051957830786705, + "rewards/margins": 1.0393271446228027, + "rewards/rejected": -1.1445229053497314, + "step": 1128 + }, + { + "epoch": 0.7023328149300155, + "grad_norm": 4.797630310058594, + "learning_rate": 2.1775e-05, + "log_odds_chosen": 14.788459777832031, + "log_odds_ratio": -0.03414743393659592, + "logits/chosen": 0.15490153431892395, + "logits/rejected": 0.9039937257766724, + "logps/chosen": -2.220851421356201, + "logps/rejected": -16.61078453063965, + "loss": 2.912, + "nll_loss": 2.908588409423828, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2220851480960846, + "rewards/margins": 1.4389934539794922, + "rewards/rejected": -1.6610784530639648, + "step": 1129 + }, + { + "epoch": 0.702954898911353, + "grad_norm": 2.169843912124634, + "learning_rate": 2.175e-05, + "log_odds_chosen": 10.93464183807373, + "log_odds_ratio": -0.214813232421875, + "logits/chosen": 0.16185230016708374, + "logits/rejected": 0.7838496565818787, + "logps/chosen": -1.3578927516937256, + "logps/rejected": -12.01943588256836, + "loss": 2.6853, + "nll_loss": 2.6637721061706543, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13578927516937256, + "rewards/margins": 1.0661543607711792, + "rewards/rejected": -1.2019436359405518, + "step": 1130 + }, + { + "epoch": 0.7035769828926906, + "grad_norm": 0.8374118804931641, + "learning_rate": 2.1725e-05, + "log_odds_chosen": 13.784649848937988, + "log_odds_ratio": -0.0661187693476677, + "logits/chosen": 0.20649418234825134, + "logits/rejected": 0.9395582675933838, + "logps/chosen": -1.278693437576294, + "logps/rejected": -14.600447654724121, + "loss": 3.1107, + "nll_loss": 3.1040682792663574, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1278693526983261, + "rewards/margins": 1.3321754932403564, + "rewards/rejected": -1.4600448608398438, + "step": 1131 + }, + { + "epoch": 0.704199066874028, + "grad_norm": 0.378974974155426, + "learning_rate": 2.1700000000000002e-05, + "log_odds_chosen": 14.10969352722168, + "log_odds_ratio": -0.10490639507770538, + "logits/chosen": 0.17890948057174683, + "logits/rejected": 0.7284139394760132, + "logps/chosen": -1.0358339548110962, + "logps/rejected": -14.461109161376953, + "loss": 3.1646, + "nll_loss": 3.1540842056274414, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10358339548110962, + "rewards/margins": 1.3425275087356567, + "rewards/rejected": -1.4461108446121216, + "step": 1132 + }, + { + "epoch": 0.7048211508553655, + "grad_norm": 0.41824260354042053, + "learning_rate": 2.1675e-05, + "log_odds_chosen": 11.248109817504883, + "log_odds_ratio": -0.08439905941486359, + "logits/chosen": 0.22522927820682526, + "logits/rejected": 0.8612341284751892, + "logps/chosen": -0.9619340896606445, + "logps/rejected": -11.490693092346191, + "loss": 3.2554, + "nll_loss": 3.2469184398651123, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09619341045618057, + "rewards/margins": 1.0528758764266968, + "rewards/rejected": -1.1490693092346191, + "step": 1133 + }, + { + "epoch": 0.7054432348367029, + "grad_norm": 0.5656015872955322, + "learning_rate": 2.165e-05, + "log_odds_chosen": 9.454839706420898, + "log_odds_ratio": -0.18580016493797302, + "logits/chosen": 0.1890837401151657, + "logits/rejected": 0.6705374717712402, + "logps/chosen": -1.2932785749435425, + "logps/rejected": -10.466346740722656, + "loss": 3.2662, + "nll_loss": 3.2475852966308594, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12932786345481873, + "rewards/margins": 0.9173068404197693, + "rewards/rejected": -1.0466346740722656, + "step": 1134 + }, + { + "epoch": 0.7060653188180405, + "grad_norm": 0.48979511857032776, + "learning_rate": 2.1625e-05, + "log_odds_chosen": 8.302387237548828, + "log_odds_ratio": -0.10199929028749466, + "logits/chosen": 0.2716232240200043, + "logits/rejected": 0.7033010125160217, + "logps/chosen": -1.0194844007492065, + "logps/rejected": -8.780854225158691, + "loss": 2.9983, + "nll_loss": 2.9880552291870117, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10194844752550125, + "rewards/margins": 0.7761369943618774, + "rewards/rejected": -0.8780854940414429, + "step": 1135 + }, + { + "epoch": 0.7066874027993779, + "grad_norm": 0.7489378452301025, + "learning_rate": 2.16e-05, + "log_odds_chosen": 10.226913452148438, + "log_odds_ratio": -0.23562005162239075, + "logits/chosen": 0.32799530029296875, + "logits/rejected": 0.7691959142684937, + "logps/chosen": -1.3166583776474, + "logps/rejected": -11.2990083694458, + "loss": 3.5254, + "nll_loss": 3.501816511154175, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13166584074497223, + "rewards/margins": 0.9982350468635559, + "rewards/rejected": -1.1299008131027222, + "step": 1136 + }, + { + "epoch": 0.7073094867807154, + "grad_norm": 0.4850473701953888, + "learning_rate": 2.1575e-05, + "log_odds_chosen": 7.751132488250732, + "log_odds_ratio": -0.12384605407714844, + "logits/chosen": 0.0676664263010025, + "logits/rejected": 0.3730185627937317, + "logps/chosen": -1.1014970541000366, + "logps/rejected": -8.484350204467773, + "loss": 2.7592, + "nll_loss": 2.746859312057495, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11014970391988754, + "rewards/margins": 0.7382854223251343, + "rewards/rejected": -0.8484350442886353, + "step": 1137 + }, + { + "epoch": 0.7079315707620529, + "grad_norm": 0.5743958950042725, + "learning_rate": 2.1550000000000002e-05, + "log_odds_chosen": 11.897217750549316, + "log_odds_ratio": -0.22582495212554932, + "logits/chosen": 0.14629925787448883, + "logits/rejected": 0.6454797983169556, + "logps/chosen": -1.0044411420822144, + "logps/rejected": -12.506041526794434, + "loss": 3.048, + "nll_loss": 3.025416135787964, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10044410824775696, + "rewards/margins": 1.1501600742340088, + "rewards/rejected": -1.2506040334701538, + "step": 1138 + }, + { + "epoch": 0.7085536547433904, + "grad_norm": 0.38661685585975647, + "learning_rate": 2.1525e-05, + "log_odds_chosen": 8.775971412658691, + "log_odds_ratio": -0.05596862733364105, + "logits/chosen": 0.31263870000839233, + "logits/rejected": 1.1504080295562744, + "logps/chosen": -1.1882120370864868, + "logps/rejected": -9.621572494506836, + "loss": 3.3544, + "nll_loss": 3.3488073348999023, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11882121115922928, + "rewards/margins": 0.8433359861373901, + "rewards/rejected": -0.9621572494506836, + "step": 1139 + }, + { + "epoch": 0.7091757387247278, + "grad_norm": 0.4194515645503998, + "learning_rate": 2.15e-05, + "log_odds_chosen": 7.772745609283447, + "log_odds_ratio": -0.15363647043704987, + "logits/chosen": 0.05815325677394867, + "logits/rejected": 0.4343949854373932, + "logps/chosen": -1.258744478225708, + "logps/rejected": -8.714080810546875, + "loss": 2.8419, + "nll_loss": 2.8265483379364014, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12587445974349976, + "rewards/margins": 0.7455336451530457, + "rewards/rejected": -0.8714081048965454, + "step": 1140 + }, + { + "epoch": 0.7097978227060653, + "grad_norm": 0.5058637261390686, + "learning_rate": 2.1475e-05, + "log_odds_chosen": 8.859088897705078, + "log_odds_ratio": -0.19793754816055298, + "logits/chosen": 0.10581560432910919, + "logits/rejected": 0.7795206904411316, + "logps/chosen": -1.165850043296814, + "logps/rejected": -9.689474105834961, + "loss": 2.6781, + "nll_loss": 2.658310651779175, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11658500134944916, + "rewards/margins": 0.8523623943328857, + "rewards/rejected": -0.9689474105834961, + "step": 1141 + }, + { + "epoch": 0.7104199066874028, + "grad_norm": 0.5065990090370178, + "learning_rate": 2.145e-05, + "log_odds_chosen": 7.223630905151367, + "log_odds_ratio": -0.19884894788265228, + "logits/chosen": 0.01418336108326912, + "logits/rejected": 0.1823025494813919, + "logps/chosen": -1.134019136428833, + "logps/rejected": -7.986211776733398, + "loss": 2.8221, + "nll_loss": 2.802203416824341, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11340190470218658, + "rewards/margins": 0.6852192878723145, + "rewards/rejected": -0.7986211776733398, + "step": 1142 + }, + { + "epoch": 0.7110419906687403, + "grad_norm": 1.9043158292770386, + "learning_rate": 2.1425e-05, + "log_odds_chosen": 7.798857688903809, + "log_odds_ratio": -0.3148168623447418, + "logits/chosen": -0.015639889985322952, + "logits/rejected": 0.4448607861995697, + "logps/chosen": -1.3772170543670654, + "logps/rejected": -8.906472206115723, + "loss": 2.6039, + "nll_loss": 2.5723979473114014, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1377217024564743, + "rewards/margins": 0.7529255747795105, + "rewards/rejected": -0.8906472325325012, + "step": 1143 + }, + { + "epoch": 0.7116640746500777, + "grad_norm": 0.4393058717250824, + "learning_rate": 2.1400000000000002e-05, + "log_odds_chosen": 9.942191123962402, + "log_odds_ratio": -0.08139564841985703, + "logits/chosen": 0.32947584986686707, + "logits/rejected": 0.6434736251831055, + "logps/chosen": -1.153541088104248, + "logps/rejected": -10.689937591552734, + "loss": 3.8071, + "nll_loss": 3.7989487648010254, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11535410583019257, + "rewards/margins": 0.953639566898346, + "rewards/rejected": -1.0689938068389893, + "step": 1144 + }, + { + "epoch": 0.7122861586314152, + "grad_norm": 0.6094390749931335, + "learning_rate": 2.1375e-05, + "log_odds_chosen": 8.202652931213379, + "log_odds_ratio": -0.24067726731300354, + "logits/chosen": 0.2636764943599701, + "logits/rejected": 0.8318098783493042, + "logps/chosen": -1.185968041419983, + "logps/rejected": -9.15146255493164, + "loss": 2.7167, + "nll_loss": 2.6925878524780273, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11859681457281113, + "rewards/margins": 0.7965494394302368, + "rewards/rejected": -0.9151462316513062, + "step": 1145 + }, + { + "epoch": 0.7129082426127528, + "grad_norm": 0.4632388651371002, + "learning_rate": 2.135e-05, + "log_odds_chosen": 12.712577819824219, + "log_odds_ratio": -0.0024499651044607162, + "logits/chosen": 0.21387840807437897, + "logits/rejected": 0.5963307023048401, + "logps/chosen": -0.9470509886741638, + "logps/rejected": -12.899580955505371, + "loss": 3.5905, + "nll_loss": 3.590237617492676, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09470509737730026, + "rewards/margins": 1.1952530145645142, + "rewards/rejected": -1.289958119392395, + "step": 1146 + }, + { + "epoch": 0.7135303265940902, + "grad_norm": 0.6410781741142273, + "learning_rate": 2.1325e-05, + "log_odds_chosen": 9.504288673400879, + "log_odds_ratio": -0.2655012309551239, + "logits/chosen": 0.016082381829619408, + "logits/rejected": 0.46460607647895813, + "logps/chosen": -1.0009565353393555, + "logps/rejected": -10.186278343200684, + "loss": 2.1953, + "nll_loss": 2.16878080368042, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10009565949440002, + "rewards/margins": 0.9185321927070618, + "rewards/rejected": -1.0186278820037842, + "step": 1147 + }, + { + "epoch": 0.7141524105754277, + "grad_norm": 2.453840494155884, + "learning_rate": 2.13e-05, + "log_odds_chosen": 15.4404935836792, + "log_odds_ratio": -0.07841021567583084, + "logits/chosen": 0.31317567825317383, + "logits/rejected": 1.2708203792572021, + "logps/chosen": -1.4197622537612915, + "logps/rejected": -16.562503814697266, + "loss": 3.4137, + "nll_loss": 3.405905246734619, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1419762372970581, + "rewards/margins": 1.5142741203308105, + "rewards/rejected": -1.6562504768371582, + "step": 1148 + }, + { + "epoch": 0.7147744945567651, + "grad_norm": 0.601822018623352, + "learning_rate": 2.1275000000000002e-05, + "log_odds_chosen": 6.593123435974121, + "log_odds_ratio": -0.3577158451080322, + "logits/chosen": 0.14801143109798431, + "logits/rejected": 0.33144474029541016, + "logps/chosen": -1.1508957147598267, + "logps/rejected": -7.38616418838501, + "loss": 3.1124, + "nll_loss": 3.0766658782958984, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11508957296609879, + "rewards/margins": 0.623526930809021, + "rewards/rejected": -0.738616406917572, + "step": 1149 + }, + { + "epoch": 0.7153965785381027, + "grad_norm": 0.49013224244117737, + "learning_rate": 2.125e-05, + "log_odds_chosen": 9.185380935668945, + "log_odds_ratio": -0.061306171119213104, + "logits/chosen": 0.13947425782680511, + "logits/rejected": 0.6735214591026306, + "logps/chosen": -1.1954495906829834, + "logps/rejected": -9.900875091552734, + "loss": 3.3637, + "nll_loss": 3.357612133026123, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11954495310783386, + "rewards/margins": 0.870542585849762, + "rewards/rejected": -0.9900875091552734, + "step": 1150 + }, + { + "epoch": 0.7160186625194401, + "grad_norm": 0.7656285166740417, + "learning_rate": 2.1225e-05, + "log_odds_chosen": 5.765224933624268, + "log_odds_ratio": -0.44179147481918335, + "logits/chosen": 0.28343236446380615, + "logits/rejected": 0.5950878858566284, + "logps/chosen": -1.435044765472412, + "logps/rejected": -7.039450645446777, + "loss": 3.4903, + "nll_loss": 3.446166515350342, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14350447058677673, + "rewards/margins": 0.5604405999183655, + "rewards/rejected": -0.7039451003074646, + "step": 1151 + }, + { + "epoch": 0.7166407465007776, + "grad_norm": 0.5079005360603333, + "learning_rate": 2.12e-05, + "log_odds_chosen": 11.404075622558594, + "log_odds_ratio": -0.010933519341051579, + "logits/chosen": 0.22485215961933136, + "logits/rejected": 1.0881459712982178, + "logps/chosen": -0.9811384677886963, + "logps/rejected": -11.846966743469238, + "loss": 3.0934, + "nll_loss": 3.092313289642334, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09811384975910187, + "rewards/margins": 1.086582899093628, + "rewards/rejected": -1.1846967935562134, + "step": 1152 + }, + { + "epoch": 0.717262830482115, + "grad_norm": 0.5223401784896851, + "learning_rate": 2.1175e-05, + "log_odds_chosen": 11.468758583068848, + "log_odds_ratio": -0.06769321858882904, + "logits/chosen": 0.16616982221603394, + "logits/rejected": 0.9403862953186035, + "logps/chosen": -1.4131141901016235, + "logps/rejected": -12.616225242614746, + "loss": 2.7869, + "nll_loss": 2.7801098823547363, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1413114219903946, + "rewards/margins": 1.1203110218048096, + "rewards/rejected": -1.261622428894043, + "step": 1153 + }, + { + "epoch": 0.7178849144634526, + "grad_norm": 0.6829721331596375, + "learning_rate": 2.115e-05, + "log_odds_chosen": 12.060152053833008, + "log_odds_ratio": -0.08659256994724274, + "logits/chosen": 0.30466094613075256, + "logits/rejected": 1.010999321937561, + "logps/chosen": -1.0162991285324097, + "logps/rejected": -12.572675704956055, + "loss": 2.9428, + "nll_loss": 2.9341888427734375, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10162991285324097, + "rewards/margins": 1.1556376218795776, + "rewards/rejected": -1.2572674751281738, + "step": 1154 + }, + { + "epoch": 0.71850699844479, + "grad_norm": 0.4815412163734436, + "learning_rate": 2.1125000000000002e-05, + "log_odds_chosen": 15.830819129943848, + "log_odds_ratio": -1.4529315194522496e-05, + "logits/chosen": 0.10796906799077988, + "logits/rejected": 1.0258997678756714, + "logps/chosen": -1.0770153999328613, + "logps/rejected": -16.436660766601562, + "loss": 2.605, + "nll_loss": 2.6049692630767822, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10770154744386673, + "rewards/margins": 1.5359647274017334, + "rewards/rejected": -1.64366614818573, + "step": 1155 + }, + { + "epoch": 0.7191290824261275, + "grad_norm": 0.5743563175201416, + "learning_rate": 2.11e-05, + "log_odds_chosen": 5.235894203186035, + "log_odds_ratio": -0.3245120048522949, + "logits/chosen": 0.1398739218711853, + "logits/rejected": 0.36136549711227417, + "logps/chosen": -0.993781566619873, + "logps/rejected": -5.917914390563965, + "loss": 2.8791, + "nll_loss": 2.8466100692749023, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09937816113233566, + "rewards/margins": 0.4924132525920868, + "rewards/rejected": -0.5917914509773254, + "step": 1156 + }, + { + "epoch": 0.7197511664074651, + "grad_norm": 0.39877840876579285, + "learning_rate": 2.1075e-05, + "log_odds_chosen": 9.245841026306152, + "log_odds_ratio": -0.13619744777679443, + "logits/chosen": 0.3372015058994293, + "logits/rejected": 0.5857275724411011, + "logps/chosen": -0.8995043635368347, + "logps/rejected": -9.580745697021484, + "loss": 3.9532, + "nll_loss": 3.939554452896118, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08995043486356735, + "rewards/margins": 0.8681241273880005, + "rewards/rejected": -0.9580744504928589, + "step": 1157 + }, + { + "epoch": 0.7203732503888025, + "grad_norm": 0.4954541027545929, + "learning_rate": 2.105e-05, + "log_odds_chosen": 7.235276222229004, + "log_odds_ratio": -0.4196551442146301, + "logits/chosen": 0.24806271493434906, + "logits/rejected": 0.6260412931442261, + "logps/chosen": -1.104307770729065, + "logps/rejected": -8.139066696166992, + "loss": 2.7239, + "nll_loss": 2.681976079940796, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1104307696223259, + "rewards/margins": 0.703475832939148, + "rewards/rejected": -0.8139066696166992, + "step": 1158 + }, + { + "epoch": 0.72099533437014, + "grad_norm": 0.5344847440719604, + "learning_rate": 2.1025e-05, + "log_odds_chosen": 8.476509094238281, + "log_odds_ratio": -0.1521918922662735, + "logits/chosen": 0.05145689472556114, + "logits/rejected": 0.5481857061386108, + "logps/chosen": -0.9891431331634521, + "logps/rejected": -9.036706924438477, + "loss": 2.502, + "nll_loss": 2.486774206161499, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09891431033611298, + "rewards/margins": 0.8047564029693604, + "rewards/rejected": -0.9036707878112793, + "step": 1159 + }, + { + "epoch": 0.7216174183514774, + "grad_norm": 0.42739880084991455, + "learning_rate": 2.1e-05, + "log_odds_chosen": 4.0024614334106445, + "log_odds_ratio": -0.2541469633579254, + "logits/chosen": 0.2720869779586792, + "logits/rejected": 0.26361843943595886, + "logps/chosen": -1.196718692779541, + "logps/rejected": -4.947728633880615, + "loss": 3.7707, + "nll_loss": 3.745288372039795, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11967186629772186, + "rewards/margins": 0.3751010000705719, + "rewards/rejected": -0.49477288126945496, + "step": 1160 + }, + { + "epoch": 0.722239502332815, + "grad_norm": 0.3615352511405945, + "learning_rate": 2.0975e-05, + "log_odds_chosen": 10.977020263671875, + "log_odds_ratio": -0.09265509247779846, + "logits/chosen": 0.10474146902561188, + "logits/rejected": 0.502599835395813, + "logps/chosen": -0.8850367665290833, + "logps/rejected": -11.234212875366211, + "loss": 2.697, + "nll_loss": 2.6877830028533936, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08850367367267609, + "rewards/margins": 1.0349177122116089, + "rewards/rejected": -1.1234214305877686, + "step": 1161 + }, + { + "epoch": 0.7228615863141524, + "grad_norm": 0.597377359867096, + "learning_rate": 2.095e-05, + "log_odds_chosen": 10.106073379516602, + "log_odds_ratio": -0.1266728639602661, + "logits/chosen": 0.26163730025291443, + "logits/rejected": 0.9807031154632568, + "logps/chosen": -1.2175838947296143, + "logps/rejected": -10.95804500579834, + "loss": 2.8703, + "nll_loss": 2.8576431274414062, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12175838649272919, + "rewards/margins": 0.9740461111068726, + "rewards/rejected": -1.0958044528961182, + "step": 1162 + }, + { + "epoch": 0.7234836702954899, + "grad_norm": 0.6204828023910522, + "learning_rate": 2.0925e-05, + "log_odds_chosen": 10.355432510375977, + "log_odds_ratio": -0.12557263672351837, + "logits/chosen": 0.21904480457305908, + "logits/rejected": 0.7396771311759949, + "logps/chosen": -1.0724170207977295, + "logps/rejected": -10.99392318725586, + "loss": 2.9639, + "nll_loss": 2.951324224472046, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10724169760942459, + "rewards/margins": 0.9921507239341736, + "rewards/rejected": -1.0993924140930176, + "step": 1163 + }, + { + "epoch": 0.7241057542768273, + "grad_norm": 0.3468632698059082, + "learning_rate": 2.09e-05, + "log_odds_chosen": 7.67006254196167, + "log_odds_ratio": -0.1240687444806099, + "logits/chosen": 0.30150219798088074, + "logits/rejected": 0.5070857405662537, + "logps/chosen": -1.1041569709777832, + "logps/rejected": -8.211764335632324, + "loss": 3.6166, + "nll_loss": 3.604192018508911, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11041569709777832, + "rewards/margins": 0.710760772228241, + "rewards/rejected": -0.8211764693260193, + "step": 1164 + }, + { + "epoch": 0.7247278382581649, + "grad_norm": 0.5467685461044312, + "learning_rate": 2.0875e-05, + "log_odds_chosen": 7.7747802734375, + "log_odds_ratio": -0.09820210188627243, + "logits/chosen": 0.24225030839443207, + "logits/rejected": 0.515572190284729, + "logps/chosen": -0.9557793736457825, + "logps/rejected": -7.976073265075684, + "loss": 3.2347, + "nll_loss": 3.2248952388763428, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09557794034481049, + "rewards/margins": 0.7020293474197388, + "rewards/rejected": -0.7976073622703552, + "step": 1165 + }, + { + "epoch": 0.7253499222395023, + "grad_norm": 0.5304726362228394, + "learning_rate": 2.085e-05, + "log_odds_chosen": 8.300542831420898, + "log_odds_ratio": -0.4137728810310364, + "logits/chosen": 0.22537773847579956, + "logits/rejected": 0.5865037441253662, + "logps/chosen": -0.9393926858901978, + "logps/rejected": -8.798408508300781, + "loss": 2.8672, + "nll_loss": 2.825866460800171, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09393927454948425, + "rewards/margins": 0.785901665687561, + "rewards/rejected": -0.8798408508300781, + "step": 1166 + }, + { + "epoch": 0.7259720062208398, + "grad_norm": 0.5175058245658875, + "learning_rate": 2.0825e-05, + "log_odds_chosen": 7.007765769958496, + "log_odds_ratio": -0.257230281829834, + "logits/chosen": 0.0327136255800724, + "logits/rejected": 0.591026246547699, + "logps/chosen": -0.9878470301628113, + "logps/rejected": -7.626405239105225, + "loss": 1.8304, + "nll_loss": 1.8046331405639648, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09878470748662949, + "rewards/margins": 0.6638558506965637, + "rewards/rejected": -0.7626405954360962, + "step": 1167 + }, + { + "epoch": 0.7265940902021772, + "grad_norm": 0.4381992220878601, + "learning_rate": 2.08e-05, + "log_odds_chosen": 10.2841215133667, + "log_odds_ratio": -0.010029378347098827, + "logits/chosen": 0.3482717275619507, + "logits/rejected": 1.1683536767959595, + "logps/chosen": -1.002194881439209, + "logps/rejected": -10.818892478942871, + "loss": 2.7471, + "nll_loss": 2.7460508346557617, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1002194732427597, + "rewards/margins": 0.9816697835922241, + "rewards/rejected": -1.081889271736145, + "step": 1168 + }, + { + "epoch": 0.7272161741835148, + "grad_norm": 0.9522091150283813, + "learning_rate": 2.0775e-05, + "log_odds_chosen": 13.303786277770996, + "log_odds_ratio": -0.09151265770196915, + "logits/chosen": 0.16112789511680603, + "logits/rejected": 0.7550581693649292, + "logps/chosen": -1.8442214727401733, + "logps/rejected": -14.88370132446289, + "loss": 2.7658, + "nll_loss": 2.756631851196289, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18442216515541077, + "rewards/margins": 1.3039480447769165, + "rewards/rejected": -1.4883701801300049, + "step": 1169 + }, + { + "epoch": 0.7278382581648523, + "grad_norm": 0.5005118250846863, + "learning_rate": 2.075e-05, + "log_odds_chosen": 9.357019424438477, + "log_odds_ratio": -0.1308477371931076, + "logits/chosen": 0.16465628147125244, + "logits/rejected": 0.4376339018344879, + "logps/chosen": -1.0926101207733154, + "logps/rejected": -9.923200607299805, + "loss": 3.1157, + "nll_loss": 3.1025681495666504, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10926100611686707, + "rewards/margins": 0.883059024810791, + "rewards/rejected": -0.9923200607299805, + "step": 1170 + }, + { + "epoch": 0.7284603421461897, + "grad_norm": 0.5556590557098389, + "learning_rate": 2.0725e-05, + "log_odds_chosen": 8.283122062683105, + "log_odds_ratio": -0.22327569127082825, + "logits/chosen": 0.0778142660856247, + "logits/rejected": 0.35505211353302, + "logps/chosen": -0.9077842235565186, + "logps/rejected": -8.489635467529297, + "loss": 2.5729, + "nll_loss": 2.550588607788086, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0907784253358841, + "rewards/margins": 0.758185088634491, + "rewards/rejected": -0.8489634990692139, + "step": 1171 + }, + { + "epoch": 0.7290824261275272, + "grad_norm": 0.46830663084983826, + "learning_rate": 2.07e-05, + "log_odds_chosen": 6.0396928787231445, + "log_odds_ratio": -0.23946940898895264, + "logits/chosen": 0.1411685347557068, + "logits/rejected": 0.3880601227283478, + "logps/chosen": -1.1312123537063599, + "logps/rejected": -6.760911464691162, + "loss": 3.0798, + "nll_loss": 3.055830717086792, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11312123388051987, + "rewards/margins": 0.562969982624054, + "rewards/rejected": -0.676091194152832, + "step": 1172 + }, + { + "epoch": 0.7297045101088647, + "grad_norm": 2.1477103233337402, + "learning_rate": 2.0675e-05, + "log_odds_chosen": 9.316595077514648, + "log_odds_ratio": -0.3114702105522156, + "logits/chosen": 0.14469477534294128, + "logits/rejected": 0.43679317831993103, + "logps/chosen": -1.1385347843170166, + "logps/rejected": -9.948036193847656, + "loss": 2.5376, + "nll_loss": 2.5064356327056885, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11385348439216614, + "rewards/margins": 0.8809502124786377, + "rewards/rejected": -0.9948036670684814, + "step": 1173 + }, + { + "epoch": 0.7303265940902022, + "grad_norm": 0.48001450300216675, + "learning_rate": 2.065e-05, + "log_odds_chosen": 13.939992904663086, + "log_odds_ratio": -0.14188605546951294, + "logits/chosen": 0.16776786744594574, + "logits/rejected": 0.987214207649231, + "logps/chosen": -0.7123444676399231, + "logps/rejected": -13.940017700195312, + "loss": 2.7787, + "nll_loss": 2.764474391937256, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.07123444974422455, + "rewards/margins": 1.3227672576904297, + "rewards/rejected": -1.394001841545105, + "step": 1174 + }, + { + "epoch": 0.7309486780715396, + "grad_norm": 0.46293526887893677, + "learning_rate": 2.0625e-05, + "log_odds_chosen": 7.249242782592773, + "log_odds_ratio": -0.24686285853385925, + "logits/chosen": 0.16540293395519257, + "logits/rejected": 0.614818811416626, + "logps/chosen": -1.5638196468353271, + "logps/rejected": -8.594873428344727, + "loss": 2.9741, + "nll_loss": 2.9493930339813232, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15638196468353271, + "rewards/margins": 0.7031053900718689, + "rewards/rejected": -0.8594873547554016, + "step": 1175 + }, + { + "epoch": 0.7315707620528772, + "grad_norm": 0.3698181211948395, + "learning_rate": 2.06e-05, + "log_odds_chosen": 9.808906555175781, + "log_odds_ratio": -0.025691796094179153, + "logits/chosen": 0.38358452916145325, + "logits/rejected": 0.8628143072128296, + "logps/chosen": -1.1391582489013672, + "logps/rejected": -10.532305717468262, + "loss": 3.9937, + "nll_loss": 3.991159439086914, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1139158234000206, + "rewards/margins": 0.9393147826194763, + "rewards/rejected": -1.0532306432724, + "step": 1176 + }, + { + "epoch": 0.7321928460342146, + "grad_norm": 0.3679634928703308, + "learning_rate": 2.0575e-05, + "log_odds_chosen": 6.190854549407959, + "log_odds_ratio": -0.20879708230495453, + "logits/chosen": 0.4313851594924927, + "logits/rejected": 0.8396764993667603, + "logps/chosen": -1.062203049659729, + "logps/rejected": -6.8761305809021, + "loss": 3.8049, + "nll_loss": 3.7840020656585693, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1062203049659729, + "rewards/margins": 0.581392765045166, + "rewards/rejected": -0.6876130700111389, + "step": 1177 + }, + { + "epoch": 0.7328149300155521, + "grad_norm": 0.5036177039146423, + "learning_rate": 2.055e-05, + "log_odds_chosen": 9.593914985656738, + "log_odds_ratio": -0.14493709802627563, + "logits/chosen": 0.303832471370697, + "logits/rejected": 0.7071282863616943, + "logps/chosen": -1.0161112546920776, + "logps/rejected": -10.205410957336426, + "loss": 3.3057, + "nll_loss": 3.291208267211914, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10161112993955612, + "rewards/margins": 0.9189299941062927, + "rewards/rejected": -1.0205411911010742, + "step": 1178 + }, + { + "epoch": 0.7334370139968895, + "grad_norm": 0.6572273969650269, + "learning_rate": 2.0525e-05, + "log_odds_chosen": 12.69178581237793, + "log_odds_ratio": -0.24341538548469543, + "logits/chosen": 0.22046558558940887, + "logits/rejected": 0.9849398136138916, + "logps/chosen": -1.237760066986084, + "logps/rejected": -13.610360145568848, + "loss": 2.8629, + "nll_loss": 2.8385534286499023, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12377601116895676, + "rewards/margins": 1.237259864807129, + "rewards/rejected": -1.3610360622406006, + "step": 1179 + }, + { + "epoch": 0.7340590979782271, + "grad_norm": 0.41224968433380127, + "learning_rate": 2.05e-05, + "log_odds_chosen": 6.243864059448242, + "log_odds_ratio": -0.4956444501876831, + "logits/chosen": 0.09433531761169434, + "logits/rejected": 0.4423058032989502, + "logps/chosen": -1.109994649887085, + "logps/rejected": -7.267676830291748, + "loss": 3.0106, + "nll_loss": 2.9610538482666016, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11099947988986969, + "rewards/margins": 0.6157682538032532, + "rewards/rejected": -0.7267676591873169, + "step": 1180 + }, + { + "epoch": 0.7346811819595646, + "grad_norm": 0.5551636219024658, + "learning_rate": 2.0475e-05, + "log_odds_chosen": 12.350042343139648, + "log_odds_ratio": -0.10465382039546967, + "logits/chosen": 0.1587362140417099, + "logits/rejected": 0.8673467636108398, + "logps/chosen": -1.106932520866394, + "logps/rejected": -13.051389694213867, + "loss": 2.8993, + "nll_loss": 2.8888251781463623, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11069324612617493, + "rewards/margins": 1.1944458484649658, + "rewards/rejected": -1.3051390647888184, + "step": 1181 + }, + { + "epoch": 0.735303265940902, + "grad_norm": 0.45288923382759094, + "learning_rate": 2.045e-05, + "log_odds_chosen": 17.103357315063477, + "log_odds_ratio": -1.2666027942032088e-06, + "logits/chosen": 0.30812063813209534, + "logits/rejected": 1.196515679359436, + "logps/chosen": -0.8819127082824707, + "logps/rejected": -17.416370391845703, + "loss": 3.2262, + "nll_loss": 3.226205825805664, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08819127082824707, + "rewards/margins": 1.6534457206726074, + "rewards/rejected": -1.7416372299194336, + "step": 1182 + }, + { + "epoch": 0.7359253499222395, + "grad_norm": 0.5740547180175781, + "learning_rate": 2.0425e-05, + "log_odds_chosen": 11.144023895263672, + "log_odds_ratio": -0.13786615431308746, + "logits/chosen": 0.33946895599365234, + "logits/rejected": 1.2792378664016724, + "logps/chosen": -1.425415277481079, + "logps/rejected": -12.355165481567383, + "loss": 2.9367, + "nll_loss": 2.9228994846343994, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1425415277481079, + "rewards/margins": 1.0929750204086304, + "rewards/rejected": -1.2355165481567383, + "step": 1183 + }, + { + "epoch": 0.736547433903577, + "grad_norm": 0.5895634889602661, + "learning_rate": 2.04e-05, + "log_odds_chosen": 14.83269214630127, + "log_odds_ratio": -0.07303274422883987, + "logits/chosen": 0.09696874767541885, + "logits/rejected": 1.0919651985168457, + "logps/chosen": -1.0911352634429932, + "logps/rejected": -15.479246139526367, + "loss": 2.4192, + "nll_loss": 2.411862373352051, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10911353677511215, + "rewards/margins": 1.4388110637664795, + "rewards/rejected": -1.5479246377944946, + "step": 1184 + }, + { + "epoch": 0.7371695178849145, + "grad_norm": 0.46647828817367554, + "learning_rate": 2.0375e-05, + "log_odds_chosen": 10.80234146118164, + "log_odds_ratio": -0.21883045136928558, + "logits/chosen": 0.19847188889980316, + "logits/rejected": 0.8998786211013794, + "logps/chosen": -1.2073338031768799, + "logps/rejected": -11.712199211120605, + "loss": 3.0496, + "nll_loss": 3.027686595916748, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12073338031768799, + "rewards/margins": 1.0504865646362305, + "rewards/rejected": -1.171220064163208, + "step": 1185 + }, + { + "epoch": 0.7377916018662519, + "grad_norm": 0.5875625014305115, + "learning_rate": 2.035e-05, + "log_odds_chosen": 4.775179862976074, + "log_odds_ratio": -0.42339253425598145, + "logits/chosen": 0.04878817871212959, + "logits/rejected": 0.5890315175056458, + "logps/chosen": -1.3256160020828247, + "logps/rejected": -5.962397575378418, + "loss": 2.4948, + "nll_loss": 2.4524922370910645, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.132561594247818, + "rewards/margins": 0.46367818117141724, + "rewards/rejected": -0.5962398052215576, + "step": 1186 + }, + { + "epoch": 0.7384136858475894, + "grad_norm": 0.5900309085845947, + "learning_rate": 2.0325e-05, + "log_odds_chosen": 12.905298233032227, + "log_odds_ratio": -0.13436618447303772, + "logits/chosen": 0.22156783938407898, + "logits/rejected": 0.969600260257721, + "logps/chosen": -1.015760898590088, + "logps/rejected": -13.515449523925781, + "loss": 2.7867, + "nll_loss": 2.7732279300689697, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10157608985900879, + "rewards/margins": 1.2499690055847168, + "rewards/rejected": -1.3515450954437256, + "step": 1187 + }, + { + "epoch": 0.7390357698289269, + "grad_norm": 0.4624529182910919, + "learning_rate": 2.0300000000000002e-05, + "log_odds_chosen": 17.161376953125, + "log_odds_ratio": -6.544519419549033e-05, + "logits/chosen": 0.1179753914475441, + "logits/rejected": 0.8243229985237122, + "logps/chosen": -1.4011576175689697, + "logps/rejected": -18.230491638183594, + "loss": 3.0534, + "nll_loss": 3.053412437438965, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14011575281620026, + "rewards/margins": 1.6829335689544678, + "rewards/rejected": -1.8230493068695068, + "step": 1188 + }, + { + "epoch": 0.7396578538102644, + "grad_norm": 0.393663614988327, + "learning_rate": 2.0275e-05, + "log_odds_chosen": 15.431196212768555, + "log_odds_ratio": -3.939667294616811e-05, + "logits/chosen": 0.39448249340057373, + "logits/rejected": 1.1981537342071533, + "logps/chosen": -1.0742273330688477, + "logps/rejected": -15.890814781188965, + "loss": 3.54, + "nll_loss": 3.5400190353393555, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10742272436618805, + "rewards/margins": 1.4816588163375854, + "rewards/rejected": -1.5890816450119019, + "step": 1189 + }, + { + "epoch": 0.7402799377916018, + "grad_norm": 0.6178926825523376, + "learning_rate": 2.025e-05, + "log_odds_chosen": 9.652229309082031, + "log_odds_ratio": -0.16254249215126038, + "logits/chosen": 0.3837137520313263, + "logits/rejected": 0.9006325006484985, + "logps/chosen": -1.0126757621765137, + "logps/rejected": -10.24832534790039, + "loss": 3.3, + "nll_loss": 3.28379487991333, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10126757621765137, + "rewards/margins": 0.9235650300979614, + "rewards/rejected": -1.0248326063156128, + "step": 1190 + }, + { + "epoch": 0.7409020217729394, + "grad_norm": 0.5684689283370972, + "learning_rate": 2.0225000000000004e-05, + "log_odds_chosen": 11.131904602050781, + "log_odds_ratio": -0.11341032385826111, + "logits/chosen": 0.2865346074104309, + "logits/rejected": 0.7895666360855103, + "logps/chosen": -0.9756239056587219, + "logps/rejected": -11.580063819885254, + "loss": 3.0371, + "nll_loss": 3.0257105827331543, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09756238758563995, + "rewards/margins": 1.0604439973831177, + "rewards/rejected": -1.1580064296722412, + "step": 1191 + }, + { + "epoch": 0.7415241057542769, + "grad_norm": 0.47814252972602844, + "learning_rate": 2.0200000000000003e-05, + "log_odds_chosen": 13.986754417419434, + "log_odds_ratio": -0.060169950127601624, + "logits/chosen": 0.5951923727989197, + "logits/rejected": 1.518169641494751, + "logps/chosen": -1.3645962476730347, + "logps/rejected": -15.049614906311035, + "loss": 4.0235, + "nll_loss": 4.01747465133667, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.136459618806839, + "rewards/margins": 1.368501901626587, + "rewards/rejected": -1.5049614906311035, + "step": 1192 + }, + { + "epoch": 0.7421461897356143, + "grad_norm": 0.4866500794887543, + "learning_rate": 2.0175000000000003e-05, + "log_odds_chosen": 13.448404312133789, + "log_odds_ratio": -0.11860945075750351, + "logits/chosen": 0.3314950466156006, + "logits/rejected": 1.0111912488937378, + "logps/chosen": -1.384364366531372, + "logps/rejected": -14.545913696289062, + "loss": 3.0946, + "nll_loss": 3.0826945304870605, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1384364366531372, + "rewards/margins": 1.316154956817627, + "rewards/rejected": -1.4545913934707642, + "step": 1193 + }, + { + "epoch": 0.7427682737169518, + "grad_norm": 0.6490217447280884, + "learning_rate": 2.0150000000000002e-05, + "log_odds_chosen": 16.155012130737305, + "log_odds_ratio": -0.07999815791845322, + "logits/chosen": 0.10705263912677765, + "logits/rejected": 1.093653917312622, + "logps/chosen": -1.271554708480835, + "logps/rejected": -17.101104736328125, + "loss": 2.5376, + "nll_loss": 2.529562473297119, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12715548276901245, + "rewards/margins": 1.5829551219940186, + "rewards/rejected": -1.7101104259490967, + "step": 1194 + }, + { + "epoch": 0.7433903576982893, + "grad_norm": 0.5161392688751221, + "learning_rate": 2.0125e-05, + "log_odds_chosen": 11.450945854187012, + "log_odds_ratio": -0.14187860488891602, + "logits/chosen": 0.4051249921321869, + "logits/rejected": 1.25907301902771, + "logps/chosen": -1.2657084465026855, + "logps/rejected": -12.435002326965332, + "loss": 3.3354, + "nll_loss": 3.3212480545043945, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12657085061073303, + "rewards/margins": 1.1169294118881226, + "rewards/rejected": -1.2435002326965332, + "step": 1195 + }, + { + "epoch": 0.7440124416796268, + "grad_norm": 0.4983889162540436, + "learning_rate": 2.01e-05, + "log_odds_chosen": 15.152313232421875, + "log_odds_ratio": -0.002952311420813203, + "logits/chosen": 0.20715823769569397, + "logits/rejected": 1.1005414724349976, + "logps/chosen": -1.2786905765533447, + "logps/rejected": -16.09111213684082, + "loss": 2.8896, + "nll_loss": 2.8893094062805176, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12786906957626343, + "rewards/margins": 1.4812421798706055, + "rewards/rejected": -1.6091113090515137, + "step": 1196 + }, + { + "epoch": 0.7446345256609642, + "grad_norm": 0.3791523277759552, + "learning_rate": 2.0075000000000003e-05, + "log_odds_chosen": 16.003755569458008, + "log_odds_ratio": -0.0002835804771166295, + "logits/chosen": 0.3261744976043701, + "logits/rejected": 1.0396631956100464, + "logps/chosen": -0.8366090059280396, + "logps/rejected": -15.989496231079102, + "loss": 3.6367, + "nll_loss": 3.6366686820983887, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08366090059280396, + "rewards/margins": 1.5152887105941772, + "rewards/rejected": -1.598949670791626, + "step": 1197 + }, + { + "epoch": 0.7452566096423017, + "grad_norm": 0.594624936580658, + "learning_rate": 2.0050000000000003e-05, + "log_odds_chosen": 15.558506965637207, + "log_odds_ratio": -0.0004107086278963834, + "logits/chosen": 0.210353821516037, + "logits/rejected": 0.9793530106544495, + "logps/chosen": -0.8724642395973206, + "logps/rejected": -15.786218643188477, + "loss": 2.7662, + "nll_loss": 2.7662012577056885, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08724642544984818, + "rewards/margins": 1.49137544631958, + "rewards/rejected": -1.5786218643188477, + "step": 1198 + }, + { + "epoch": 0.7458786936236392, + "grad_norm": 3.2293317317962646, + "learning_rate": 2.0025000000000002e-05, + "log_odds_chosen": 12.671496391296387, + "log_odds_ratio": -0.10842613130807877, + "logits/chosen": 0.23085221648216248, + "logits/rejected": 1.1595039367675781, + "logps/chosen": -1.3611061573028564, + "logps/rejected": -13.689292907714844, + "loss": 2.904, + "nll_loss": 2.8931233882904053, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13611061871051788, + "rewards/margins": 1.232818603515625, + "rewards/rejected": -1.3689292669296265, + "step": 1199 + }, + { + "epoch": 0.7465007776049767, + "grad_norm": 0.4771862328052521, + "learning_rate": 2e-05, + "log_odds_chosen": 22.102876663208008, + "log_odds_ratio": -0.001636207103729248, + "logits/chosen": 0.20666168630123138, + "logits/rejected": 1.0230656862258911, + "logps/chosen": -1.34181547164917, + "logps/rejected": -23.12851333618164, + "loss": 3.159, + "nll_loss": 3.1588516235351562, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13418152928352356, + "rewards/margins": 2.1786699295043945, + "rewards/rejected": -2.3128514289855957, + "step": 1200 + }, + { + "epoch": 0.7471228615863141, + "grad_norm": 0.4705706536769867, + "learning_rate": 1.9975e-05, + "log_odds_chosen": 13.418475151062012, + "log_odds_ratio": -0.17137664556503296, + "logits/chosen": 0.3189517855644226, + "logits/rejected": 0.7763543725013733, + "logps/chosen": -1.1539502143859863, + "logps/rejected": -14.102031707763672, + "loss": 3.4356, + "nll_loss": 3.418506383895874, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11539502441883087, + "rewards/margins": 1.2948081493377686, + "rewards/rejected": -1.410203218460083, + "step": 1201 + }, + { + "epoch": 0.7477449455676516, + "grad_norm": 1.8948168754577637, + "learning_rate": 1.995e-05, + "log_odds_chosen": 16.753978729248047, + "log_odds_ratio": -2.4275623218272813e-05, + "logits/chosen": 0.33771950006484985, + "logits/rejected": 1.016837239265442, + "logps/chosen": -3.4603779315948486, + "logps/rejected": -19.40519905090332, + "loss": 3.5931, + "nll_loss": 3.593096971511841, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.3460378050804138, + "rewards/margins": 1.594482183456421, + "rewards/rejected": -1.94051992893219, + "step": 1202 + }, + { + "epoch": 0.7483670295489891, + "grad_norm": 0.5406678915023804, + "learning_rate": 1.9925000000000003e-05, + "log_odds_chosen": 13.841324806213379, + "log_odds_ratio": -0.025037458166480064, + "logits/chosen": 0.422190397977829, + "logits/rejected": 1.2798744440078735, + "logps/chosen": -1.0436694622039795, + "logps/rejected": -14.358189582824707, + "loss": 3.5902, + "nll_loss": 3.5876641273498535, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10436695069074631, + "rewards/margins": 1.3314520120620728, + "rewards/rejected": -1.4358189105987549, + "step": 1203 + }, + { + "epoch": 0.7489891135303266, + "grad_norm": 0.8551453948020935, + "learning_rate": 1.9900000000000003e-05, + "log_odds_chosen": 9.493968963623047, + "log_odds_ratio": -0.198939248919487, + "logits/chosen": 0.15752126276493073, + "logits/rejected": 0.5775033235549927, + "logps/chosen": -2.0665013790130615, + "logps/rejected": -11.248618125915527, + "loss": 3.215, + "nll_loss": 3.1950745582580566, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.20665015280246735, + "rewards/margins": 0.9182117581367493, + "rewards/rejected": -1.1248618364334106, + "step": 1204 + }, + { + "epoch": 0.749611197511664, + "grad_norm": 0.5271828770637512, + "learning_rate": 1.9875000000000002e-05, + "log_odds_chosen": 12.457223892211914, + "log_odds_ratio": -0.1530085653066635, + "logits/chosen": 0.21353940665721893, + "logits/rejected": 0.7389912605285645, + "logps/chosen": -1.1457003355026245, + "logps/rejected": -13.223892211914062, + "loss": 2.9948, + "nll_loss": 2.9794840812683105, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11457002907991409, + "rewards/margins": 1.2078192234039307, + "rewards/rejected": -1.3223892450332642, + "step": 1205 + }, + { + "epoch": 0.7502332814930015, + "grad_norm": 0.769097089767456, + "learning_rate": 1.985e-05, + "log_odds_chosen": 14.107221603393555, + "log_odds_ratio": -0.1394292563199997, + "logits/chosen": 0.155813530087471, + "logits/rejected": 1.3736224174499512, + "logps/chosen": -1.1750849485397339, + "logps/rejected": -14.946660995483398, + "loss": 2.6975, + "nll_loss": 2.6836023330688477, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11750849336385727, + "rewards/margins": 1.3771576881408691, + "rewards/rejected": -1.4946662187576294, + "step": 1206 + }, + { + "epoch": 0.7508553654743391, + "grad_norm": 2.034923553466797, + "learning_rate": 1.9825e-05, + "log_odds_chosen": 9.450210571289062, + "log_odds_ratio": -0.09710147231817245, + "logits/chosen": 0.12488899379968643, + "logits/rejected": 0.5777722597122192, + "logps/chosen": -1.1520278453826904, + "logps/rejected": -10.04898738861084, + "loss": 3.1795, + "nll_loss": 3.1698226928710938, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11520278453826904, + "rewards/margins": 0.8896960020065308, + "rewards/rejected": -1.0048987865447998, + "step": 1207 + }, + { + "epoch": 0.7514774494556765, + "grad_norm": 1.6488580703735352, + "learning_rate": 1.9800000000000004e-05, + "log_odds_chosen": 7.367974281311035, + "log_odds_ratio": -0.11552385240793228, + "logits/chosen": 0.10463400185108185, + "logits/rejected": 0.6088777184486389, + "logps/chosen": -1.4335150718688965, + "logps/rejected": -8.32715892791748, + "loss": 2.7639, + "nll_loss": 2.7523436546325684, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1433514952659607, + "rewards/margins": 0.6893644332885742, + "rewards/rejected": -0.8327159285545349, + "step": 1208 + }, + { + "epoch": 0.752099533437014, + "grad_norm": 0.4398881196975708, + "learning_rate": 1.9775000000000003e-05, + "log_odds_chosen": 8.662324905395508, + "log_odds_ratio": -0.0030407479498535395, + "logits/chosen": 0.07866299152374268, + "logits/rejected": 0.6986607909202576, + "logps/chosen": -1.4439903497695923, + "logps/rejected": -9.80562973022461, + "loss": 3.1564, + "nll_loss": 3.1560733318328857, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14439904689788818, + "rewards/margins": 0.8361639976501465, + "rewards/rejected": -0.9805630445480347, + "step": 1209 + }, + { + "epoch": 0.7527216174183515, + "grad_norm": 0.5735336542129517, + "learning_rate": 1.9750000000000002e-05, + "log_odds_chosen": 5.275170803070068, + "log_odds_ratio": -0.3947235345840454, + "logits/chosen": 0.18500526249408722, + "logits/rejected": 0.6523492932319641, + "logps/chosen": -1.1334285736083984, + "logps/rejected": -6.1533002853393555, + "loss": 3.3193, + "nll_loss": 3.279778003692627, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11334285140037537, + "rewards/margins": 0.5019872188568115, + "rewards/rejected": -0.6153301000595093, + "step": 1210 + }, + { + "epoch": 0.753343701399689, + "grad_norm": 0.4635184407234192, + "learning_rate": 1.9725000000000002e-05, + "log_odds_chosen": 4.65528678894043, + "log_odds_ratio": -0.32718393206596375, + "logits/chosen": 0.12875179946422577, + "logits/rejected": 0.4438689351081848, + "logps/chosen": -1.1663978099822998, + "logps/rejected": -5.531558036804199, + "loss": 3.2701, + "nll_loss": 3.2373604774475098, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11663977801799774, + "rewards/margins": 0.43651604652404785, + "rewards/rejected": -0.5531558394432068, + "step": 1211 + }, + { + "epoch": 0.7539657853810264, + "grad_norm": 0.37020620703697205, + "learning_rate": 1.97e-05, + "log_odds_chosen": 7.262178897857666, + "log_odds_ratio": -0.1279047131538391, + "logits/chosen": 0.03921634703874588, + "logits/rejected": 0.5070078372955322, + "logps/chosen": -1.1373803615570068, + "logps/rejected": -8.027933120727539, + "loss": 2.796, + "nll_loss": 2.783205032348633, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1137380301952362, + "rewards/margins": 0.6890553832054138, + "rewards/rejected": -0.8027933835983276, + "step": 1212 + }, + { + "epoch": 0.7545878693623639, + "grad_norm": 0.5390399694442749, + "learning_rate": 1.9675e-05, + "log_odds_chosen": 4.89652156829834, + "log_odds_ratio": -0.0828741043806076, + "logits/chosen": 0.10957112163305283, + "logits/rejected": 0.38588717579841614, + "logps/chosen": -0.9967465400695801, + "logps/rejected": -5.436730861663818, + "loss": 2.9941, + "nll_loss": 2.985842704772949, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09967464953660965, + "rewards/margins": 0.44399845600128174, + "rewards/rejected": -0.5436730980873108, + "step": 1213 + }, + { + "epoch": 0.7552099533437014, + "grad_norm": 0.5554744601249695, + "learning_rate": 1.9650000000000003e-05, + "log_odds_chosen": 5.830607891082764, + "log_odds_ratio": -0.15100139379501343, + "logits/chosen": 0.3689972162246704, + "logits/rejected": 0.6765709519386292, + "logps/chosen": -1.0688714981079102, + "logps/rejected": -6.527162551879883, + "loss": 3.2168, + "nll_loss": 3.2016830444335938, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10688715428113937, + "rewards/margins": 0.545829176902771, + "rewards/rejected": -0.6527162790298462, + "step": 1214 + }, + { + "epoch": 0.7558320373250389, + "grad_norm": 0.4852057695388794, + "learning_rate": 1.9625000000000003e-05, + "log_odds_chosen": 6.636116981506348, + "log_odds_ratio": -0.2725330591201782, + "logits/chosen": 0.10936164855957031, + "logits/rejected": 0.3851650059223175, + "logps/chosen": -1.1456758975982666, + "logps/rejected": -7.426181793212891, + "loss": 3.1894, + "nll_loss": 3.1621837615966797, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1145675927400589, + "rewards/margins": 0.6280506253242493, + "rewards/rejected": -0.742618203163147, + "step": 1215 + }, + { + "epoch": 0.7564541213063763, + "grad_norm": 0.4729749262332916, + "learning_rate": 1.9600000000000002e-05, + "log_odds_chosen": 3.1161768436431885, + "log_odds_ratio": -0.2609820067882538, + "logits/chosen": -0.028117988258600235, + "logits/rejected": 0.07570795714855194, + "logps/chosen": -1.1444841623306274, + "logps/rejected": -3.978320598602295, + "loss": 2.8225, + "nll_loss": 2.796421766281128, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1144484132528305, + "rewards/margins": 0.28338363766670227, + "rewards/rejected": -0.39783206582069397, + "step": 1216 + }, + { + "epoch": 0.7570762052877138, + "grad_norm": 1.1897724866867065, + "learning_rate": 1.9575e-05, + "log_odds_chosen": 6.136231422424316, + "log_odds_ratio": -0.28731441497802734, + "logits/chosen": 0.09058015048503876, + "logits/rejected": 0.37315765023231506, + "logps/chosen": -1.25123929977417, + "logps/rejected": -6.890491008758545, + "loss": 2.9212, + "nll_loss": 2.8924193382263184, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12512391805648804, + "rewards/margins": 0.5639252066612244, + "rewards/rejected": -0.6890491247177124, + "step": 1217 + }, + { + "epoch": 0.7576982892690514, + "grad_norm": 0.6197946667671204, + "learning_rate": 1.955e-05, + "log_odds_chosen": 6.062702655792236, + "log_odds_ratio": -0.08814633637666702, + "logits/chosen": 0.21369720995426178, + "logits/rejected": 0.5749257206916809, + "logps/chosen": -1.2833783626556396, + "logps/rejected": -7.003604412078857, + "loss": 2.8867, + "nll_loss": 2.877854347229004, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12833784520626068, + "rewards/margins": 0.572022557258606, + "rewards/rejected": -0.7003604173660278, + "step": 1218 + }, + { + "epoch": 0.7583203732503888, + "grad_norm": 0.48757174611091614, + "learning_rate": 1.9525e-05, + "log_odds_chosen": 6.600275039672852, + "log_odds_ratio": -0.08514364063739777, + "logits/chosen": 0.21454694867134094, + "logits/rejected": 0.7358257174491882, + "logps/chosen": -0.9961150288581848, + "logps/rejected": -7.084440231323242, + "loss": 2.9164, + "nll_loss": 2.90787672996521, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09961150586605072, + "rewards/margins": 0.6088324785232544, + "rewards/rejected": -0.7084439992904663, + "step": 1219 + }, + { + "epoch": 0.7589424572317263, + "grad_norm": 0.45170265436172485, + "learning_rate": 1.9500000000000003e-05, + "log_odds_chosen": 6.516654968261719, + "log_odds_ratio": -0.2116156965494156, + "logits/chosen": 0.2163967788219452, + "logits/rejected": 0.47737014293670654, + "logps/chosen": -1.4186476469039917, + "logps/rejected": -7.741004943847656, + "loss": 3.5083, + "nll_loss": 3.487144947052002, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14186474680900574, + "rewards/margins": 0.6322357654571533, + "rewards/rejected": -0.7741005420684814, + "step": 1220 + }, + { + "epoch": 0.7595645412130637, + "grad_norm": 0.33577945828437805, + "learning_rate": 1.9475000000000002e-05, + "log_odds_chosen": 5.055993556976318, + "log_odds_ratio": -0.16657014191150665, + "logits/chosen": 0.1790454089641571, + "logits/rejected": 0.42467498779296875, + "logps/chosen": -0.9846662282943726, + "logps/rejected": -5.626519680023193, + "loss": 3.2669, + "nll_loss": 3.250199556350708, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09846662729978561, + "rewards/margins": 0.46418535709381104, + "rewards/rejected": -0.5626519918441772, + "step": 1221 + }, + { + "epoch": 0.7601866251944013, + "grad_norm": 0.6504151821136475, + "learning_rate": 1.9450000000000002e-05, + "log_odds_chosen": 6.201626777648926, + "log_odds_ratio": -0.06464290618896484, + "logits/chosen": 0.04870191961526871, + "logits/rejected": 0.5722784996032715, + "logps/chosen": -0.9842745661735535, + "logps/rejected": -6.750699043273926, + "loss": 2.2712, + "nll_loss": 2.264713764190674, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09842745959758759, + "rewards/margins": 0.5766425132751465, + "rewards/rejected": -0.6750699281692505, + "step": 1222 + }, + { + "epoch": 0.7608087091757387, + "grad_norm": 0.7092140913009644, + "learning_rate": 1.9425e-05, + "log_odds_chosen": 2.3777732849121094, + "log_odds_ratio": -0.3354724943637848, + "logits/chosen": 0.11024624109268188, + "logits/rejected": 0.2564601004123688, + "logps/chosen": -1.0713608264923096, + "logps/rejected": -3.2102227210998535, + "loss": 2.7334, + "nll_loss": 2.6998963356018066, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1071360781788826, + "rewards/margins": 0.21388620138168335, + "rewards/rejected": -0.32102227210998535, + "step": 1223 + }, + { + "epoch": 0.7614307931570762, + "grad_norm": 0.48227447271347046, + "learning_rate": 1.94e-05, + "log_odds_chosen": 4.966604232788086, + "log_odds_ratio": -0.19418928027153015, + "logits/chosen": 0.17129558324813843, + "logits/rejected": 0.4398013651371002, + "logps/chosen": -1.319381833076477, + "logps/rejected": -5.987258434295654, + "loss": 3.2791, + "nll_loss": 3.2596778869628906, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13193818926811218, + "rewards/margins": 0.4667876362800598, + "rewards/rejected": -0.5987257957458496, + "step": 1224 + }, + { + "epoch": 0.7620528771384136, + "grad_norm": 0.5294142365455627, + "learning_rate": 1.9375e-05, + "log_odds_chosen": 3.5622966289520264, + "log_odds_ratio": -0.25366440415382385, + "logits/chosen": 0.3200629949569702, + "logits/rejected": 0.5575484037399292, + "logps/chosen": -1.051979899406433, + "logps/rejected": -4.283415794372559, + "loss": 3.1231, + "nll_loss": 3.0977699756622314, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1051979809999466, + "rewards/margins": 0.3231435716152191, + "rewards/rejected": -0.4283415675163269, + "step": 1225 + }, + { + "epoch": 0.7626749611197512, + "grad_norm": 1.2428282499313354, + "learning_rate": 1.9350000000000003e-05, + "log_odds_chosen": 5.334478378295898, + "log_odds_ratio": -0.14889080822467804, + "logits/chosen": 0.2575344145298004, + "logits/rejected": 0.5913892984390259, + "logps/chosen": -1.0900267362594604, + "logps/rejected": -6.102575778961182, + "loss": 3.0042, + "nll_loss": 2.9893410205841064, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10900267213582993, + "rewards/margins": 0.5012549161911011, + "rewards/rejected": -0.610257625579834, + "step": 1226 + }, + { + "epoch": 0.7632970451010886, + "grad_norm": 0.9420454502105713, + "learning_rate": 1.9325000000000002e-05, + "log_odds_chosen": 6.546796798706055, + "log_odds_ratio": -0.18457339704036713, + "logits/chosen": 0.03549651801586151, + "logits/rejected": 0.4685467481613159, + "logps/chosen": -1.2222778797149658, + "logps/rejected": -7.488654136657715, + "loss": 2.3855, + "nll_loss": 2.367008924484253, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12222778797149658, + "rewards/margins": 0.6266376972198486, + "rewards/rejected": -0.7488654851913452, + "step": 1227 + }, + { + "epoch": 0.7639191290824261, + "grad_norm": 0.5488473176956177, + "learning_rate": 1.93e-05, + "log_odds_chosen": 3.563775062561035, + "log_odds_ratio": -0.5443033576011658, + "logits/chosen": 0.23504066467285156, + "logits/rejected": 0.41619762778282166, + "logps/chosen": -1.547841191291809, + "logps/rejected": -5.002470970153809, + "loss": 3.5391, + "nll_loss": 3.4846863746643066, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.15478411316871643, + "rewards/margins": 0.34546297788619995, + "rewards/rejected": -0.5002471208572388, + "step": 1228 + }, + { + "epoch": 0.7645412130637637, + "grad_norm": 0.8373052477836609, + "learning_rate": 1.9275e-05, + "log_odds_chosen": 3.503401041030884, + "log_odds_ratio": -0.4526802897453308, + "logits/chosen": 0.2677290439605713, + "logits/rejected": 0.4133756160736084, + "logps/chosen": -1.0986255407333374, + "logps/rejected": -4.06947660446167, + "loss": 3.2135, + "nll_loss": 3.1682019233703613, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1098625585436821, + "rewards/margins": 0.29708513617515564, + "rewards/rejected": -0.40694767236709595, + "step": 1229 + }, + { + "epoch": 0.7651632970451011, + "grad_norm": 0.5462814569473267, + "learning_rate": 1.925e-05, + "log_odds_chosen": 7.460938930511475, + "log_odds_ratio": -0.36012497544288635, + "logits/chosen": 0.15392319858074188, + "logits/rejected": 0.5377407670021057, + "logps/chosen": -1.2988390922546387, + "logps/rejected": -8.361101150512695, + "loss": 2.9064, + "nll_loss": 2.8703439235687256, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12988391518592834, + "rewards/margins": 0.7062261700630188, + "rewards/rejected": -0.8361101150512695, + "step": 1230 + }, + { + "epoch": 0.7657853810264386, + "grad_norm": 0.70634526014328, + "learning_rate": 1.9225e-05, + "log_odds_chosen": 6.823087692260742, + "log_odds_ratio": -0.15130549669265747, + "logits/chosen": 0.24029546976089478, + "logits/rejected": 0.8185093402862549, + "logps/chosen": -1.2564562559127808, + "logps/rejected": -7.753248691558838, + "loss": 2.9292, + "nll_loss": 2.9141178131103516, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12564562261104584, + "rewards/margins": 0.6496793031692505, + "rewards/rejected": -0.775324821472168, + "step": 1231 + }, + { + "epoch": 0.766407465007776, + "grad_norm": 0.596670389175415, + "learning_rate": 1.9200000000000003e-05, + "log_odds_chosen": 6.75174617767334, + "log_odds_ratio": -0.20460355281829834, + "logits/chosen": 0.15210659801959991, + "logits/rejected": 0.29200881719589233, + "logps/chosen": -0.8481846451759338, + "logps/rejected": -7.019330978393555, + "loss": 3.118, + "nll_loss": 3.0975799560546875, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08481846749782562, + "rewards/margins": 0.6171146631240845, + "rewards/rejected": -0.7019331455230713, + "step": 1232 + }, + { + "epoch": 0.7670295489891136, + "grad_norm": 1.2167954444885254, + "learning_rate": 1.9175000000000002e-05, + "log_odds_chosen": 7.575688362121582, + "log_odds_ratio": -0.04037388786673546, + "logits/chosen": 0.16587582230567932, + "logits/rejected": 0.7371760606765747, + "logps/chosen": -1.3697576522827148, + "logps/rejected": -8.56037712097168, + "loss": 3.0045, + "nll_loss": 3.000467300415039, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13697576522827148, + "rewards/margins": 0.7190619111061096, + "rewards/rejected": -0.8560376167297363, + "step": 1233 + }, + { + "epoch": 0.767651632970451, + "grad_norm": 0.5440263152122498, + "learning_rate": 1.915e-05, + "log_odds_chosen": 5.056829452514648, + "log_odds_ratio": -0.18205265700817108, + "logits/chosen": 0.08534614741802216, + "logits/rejected": 0.38158437609672546, + "logps/chosen": -1.172467827796936, + "logps/rejected": -5.902404308319092, + "loss": 2.4827, + "nll_loss": 2.4645302295684814, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11724679172039032, + "rewards/margins": 0.4729937016963959, + "rewards/rejected": -0.590240478515625, + "step": 1234 + }, + { + "epoch": 0.7682737169517885, + "grad_norm": 0.47702041268348694, + "learning_rate": 1.9125e-05, + "log_odds_chosen": 6.737882614135742, + "log_odds_ratio": -0.1277119666337967, + "logits/chosen": 0.1412731111049652, + "logits/rejected": 0.39362943172454834, + "logps/chosen": -1.181764841079712, + "logps/rejected": -7.580643653869629, + "loss": 3.2685, + "nll_loss": 3.255748748779297, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11817647516727448, + "rewards/margins": 0.6398878693580627, + "rewards/rejected": -0.7580643892288208, + "step": 1235 + }, + { + "epoch": 0.7688958009331259, + "grad_norm": 0.5562854409217834, + "learning_rate": 1.91e-05, + "log_odds_chosen": 5.550592422485352, + "log_odds_ratio": -0.05007840692996979, + "logits/chosen": 0.15344038605690002, + "logits/rejected": 0.5068711638450623, + "logps/chosen": -1.3114280700683594, + "logps/rejected": -6.509056091308594, + "loss": 2.9737, + "nll_loss": 2.9687082767486572, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13114279508590698, + "rewards/margins": 0.5197628140449524, + "rewards/rejected": -0.6509056687355042, + "step": 1236 + }, + { + "epoch": 0.7695178849144635, + "grad_norm": 0.5004754662513733, + "learning_rate": 1.9075000000000003e-05, + "log_odds_chosen": 9.40194034576416, + "log_odds_ratio": -0.15184959769248962, + "logits/chosen": 0.2978594899177551, + "logits/rejected": 0.9191336631774902, + "logps/chosen": -1.4295010566711426, + "logps/rejected": -10.342602729797363, + "loss": 3.2896, + "nll_loss": 3.274416923522949, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1429501175880432, + "rewards/margins": 0.8913102149963379, + "rewards/rejected": -1.0342603921890259, + "step": 1237 + }, + { + "epoch": 0.7701399688958009, + "grad_norm": 0.46838802099227905, + "learning_rate": 1.9050000000000002e-05, + "log_odds_chosen": 7.5221266746521, + "log_odds_ratio": -0.21935422718524933, + "logits/chosen": 0.22468611598014832, + "logits/rejected": 0.4334770441055298, + "logps/chosen": -0.879149854183197, + "logps/rejected": -7.8995866775512695, + "loss": 3.4049, + "nll_loss": 3.3830127716064453, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08791498839855194, + "rewards/margins": 0.7020437121391296, + "rewards/rejected": -0.7899587154388428, + "step": 1238 + }, + { + "epoch": 0.7707620528771384, + "grad_norm": 0.517456591129303, + "learning_rate": 1.9025e-05, + "log_odds_chosen": 6.344799995422363, + "log_odds_ratio": -0.27640923857688904, + "logits/chosen": 0.12453234195709229, + "logits/rejected": 0.4667607545852661, + "logps/chosen": -1.0894925594329834, + "logps/rejected": -7.145116329193115, + "loss": 2.604, + "nll_loss": 2.5763416290283203, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10894925892353058, + "rewards/margins": 0.6055623888969421, + "rewards/rejected": -0.7145116925239563, + "step": 1239 + }, + { + "epoch": 0.7713841368584758, + "grad_norm": 0.6335269808769226, + "learning_rate": 1.9e-05, + "log_odds_chosen": 5.562659740447998, + "log_odds_ratio": -0.2744099497795105, + "logits/chosen": 0.23254889249801636, + "logits/rejected": 0.3621961772441864, + "logps/chosen": -0.8226711750030518, + "logps/rejected": -5.803696155548096, + "loss": 3.2831, + "nll_loss": 3.2556722164154053, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08226712048053741, + "rewards/margins": 0.4981025159358978, + "rewards/rejected": -0.5803696513175964, + "step": 1240 + }, + { + "epoch": 0.7720062208398134, + "grad_norm": 0.6518879532814026, + "learning_rate": 1.8975e-05, + "log_odds_chosen": 7.919089317321777, + "log_odds_ratio": -0.3184622526168823, + "logits/chosen": 0.043444469571113586, + "logits/rejected": 0.4088301658630371, + "logps/chosen": -1.1469988822937012, + "logps/rejected": -8.743982315063477, + "loss": 2.9404, + "nll_loss": 2.9085514545440674, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11469988524913788, + "rewards/margins": 0.7596983909606934, + "rewards/rejected": -0.8743982315063477, + "step": 1241 + }, + { + "epoch": 0.7726283048211509, + "grad_norm": 0.5594248175621033, + "learning_rate": 1.895e-05, + "log_odds_chosen": 3.817509889602661, + "log_odds_ratio": -0.41913437843322754, + "logits/chosen": 0.22413289546966553, + "logits/rejected": 0.42118874192237854, + "logps/chosen": -1.2284491062164307, + "logps/rejected": -4.893716812133789, + "loss": 3.0401, + "nll_loss": 2.9981565475463867, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12284490466117859, + "rewards/margins": 0.3665267825126648, + "rewards/rejected": -0.489371657371521, + "step": 1242 + }, + { + "epoch": 0.7732503888024883, + "grad_norm": 0.508392870426178, + "learning_rate": 1.8925000000000003e-05, + "log_odds_chosen": 10.411176681518555, + "log_odds_ratio": -0.10630620270967484, + "logits/chosen": 0.24550384283065796, + "logits/rejected": 1.1542640924453735, + "logps/chosen": -1.27555251121521, + "logps/rejected": -11.42074203491211, + "loss": 3.0676, + "nll_loss": 3.0569348335266113, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.127555251121521, + "rewards/margins": 1.0145189762115479, + "rewards/rejected": -1.1420742273330688, + "step": 1243 + }, + { + "epoch": 0.7738724727838259, + "grad_norm": 0.4941979944705963, + "learning_rate": 1.8900000000000002e-05, + "log_odds_chosen": 9.65025806427002, + "log_odds_ratio": -0.18365329504013062, + "logits/chosen": 0.12889467179775238, + "logits/rejected": 0.5754642486572266, + "logps/chosen": -1.2588895559310913, + "logps/rejected": -10.60193920135498, + "loss": 3.0992, + "nll_loss": 3.080859422683716, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12588895857334137, + "rewards/margins": 0.9343051314353943, + "rewards/rejected": -1.0601940155029297, + "step": 1244 + }, + { + "epoch": 0.7744945567651633, + "grad_norm": 0.4508911073207855, + "learning_rate": 1.8875e-05, + "log_odds_chosen": 9.714991569519043, + "log_odds_ratio": -0.18170523643493652, + "logits/chosen": 0.19511641561985016, + "logits/rejected": 1.2209863662719727, + "logps/chosen": -1.2065391540527344, + "logps/rejected": -10.629037857055664, + "loss": 2.6773, + "nll_loss": 2.659090042114258, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1206539198756218, + "rewards/margins": 0.9422498941421509, + "rewards/rejected": -1.0629037618637085, + "step": 1245 + }, + { + "epoch": 0.7751166407465008, + "grad_norm": 0.8513448238372803, + "learning_rate": 1.885e-05, + "log_odds_chosen": 13.84284782409668, + "log_odds_ratio": -0.002261546440422535, + "logits/chosen": 0.2347557246685028, + "logits/rejected": 0.9577349424362183, + "logps/chosen": -1.4056241512298584, + "logps/rejected": -14.550141334533691, + "loss": 3.0029, + "nll_loss": 3.002654552459717, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14056241512298584, + "rewards/margins": 1.3144516944885254, + "rewards/rejected": -1.4550142288208008, + "step": 1246 + }, + { + "epoch": 0.7757387247278382, + "grad_norm": 0.7846004962921143, + "learning_rate": 1.8825e-05, + "log_odds_chosen": 6.237817764282227, + "log_odds_ratio": -0.4033685028553009, + "logits/chosen": 0.2902659773826599, + "logits/rejected": 0.553649365901947, + "logps/chosen": -1.4968538284301758, + "logps/rejected": -7.529121398925781, + "loss": 3.3566, + "nll_loss": 3.3162434101104736, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14968539774417877, + "rewards/margins": 0.6032267808914185, + "rewards/rejected": -0.7529121041297913, + "step": 1247 + }, + { + "epoch": 0.7763608087091758, + "grad_norm": 0.4592325985431671, + "learning_rate": 1.88e-05, + "log_odds_chosen": 7.852267265319824, + "log_odds_ratio": -0.09105556458234787, + "logits/chosen": 0.3451773524284363, + "logits/rejected": 0.6059900522232056, + "logps/chosen": -0.6744275093078613, + "logps/rejected": -7.7031354904174805, + "loss": 3.3566, + "nll_loss": 3.3474724292755127, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06744275242090225, + "rewards/margins": 0.7028707265853882, + "rewards/rejected": -0.7703135013580322, + "step": 1248 + }, + { + "epoch": 0.7769828926905132, + "grad_norm": 0.5257586240768433, + "learning_rate": 1.8775000000000002e-05, + "log_odds_chosen": 9.442645072937012, + "log_odds_ratio": -0.26668232679367065, + "logits/chosen": 0.2947607636451721, + "logits/rejected": 0.8265994787216187, + "logps/chosen": -1.1143666505813599, + "logps/rejected": -10.230154991149902, + "loss": 3.19, + "nll_loss": 3.163308620452881, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11143667250871658, + "rewards/margins": 0.911578893661499, + "rewards/rejected": -1.0230154991149902, + "step": 1249 + }, + { + "epoch": 0.7776049766718507, + "grad_norm": 0.6123740077018738, + "learning_rate": 1.8750000000000002e-05, + "log_odds_chosen": 11.2650146484375, + "log_odds_ratio": -0.08563439548015594, + "logits/chosen": 0.25241217017173767, + "logits/rejected": 1.0535790920257568, + "logps/chosen": -0.7609401345252991, + "logps/rejected": -11.140617370605469, + "loss": 2.7892, + "nll_loss": 2.780632972717285, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07609401643276215, + "rewards/margins": 1.0379679203033447, + "rewards/rejected": -1.1140618324279785, + "step": 1250 + }, + { + "epoch": 0.7782270606531881, + "grad_norm": 0.43171098828315735, + "learning_rate": 1.8725e-05, + "log_odds_chosen": 8.13177490234375, + "log_odds_ratio": -0.1548667997121811, + "logits/chosen": 0.23373821377754211, + "logits/rejected": 0.5560605525970459, + "logps/chosen": -0.9117122888565063, + "logps/rejected": -8.333196640014648, + "loss": 3.3968, + "nll_loss": 3.3813085556030273, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09117123484611511, + "rewards/margins": 0.7421483397483826, + "rewards/rejected": -0.8333196640014648, + "step": 1251 + }, + { + "epoch": 0.7788491446345257, + "grad_norm": 0.5646179914474487, + "learning_rate": 1.87e-05, + "log_odds_chosen": 9.996572494506836, + "log_odds_ratio": -0.07235460728406906, + "logits/chosen": 0.1836574822664261, + "logits/rejected": 0.9524223208427429, + "logps/chosen": -1.001600980758667, + "logps/rejected": -10.505570411682129, + "loss": 2.9528, + "nll_loss": 2.945585250854492, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10016010701656342, + "rewards/margins": 0.9503970146179199, + "rewards/rejected": -1.0505571365356445, + "step": 1252 + }, + { + "epoch": 0.7794712286158632, + "grad_norm": 0.5360233783721924, + "learning_rate": 1.8675e-05, + "log_odds_chosen": 8.815251350402832, + "log_odds_ratio": -0.2755727171897888, + "logits/chosen": 0.17109717428684235, + "logits/rejected": 0.4950663447380066, + "logps/chosen": -0.9574038982391357, + "logps/rejected": -9.33325481414795, + "loss": 2.9651, + "nll_loss": 2.9375319480895996, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09574039280414581, + "rewards/margins": 0.8375850915908813, + "rewards/rejected": -0.9333254098892212, + "step": 1253 + }, + { + "epoch": 0.7800933125972006, + "grad_norm": 0.5576890110969543, + "learning_rate": 1.865e-05, + "log_odds_chosen": 12.056558609008789, + "log_odds_ratio": -0.0050803543999791145, + "logits/chosen": 0.157872274518013, + "logits/rejected": 0.9030807614326477, + "logps/chosen": -0.9107524156570435, + "logps/rejected": -12.303834915161133, + "loss": 2.8042, + "nll_loss": 2.8037121295928955, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09107524901628494, + "rewards/margins": 1.1393083333969116, + "rewards/rejected": -1.2303835153579712, + "step": 1254 + }, + { + "epoch": 0.7807153965785381, + "grad_norm": 0.4668339788913727, + "learning_rate": 1.8625000000000002e-05, + "log_odds_chosen": 7.626652717590332, + "log_odds_ratio": -0.1740056425333023, + "logits/chosen": 0.19568422436714172, + "logits/rejected": 0.7308311462402344, + "logps/chosen": -1.2841660976409912, + "logps/rejected": -8.55526351928711, + "loss": 3.0909, + "nll_loss": 3.0734806060791016, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12841661274433136, + "rewards/margins": 0.7271097898483276, + "rewards/rejected": -0.8555263876914978, + "step": 1255 + }, + { + "epoch": 0.7813374805598756, + "grad_norm": 0.7606042623519897, + "learning_rate": 1.86e-05, + "log_odds_chosen": 11.425827026367188, + "log_odds_ratio": -0.06626426428556442, + "logits/chosen": 0.22620922327041626, + "logits/rejected": 1.0646039247512817, + "logps/chosen": -1.135450005531311, + "logps/rejected": -11.916156768798828, + "loss": 3.2562, + "nll_loss": 3.249558448791504, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1135450154542923, + "rewards/margins": 1.0780707597732544, + "rewards/rejected": -1.1916157007217407, + "step": 1256 + }, + { + "epoch": 0.7819595645412131, + "grad_norm": 0.6749795079231262, + "learning_rate": 1.8575e-05, + "log_odds_chosen": 10.228057861328125, + "log_odds_ratio": -0.09839068353176117, + "logits/chosen": 0.22929708659648895, + "logits/rejected": 0.9187569618225098, + "logps/chosen": -1.2574495077133179, + "logps/rejected": -11.117281913757324, + "loss": 2.9522, + "nll_loss": 2.942385196685791, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12574495375156403, + "rewards/margins": 0.9859832525253296, + "rewards/rejected": -1.1117281913757324, + "step": 1257 + }, + { + "epoch": 0.7825816485225505, + "grad_norm": 0.4381241500377655, + "learning_rate": 1.855e-05, + "log_odds_chosen": 15.236520767211914, + "log_odds_ratio": -0.2253989428281784, + "logits/chosen": 0.24419057369232178, + "logits/rejected": 1.12411367893219, + "logps/chosen": -0.9931966066360474, + "logps/rejected": -15.840814590454102, + "loss": 3.4943, + "nll_loss": 3.4717283248901367, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09931966662406921, + "rewards/margins": 1.4847617149353027, + "rewards/rejected": -1.5840815305709839, + "step": 1258 + }, + { + "epoch": 0.783203732503888, + "grad_norm": 0.5113958120346069, + "learning_rate": 1.8525e-05, + "log_odds_chosen": 15.052928924560547, + "log_odds_ratio": -0.012858058325946331, + "logits/chosen": 0.1784391850233078, + "logits/rejected": 1.2252875566482544, + "logps/chosen": -1.29570734500885, + "logps/rejected": -16.01512336730957, + "loss": 2.7647, + "nll_loss": 2.7634403705596924, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12957073748111725, + "rewards/margins": 1.4719417095184326, + "rewards/rejected": -1.6015124320983887, + "step": 1259 + }, + { + "epoch": 0.7838258164852255, + "grad_norm": 0.6102534532546997, + "learning_rate": 1.85e-05, + "log_odds_chosen": 17.90291976928711, + "log_odds_ratio": -3.650830876722466e-06, + "logits/chosen": 0.19943833351135254, + "logits/rejected": 1.5740082263946533, + "logps/chosen": -1.2210125923156738, + "logps/rejected": -18.73495101928711, + "loss": 2.8752, + "nll_loss": 2.875162363052368, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12210125476121902, + "rewards/margins": 1.7513937950134277, + "rewards/rejected": -1.8734949827194214, + "step": 1260 + }, + { + "epoch": 0.784447900466563, + "grad_norm": 1.0051997900009155, + "learning_rate": 1.8475000000000002e-05, + "log_odds_chosen": 13.351705551147461, + "log_odds_ratio": -0.005376896820962429, + "logits/chosen": 0.1484776884317398, + "logits/rejected": 0.7685915231704712, + "logps/chosen": -1.3591516017913818, + "logps/rejected": -14.330547332763672, + "loss": 2.9603, + "nll_loss": 2.959794044494629, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13591517508029938, + "rewards/margins": 1.2971396446228027, + "rewards/rejected": -1.433054804801941, + "step": 1261 + }, + { + "epoch": 0.7850699844479004, + "grad_norm": 0.5121684670448303, + "learning_rate": 1.845e-05, + "log_odds_chosen": 15.99807357788086, + "log_odds_ratio": -0.12559840083122253, + "logits/chosen": 0.23574836552143097, + "logits/rejected": 0.7866611480712891, + "logps/chosen": -0.8544904589653015, + "logps/rejected": -16.086206436157227, + "loss": 2.9929, + "nll_loss": 2.98030424118042, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08544904738664627, + "rewards/margins": 1.5231716632843018, + "rewards/rejected": -1.6086206436157227, + "step": 1262 + }, + { + "epoch": 0.785692068429238, + "grad_norm": 0.6973137855529785, + "learning_rate": 1.8425e-05, + "log_odds_chosen": 17.51459503173828, + "log_odds_ratio": -0.104143887758255, + "logits/chosen": 0.3150438666343689, + "logits/rejected": 1.629845142364502, + "logps/chosen": -1.1097285747528076, + "logps/rejected": -18.256704330444336, + "loss": 2.4323, + "nll_loss": 2.4219167232513428, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11097285151481628, + "rewards/margins": 1.7146975994110107, + "rewards/rejected": -1.8256704807281494, + "step": 1263 + }, + { + "epoch": 0.7863141524105755, + "grad_norm": 3.6229288578033447, + "learning_rate": 1.84e-05, + "log_odds_chosen": 15.211271286010742, + "log_odds_ratio": -0.15237554907798767, + "logits/chosen": 0.19041389226913452, + "logits/rejected": 1.0611919164657593, + "logps/chosen": -1.5682026147842407, + "logps/rejected": -16.546676635742188, + "loss": 2.726, + "nll_loss": 2.710737943649292, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15682026743888855, + "rewards/margins": 1.497847557067871, + "rewards/rejected": -1.654667615890503, + "step": 1264 + }, + { + "epoch": 0.7869362363919129, + "grad_norm": 0.43393415212631226, + "learning_rate": 1.8375e-05, + "log_odds_chosen": 15.434120178222656, + "log_odds_ratio": -0.004888943396508694, + "logits/chosen": 0.360336035490036, + "logits/rejected": 1.6232982873916626, + "logps/chosen": -0.9606151580810547, + "logps/rejected": -15.767781257629395, + "loss": 3.4923, + "nll_loss": 3.4917798042297363, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09606151282787323, + "rewards/margins": 1.4807167053222656, + "rewards/rejected": -1.5767782926559448, + "step": 1265 + }, + { + "epoch": 0.7875583203732504, + "grad_norm": 0.41161367297172546, + "learning_rate": 1.8350000000000002e-05, + "log_odds_chosen": 10.203583717346191, + "log_odds_ratio": -0.26046648621559143, + "logits/chosen": 0.22639605402946472, + "logits/rejected": 0.8604259490966797, + "logps/chosen": -1.0054662227630615, + "logps/rejected": -10.743423461914062, + "loss": 3.1646, + "nll_loss": 3.138522148132324, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10054662823677063, + "rewards/margins": 0.9737956523895264, + "rewards/rejected": -1.0743423700332642, + "step": 1266 + }, + { + "epoch": 0.7881804043545879, + "grad_norm": 5.74237585067749, + "learning_rate": 1.8325e-05, + "log_odds_chosen": 7.807779312133789, + "log_odds_ratio": -0.18478822708129883, + "logits/chosen": 0.19614526629447937, + "logits/rejected": 0.776382565498352, + "logps/chosen": -1.2011544704437256, + "logps/rejected": -8.654647827148438, + "loss": 3.0134, + "nll_loss": 2.994901418685913, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12011544406414032, + "rewards/margins": 0.7453492879867554, + "rewards/rejected": -0.8654647469520569, + "step": 1267 + }, + { + "epoch": 0.7888024883359254, + "grad_norm": 0.4839012026786804, + "learning_rate": 1.83e-05, + "log_odds_chosen": 17.199420928955078, + "log_odds_ratio": -0.14982233941555023, + "logits/chosen": 0.3670274615287781, + "logits/rejected": 1.3338403701782227, + "logps/chosen": -1.1166799068450928, + "logps/rejected": -17.874608993530273, + "loss": 3.4317, + "nll_loss": 3.416668176651001, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11166798323392868, + "rewards/margins": 1.6757930517196655, + "rewards/rejected": -1.7874610424041748, + "step": 1268 + }, + { + "epoch": 0.7894245723172628, + "grad_norm": 1.5299826860427856, + "learning_rate": 1.8275e-05, + "log_odds_chosen": 11.565797805786133, + "log_odds_ratio": -0.22419725358486176, + "logits/chosen": 0.14468568563461304, + "logits/rejected": 0.8828893303871155, + "logps/chosen": -1.188100814819336, + "logps/rejected": -12.22629451751709, + "loss": 2.772, + "nll_loss": 2.749605655670166, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11881007999181747, + "rewards/margins": 1.1038193702697754, + "rewards/rejected": -1.222629427909851, + "step": 1269 + }, + { + "epoch": 0.7900466562986003, + "grad_norm": 0.3719753324985504, + "learning_rate": 1.825e-05, + "log_odds_chosen": 12.311816215515137, + "log_odds_ratio": -0.007772353012114763, + "logits/chosen": 0.21790693700313568, + "logits/rejected": 1.1610301733016968, + "logps/chosen": -1.1040740013122559, + "logps/rejected": -12.842395782470703, + "loss": 3.4675, + "nll_loss": 3.4667301177978516, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11040741205215454, + "rewards/margins": 1.1738322973251343, + "rewards/rejected": -1.284239649772644, + "step": 1270 + }, + { + "epoch": 0.7906687402799378, + "grad_norm": 0.575977087020874, + "learning_rate": 1.8225e-05, + "log_odds_chosen": 10.100236892700195, + "log_odds_ratio": -0.13360543549060822, + "logits/chosen": 0.1812443733215332, + "logits/rejected": 0.5013841390609741, + "logps/chosen": -0.996255099773407, + "logps/rejected": -10.562983512878418, + "loss": 3.1107, + "nll_loss": 3.097329616546631, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09962552040815353, + "rewards/margins": 0.9566728472709656, + "rewards/rejected": -1.0562983751296997, + "step": 1271 + }, + { + "epoch": 0.7912908242612753, + "grad_norm": 0.547460675239563, + "learning_rate": 1.8200000000000002e-05, + "log_odds_chosen": 7.538238048553467, + "log_odds_ratio": -0.10196790844202042, + "logits/chosen": -0.03788933530449867, + "logits/rejected": 0.4808521270751953, + "logps/chosen": -1.1122915744781494, + "logps/rejected": -8.138171195983887, + "loss": 2.3522, + "nll_loss": 2.341975688934326, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11122915148735046, + "rewards/margins": 0.702587902545929, + "rewards/rejected": -0.8138170838356018, + "step": 1272 + }, + { + "epoch": 0.7919129082426127, + "grad_norm": 0.5085967183113098, + "learning_rate": 1.8175e-05, + "log_odds_chosen": 10.72020149230957, + "log_odds_ratio": -0.12706585228443146, + "logits/chosen": -0.05211986228823662, + "logits/rejected": 0.890866756439209, + "logps/chosen": -0.9368724822998047, + "logps/rejected": -11.051326751708984, + "loss": 2.2072, + "nll_loss": 2.19453763961792, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09368725121021271, + "rewards/margins": 1.01144540309906, + "rewards/rejected": -1.1051325798034668, + "step": 1273 + }, + { + "epoch": 0.7925349922239502, + "grad_norm": 0.5295844674110413, + "learning_rate": 1.815e-05, + "log_odds_chosen": 6.329776763916016, + "log_odds_ratio": -0.1255560964345932, + "logits/chosen": 0.12830644845962524, + "logits/rejected": 0.505806565284729, + "logps/chosen": -1.2834556102752686, + "logps/rejected": -7.303755283355713, + "loss": 3.2161, + "nll_loss": 3.2035398483276367, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1283455491065979, + "rewards/margins": 0.6020300388336182, + "rewards/rejected": -0.7303755879402161, + "step": 1274 + }, + { + "epoch": 0.7931570762052877, + "grad_norm": 0.45801711082458496, + "learning_rate": 1.8125e-05, + "log_odds_chosen": 7.928872108459473, + "log_odds_ratio": -0.16928905248641968, + "logits/chosen": 0.1209368109703064, + "logits/rejected": 0.31357628107070923, + "logps/chosen": -0.9660826325416565, + "logps/rejected": -8.398103713989258, + "loss": 3.2755, + "nll_loss": 3.2585678100585938, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09660826623439789, + "rewards/margins": 0.7432020306587219, + "rewards/rejected": -0.839810311794281, + "step": 1275 + }, + { + "epoch": 0.7937791601866252, + "grad_norm": 0.4788154363632202, + "learning_rate": 1.81e-05, + "log_odds_chosen": 14.399748802185059, + "log_odds_ratio": -0.00020558810501825064, + "logits/chosen": 0.13003887236118317, + "logits/rejected": 1.1363701820373535, + "logps/chosen": -0.9204753041267395, + "logps/rejected": -14.68822956085205, + "loss": 2.664, + "nll_loss": 2.663971185684204, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09204752743244171, + "rewards/margins": 1.3767753839492798, + "rewards/rejected": -1.468822956085205, + "step": 1276 + }, + { + "epoch": 0.7944012441679627, + "grad_norm": 0.4245346486568451, + "learning_rate": 1.8075e-05, + "log_odds_chosen": 9.295736312866211, + "log_odds_ratio": -0.05224517732858658, + "logits/chosen": 0.0967981144785881, + "logits/rejected": 1.0028589963912964, + "logps/chosen": -1.1117591857910156, + "logps/rejected": -9.941202163696289, + "loss": 2.4129, + "nll_loss": 2.4077231884002686, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11117591708898544, + "rewards/margins": 0.8829442262649536, + "rewards/rejected": -0.9941201210021973, + "step": 1277 + }, + { + "epoch": 0.7950233281493001, + "grad_norm": 0.46397557854652405, + "learning_rate": 1.805e-05, + "log_odds_chosen": 8.459473609924316, + "log_odds_ratio": -0.0300842747092247, + "logits/chosen": 0.11754687130451202, + "logits/rejected": 0.4644424617290497, + "logps/chosen": -0.7050676941871643, + "logps/rejected": -8.29973030090332, + "loss": 2.7823, + "nll_loss": 2.779327392578125, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07050677388906479, + "rewards/margins": 0.7594663500785828, + "rewards/rejected": -0.8299731016159058, + "step": 1278 + }, + { + "epoch": 0.7956454121306377, + "grad_norm": 0.4239655137062073, + "learning_rate": 1.8025e-05, + "log_odds_chosen": 9.256122589111328, + "log_odds_ratio": -0.001191396964713931, + "logits/chosen": 0.24952790141105652, + "logits/rejected": 0.8297731280326843, + "logps/chosen": -0.9855543971061707, + "logps/rejected": -9.722004890441895, + "loss": 3.376, + "nll_loss": 3.3759002685546875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09855544567108154, + "rewards/margins": 0.8736451268196106, + "rewards/rejected": -0.9722005724906921, + "step": 1279 + }, + { + "epoch": 0.7962674961119751, + "grad_norm": 0.5233640670776367, + "learning_rate": 1.8e-05, + "log_odds_chosen": 4.858011245727539, + "log_odds_ratio": -0.24006202816963196, + "logits/chosen": 0.10965058207511902, + "logits/rejected": 0.2803415358066559, + "logps/chosen": -0.7928171157836914, + "logps/rejected": -5.210036277770996, + "loss": 2.898, + "nll_loss": 2.8740434646606445, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.07928171753883362, + "rewards/margins": 0.44172191619873047, + "rewards/rejected": -0.5210036039352417, + "step": 1280 + }, + { + "epoch": 0.7968895800933126, + "grad_norm": 0.49103713035583496, + "learning_rate": 1.7975e-05, + "log_odds_chosen": 8.95180892944336, + "log_odds_ratio": -0.08506196737289429, + "logits/chosen": 0.14098000526428223, + "logits/rejected": 0.6167709231376648, + "logps/chosen": -0.9210264086723328, + "logps/rejected": -9.39538860321045, + "loss": 2.9812, + "nll_loss": 2.9726974964141846, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09210263937711716, + "rewards/margins": 0.8474361896514893, + "rewards/rejected": -0.9395389556884766, + "step": 1281 + }, + { + "epoch": 0.7975116640746501, + "grad_norm": 0.3628371059894562, + "learning_rate": 1.795e-05, + "log_odds_chosen": 12.051719665527344, + "log_odds_ratio": -0.0018485789187252522, + "logits/chosen": 0.18384167551994324, + "logits/rejected": 0.9371912479400635, + "logps/chosen": -1.0759706497192383, + "logps/rejected": -12.439555168151855, + "loss": 3.1226, + "nll_loss": 3.1224188804626465, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10759708285331726, + "rewards/margins": 1.136358380317688, + "rewards/rejected": -1.2439554929733276, + "step": 1282 + }, + { + "epoch": 0.7981337480559876, + "grad_norm": 0.5358335971832275, + "learning_rate": 1.7925e-05, + "log_odds_chosen": 8.796299934387207, + "log_odds_ratio": -0.15110205113887787, + "logits/chosen": 0.31277087330818176, + "logits/rejected": 1.0025912523269653, + "logps/chosen": -0.9938040971755981, + "logps/rejected": -9.312862396240234, + "loss": 2.9431, + "nll_loss": 2.9279847145080566, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09938041865825653, + "rewards/margins": 0.8319058418273926, + "rewards/rejected": -0.9312862157821655, + "step": 1283 + }, + { + "epoch": 0.798755832037325, + "grad_norm": 0.3374326825141907, + "learning_rate": 1.79e-05, + "log_odds_chosen": 9.138998031616211, + "log_odds_ratio": -0.11299619823694229, + "logits/chosen": 0.2334498018026352, + "logits/rejected": 0.8283179998397827, + "logps/chosen": -1.2180832624435425, + "logps/rejected": -10.032180786132812, + "loss": 3.4034, + "nll_loss": 3.3921141624450684, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12180833518505096, + "rewards/margins": 0.881409764289856, + "rewards/rejected": -1.003218173980713, + "step": 1284 + }, + { + "epoch": 0.7993779160186625, + "grad_norm": 0.6933339238166809, + "learning_rate": 1.7875e-05, + "log_odds_chosen": 13.547820091247559, + "log_odds_ratio": -0.1648188680410385, + "logits/chosen": 0.10697955638170242, + "logits/rejected": 0.722811758518219, + "logps/chosen": -1.0336053371429443, + "logps/rejected": -14.206631660461426, + "loss": 2.5359, + "nll_loss": 2.5193803310394287, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10336053371429443, + "rewards/margins": 1.3173027038574219, + "rewards/rejected": -1.4206632375717163, + "step": 1285 + }, + { + "epoch": 0.8, + "grad_norm": 0.6732850074768066, + "learning_rate": 1.785e-05, + "log_odds_chosen": 7.440114974975586, + "log_odds_ratio": -0.18558666110038757, + "logits/chosen": 0.32390734553337097, + "logits/rejected": 0.7920575141906738, + "logps/chosen": -1.029671549797058, + "logps/rejected": -7.974099159240723, + "loss": 3.3675, + "nll_loss": 3.3489205837249756, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10296716541051865, + "rewards/margins": 0.6944427490234375, + "rewards/rejected": -0.7974098920822144, + "step": 1286 + }, + { + "epoch": 0.8006220839813375, + "grad_norm": 0.776902973651886, + "learning_rate": 1.7825e-05, + "log_odds_chosen": 12.03702449798584, + "log_odds_ratio": -0.09163986146450043, + "logits/chosen": 0.3653428256511688, + "logits/rejected": 1.1969035863876343, + "logps/chosen": -1.0206536054611206, + "logps/rejected": -12.635235786437988, + "loss": 3.414, + "nll_loss": 3.4048497676849365, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10206536203622818, + "rewards/margins": 1.1614582538604736, + "rewards/rejected": -1.2635235786437988, + "step": 1287 + }, + { + "epoch": 0.801244167962675, + "grad_norm": 0.5694941282272339, + "learning_rate": 1.78e-05, + "log_odds_chosen": 13.658385276794434, + "log_odds_ratio": -0.10558748990297318, + "logits/chosen": 0.21393881738185883, + "logits/rejected": 0.8981702923774719, + "logps/chosen": -1.2607628107070923, + "logps/rejected": -14.39406967163086, + "loss": 2.6656, + "nll_loss": 2.6550426483154297, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12607628107070923, + "rewards/margins": 1.3133307695388794, + "rewards/rejected": -1.4394071102142334, + "step": 1288 + }, + { + "epoch": 0.8018662519440124, + "grad_norm": 0.4627145230770111, + "learning_rate": 1.7775e-05, + "log_odds_chosen": 8.23801040649414, + "log_odds_ratio": -0.3038644790649414, + "logits/chosen": 0.16624096035957336, + "logits/rejected": 0.5100173950195312, + "logps/chosen": -1.0921432971954346, + "logps/rejected": -8.8154296875, + "loss": 3.0926, + "nll_loss": 3.0622284412384033, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10921433568000793, + "rewards/margins": 0.7723287343978882, + "rewards/rejected": -0.8815430402755737, + "step": 1289 + }, + { + "epoch": 0.80248833592535, + "grad_norm": 0.39549392461776733, + "learning_rate": 1.775e-05, + "log_odds_chosen": 10.66400146484375, + "log_odds_ratio": -0.07293321192264557, + "logits/chosen": 0.40711355209350586, + "logits/rejected": 0.8906251192092896, + "logps/chosen": -1.3841667175292969, + "logps/rejected": -11.750348091125488, + "loss": 4.0269, + "nll_loss": 4.019584655761719, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13841667771339417, + "rewards/margins": 1.0366181135177612, + "rewards/rejected": -1.175034761428833, + "step": 1290 + }, + { + "epoch": 0.8031104199066874, + "grad_norm": 0.5006654858589172, + "learning_rate": 1.7725e-05, + "log_odds_chosen": 10.659064292907715, + "log_odds_ratio": -0.013071035034954548, + "logits/chosen": 0.2032037228345871, + "logits/rejected": 1.0664845705032349, + "logps/chosen": -1.228370189666748, + "logps/rejected": -11.474655151367188, + "loss": 2.8391, + "nll_loss": 2.8378067016601562, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12283702194690704, + "rewards/margins": 1.0246284008026123, + "rewards/rejected": -1.147465467453003, + "step": 1291 + }, + { + "epoch": 0.8037325038880249, + "grad_norm": 0.47223320603370667, + "learning_rate": 1.77e-05, + "log_odds_chosen": 13.39532470703125, + "log_odds_ratio": -0.002649889327585697, + "logits/chosen": 0.1537506878376007, + "logits/rejected": 0.9822484254837036, + "logps/chosen": -1.164747714996338, + "logps/rejected": -14.148645401000977, + "loss": 3.4009, + "nll_loss": 3.400646448135376, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11647477746009827, + "rewards/margins": 1.2983897924423218, + "rewards/rejected": -1.4148645401000977, + "step": 1292 + }, + { + "epoch": 0.8043545878693623, + "grad_norm": 0.5693787336349487, + "learning_rate": 1.7675e-05, + "log_odds_chosen": 12.302108764648438, + "log_odds_ratio": -0.1323574185371399, + "logits/chosen": 0.2963750958442688, + "logits/rejected": 1.2182402610778809, + "logps/chosen": -1.1627578735351562, + "logps/rejected": -13.141119956970215, + "loss": 2.8084, + "nll_loss": 2.7951581478118896, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11627580225467682, + "rewards/margins": 1.1978362798690796, + "rewards/rejected": -1.3141121864318848, + "step": 1293 + }, + { + "epoch": 0.8049766718506999, + "grad_norm": 0.444240540266037, + "learning_rate": 1.765e-05, + "log_odds_chosen": 12.026140213012695, + "log_odds_ratio": -0.12117686867713928, + "logits/chosen": 0.18682369589805603, + "logits/rejected": 0.8227996230125427, + "logps/chosen": -0.9900913238525391, + "logps/rejected": -12.605978012084961, + "loss": 3.1138, + "nll_loss": 3.101719856262207, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09900914132595062, + "rewards/margins": 1.1615886688232422, + "rewards/rejected": -1.2605977058410645, + "step": 1294 + }, + { + "epoch": 0.8055987558320373, + "grad_norm": 0.5121841430664062, + "learning_rate": 1.7625e-05, + "log_odds_chosen": 8.10305404663086, + "log_odds_ratio": -0.12781129777431488, + "logits/chosen": 0.24159939587116241, + "logits/rejected": 0.5816590785980225, + "logps/chosen": -0.9372666478157043, + "logps/rejected": -8.444114685058594, + "loss": 3.1081, + "nll_loss": 3.0953598022460938, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09372667223215103, + "rewards/margins": 0.7506847381591797, + "rewards/rejected": -0.8444114923477173, + "step": 1295 + }, + { + "epoch": 0.8062208398133748, + "grad_norm": 0.6881003975868225, + "learning_rate": 1.76e-05, + "log_odds_chosen": 11.331974983215332, + "log_odds_ratio": -0.11589650809764862, + "logits/chosen": 0.15356194972991943, + "logits/rejected": 0.8771441578865051, + "logps/chosen": -0.9373379349708557, + "logps/rejected": -11.583261489868164, + "loss": 2.9828, + "nll_loss": 2.9712398052215576, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09373379498720169, + "rewards/margins": 1.0645923614501953, + "rewards/rejected": -1.1583261489868164, + "step": 1296 + }, + { + "epoch": 0.8068429237947123, + "grad_norm": 0.5609170794487, + "learning_rate": 1.7575e-05, + "log_odds_chosen": 14.964519500732422, + "log_odds_ratio": -0.2053382247686386, + "logits/chosen": 0.22482171654701233, + "logits/rejected": 0.9772385358810425, + "logps/chosen": -0.9464631080627441, + "logps/rejected": -15.543478965759277, + "loss": 2.9799, + "nll_loss": 2.959413528442383, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09464631229639053, + "rewards/margins": 1.4597017765045166, + "rewards/rejected": -1.5543479919433594, + "step": 1297 + }, + { + "epoch": 0.8074650077760498, + "grad_norm": 0.5253718495368958, + "learning_rate": 1.755e-05, + "log_odds_chosen": 13.896406173706055, + "log_odds_ratio": -0.15764853358268738, + "logits/chosen": 0.3127559423446655, + "logits/rejected": 1.3013142347335815, + "logps/chosen": -1.011033296585083, + "logps/rejected": -14.529682159423828, + "loss": 3.3237, + "nll_loss": 3.3079025745391846, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10110333561897278, + "rewards/margins": 1.3518648147583008, + "rewards/rejected": -1.4529681205749512, + "step": 1298 + }, + { + "epoch": 0.8080870917573872, + "grad_norm": 0.4803771376609802, + "learning_rate": 1.7525e-05, + "log_odds_chosen": 9.528873443603516, + "log_odds_ratio": -0.077350914478302, + "logits/chosen": 0.21042805910110474, + "logits/rejected": 0.8243046998977661, + "logps/chosen": -0.8366771936416626, + "logps/rejected": -9.790960311889648, + "loss": 2.9032, + "nll_loss": 2.8954741954803467, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08366772532463074, + "rewards/margins": 0.8954282999038696, + "rewards/rejected": -0.9790959358215332, + "step": 1299 + }, + { + "epoch": 0.8087091757387247, + "grad_norm": 5.892223834991455, + "learning_rate": 1.75e-05, + "log_odds_chosen": 4.356164932250977, + "log_odds_ratio": -0.544304370880127, + "logits/chosen": 0.14717110991477966, + "logits/rejected": 0.4145839810371399, + "logps/chosen": -2.111706018447876, + "logps/rejected": -6.22900915145874, + "loss": 3.3116, + "nll_loss": 3.2571868896484375, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.21117059886455536, + "rewards/margins": 0.4117303192615509, + "rewards/rejected": -0.6229009628295898, + "step": 1300 + }, + { + "epoch": 0.8093312597200623, + "grad_norm": 0.5840747952461243, + "learning_rate": 1.7475e-05, + "log_odds_chosen": 14.291379928588867, + "log_odds_ratio": -0.12850309908390045, + "logits/chosen": 0.1304207295179367, + "logits/rejected": 0.9051514267921448, + "logps/chosen": -0.6998139023780823, + "logps/rejected": -14.083181381225586, + "loss": 2.7482, + "nll_loss": 2.7353763580322266, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.06998138874769211, + "rewards/margins": 1.3383368253707886, + "rewards/rejected": -1.4083181619644165, + "step": 1301 + }, + { + "epoch": 0.8099533437013997, + "grad_norm": 0.679797887802124, + "learning_rate": 1.745e-05, + "log_odds_chosen": 10.464731216430664, + "log_odds_ratio": -0.007538243196904659, + "logits/chosen": 0.2101728618144989, + "logits/rejected": 0.9237760901451111, + "logps/chosen": -1.0783690214157104, + "logps/rejected": -11.021986961364746, + "loss": 2.3857, + "nll_loss": 2.384983539581299, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10783690214157104, + "rewards/margins": 0.9943617582321167, + "rewards/rejected": -1.1021987199783325, + "step": 1302 + }, + { + "epoch": 0.8105754276827372, + "grad_norm": 0.5377840399742126, + "learning_rate": 1.7425e-05, + "log_odds_chosen": 12.532342910766602, + "log_odds_ratio": -0.044694434851408005, + "logits/chosen": 0.15078105032444, + "logits/rejected": 0.8079558610916138, + "logps/chosen": -0.9499520659446716, + "logps/rejected": -12.720598220825195, + "loss": 2.845, + "nll_loss": 2.840527057647705, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09499520808458328, + "rewards/margins": 1.1770645380020142, + "rewards/rejected": -1.2720599174499512, + "step": 1303 + }, + { + "epoch": 0.8111975116640746, + "grad_norm": 0.5700002312660217, + "learning_rate": 1.74e-05, + "log_odds_chosen": 9.45265007019043, + "log_odds_ratio": -0.24174627661705017, + "logits/chosen": 0.28236669301986694, + "logits/rejected": 0.8490161299705505, + "logps/chosen": -1.2051925659179688, + "logps/rejected": -10.107574462890625, + "loss": 3.3028, + "nll_loss": 3.278662919998169, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.120519258081913, + "rewards/margins": 0.8902381658554077, + "rewards/rejected": -1.0107574462890625, + "step": 1304 + }, + { + "epoch": 0.8118195956454122, + "grad_norm": 0.4791751503944397, + "learning_rate": 1.7375e-05, + "log_odds_chosen": 15.476375579833984, + "log_odds_ratio": -0.00826264824718237, + "logits/chosen": 0.29491129517555237, + "logits/rejected": 0.9962074756622314, + "logps/chosen": -1.0027536153793335, + "logps/rejected": -15.781121253967285, + "loss": 3.181, + "nll_loss": 3.180189609527588, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10027536749839783, + "rewards/margins": 1.4778368473052979, + "rewards/rejected": -1.5781121253967285, + "step": 1305 + }, + { + "epoch": 0.8124416796267496, + "grad_norm": 0.5712665915489197, + "learning_rate": 1.7349999999999998e-05, + "log_odds_chosen": 9.660453796386719, + "log_odds_ratio": -0.08943241089582443, + "logits/chosen": 0.12958277761936188, + "logits/rejected": 0.5500991940498352, + "logps/chosen": -1.430905818939209, + "logps/rejected": -10.829052925109863, + "loss": 3.432, + "nll_loss": 3.4230165481567383, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14309059083461761, + "rewards/margins": 0.9398147463798523, + "rewards/rejected": -1.0829052925109863, + "step": 1306 + }, + { + "epoch": 0.8130637636080871, + "grad_norm": 0.514840841293335, + "learning_rate": 1.7325e-05, + "log_odds_chosen": 13.178714752197266, + "log_odds_ratio": -0.005788641981780529, + "logits/chosen": 0.336585134267807, + "logits/rejected": 1.2146720886230469, + "logps/chosen": -1.1430599689483643, + "logps/rejected": -13.882668495178223, + "loss": 3.133, + "nll_loss": 3.1323795318603516, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1143060028553009, + "rewards/margins": 1.273960828781128, + "rewards/rejected": -1.3882668018341064, + "step": 1307 + }, + { + "epoch": 0.8136858475894245, + "grad_norm": 0.6594363451004028, + "learning_rate": 1.73e-05, + "log_odds_chosen": 10.469732284545898, + "log_odds_ratio": -0.1577916145324707, + "logits/chosen": 0.3112362027168274, + "logits/rejected": 1.0076191425323486, + "logps/chosen": -1.172290563583374, + "logps/rejected": -11.286930084228516, + "loss": 2.9662, + "nll_loss": 2.9504287242889404, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11722905933856964, + "rewards/margins": 1.0114638805389404, + "rewards/rejected": -1.128692865371704, + "step": 1308 + }, + { + "epoch": 0.8143079315707621, + "grad_norm": 0.5135230422019958, + "learning_rate": 1.7275e-05, + "log_odds_chosen": 6.890636444091797, + "log_odds_ratio": -0.19977234303951263, + "logits/chosen": 0.11931162327528, + "logits/rejected": 0.6851118803024292, + "logps/chosen": -1.028935432434082, + "logps/rejected": -7.384231090545654, + "loss": 2.6068, + "nll_loss": 2.586785316467285, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10289355367422104, + "rewards/margins": 0.6355295181274414, + "rewards/rejected": -0.7384231090545654, + "step": 1309 + }, + { + "epoch": 0.8149300155520995, + "grad_norm": 0.45454153418540955, + "learning_rate": 1.725e-05, + "log_odds_chosen": 11.916482925415039, + "log_odds_ratio": -0.22180667519569397, + "logits/chosen": 0.1590307205915451, + "logits/rejected": 0.8525173664093018, + "logps/chosen": -1.089916706085205, + "logps/rejected": -12.624712944030762, + "loss": 2.9198, + "nll_loss": 2.897609233856201, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10899168252944946, + "rewards/margins": 1.1534795761108398, + "rewards/rejected": -1.2624711990356445, + "step": 1310 + }, + { + "epoch": 0.815552099533437, + "grad_norm": 0.467376172542572, + "learning_rate": 1.7225e-05, + "log_odds_chosen": 15.533833503723145, + "log_odds_ratio": -1.8030461887974525e-06, + "logits/chosen": 0.17015010118484497, + "logits/rejected": 0.8839508295059204, + "logps/chosen": -1.1618465185165405, + "logps/rejected": -16.2960147857666, + "loss": 3.1071, + "nll_loss": 3.1071434020996094, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11618465930223465, + "rewards/margins": 1.5134167671203613, + "rewards/rejected": -1.6296014785766602, + "step": 1311 + }, + { + "epoch": 0.8161741835147744, + "grad_norm": 0.5810384750366211, + "learning_rate": 1.7199999999999998e-05, + "log_odds_chosen": 5.953010559082031, + "log_odds_ratio": -0.20768365263938904, + "logits/chosen": 0.2892778515815735, + "logits/rejected": 0.5529869198799133, + "logps/chosen": -1.1497933864593506, + "logps/rejected": -6.790042400360107, + "loss": 3.2464, + "nll_loss": 3.2256784439086914, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1149793416261673, + "rewards/margins": 0.5640249252319336, + "rewards/rejected": -0.6790042519569397, + "step": 1312 + }, + { + "epoch": 0.816796267496112, + "grad_norm": 0.46326449513435364, + "learning_rate": 1.7175e-05, + "log_odds_chosen": 14.900135040283203, + "log_odds_ratio": -0.005678822286427021, + "logits/chosen": 0.25347965955734253, + "logits/rejected": 0.990155041217804, + "logps/chosen": -1.4208537340164185, + "logps/rejected": -15.883021354675293, + "loss": 3.3431, + "nll_loss": 3.34252667427063, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14208537340164185, + "rewards/margins": 1.4462168216705322, + "rewards/rejected": -1.5883021354675293, + "step": 1313 + }, + { + "epoch": 0.8174183514774495, + "grad_norm": 0.4483345150947571, + "learning_rate": 1.7150000000000004e-05, + "log_odds_chosen": 12.23083209991455, + "log_odds_ratio": -0.008293919265270233, + "logits/chosen": 0.32675403356552124, + "logits/rejected": 0.7867611050605774, + "logps/chosen": -0.9917525053024292, + "logps/rejected": -12.401079177856445, + "loss": 3.4186, + "nll_loss": 3.417818546295166, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09917524456977844, + "rewards/margins": 1.1409326791763306, + "rewards/rejected": -1.2401078939437866, + "step": 1314 + }, + { + "epoch": 0.8180404354587869, + "grad_norm": 0.3912915885448456, + "learning_rate": 1.7125000000000003e-05, + "log_odds_chosen": 15.708026885986328, + "log_odds_ratio": -9.631742432247847e-05, + "logits/chosen": 0.25834500789642334, + "logits/rejected": 0.781840980052948, + "logps/chosen": -1.0959607362747192, + "logps/rejected": -16.29987907409668, + "loss": 3.6535, + "nll_loss": 3.6535050868988037, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10959607362747192, + "rewards/margins": 1.5203917026519775, + "rewards/rejected": -1.6299877166748047, + "step": 1315 + }, + { + "epoch": 0.8186625194401245, + "grad_norm": 0.5564141869544983, + "learning_rate": 1.7100000000000002e-05, + "log_odds_chosen": 16.535018920898438, + "log_odds_ratio": -0.12745706737041473, + "logits/chosen": 0.30819642543792725, + "logits/rejected": 1.2198446989059448, + "logps/chosen": -1.0392193794250488, + "logps/rejected": -17.128646850585938, + "loss": 2.7837, + "nll_loss": 2.7709245681762695, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10392193496227264, + "rewards/margins": 1.6089427471160889, + "rewards/rejected": -1.7128647565841675, + "step": 1316 + }, + { + "epoch": 0.8192846034214619, + "grad_norm": 0.6600592136383057, + "learning_rate": 1.7075e-05, + "log_odds_chosen": 10.514989852905273, + "log_odds_ratio": -0.004317562095820904, + "logits/chosen": 0.23065491020679474, + "logits/rejected": 0.8288361430168152, + "logps/chosen": -1.0467638969421387, + "logps/rejected": -11.060022354125977, + "loss": 2.5248, + "nll_loss": 2.5243237018585205, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10467638820409775, + "rewards/margins": 1.0013258457183838, + "rewards/rejected": -1.1060023307800293, + "step": 1317 + }, + { + "epoch": 0.8199066874027994, + "grad_norm": 0.4944377839565277, + "learning_rate": 1.705e-05, + "log_odds_chosen": 11.389673233032227, + "log_odds_ratio": -0.06905973702669144, + "logits/chosen": 0.1772024929523468, + "logits/rejected": 0.6251535415649414, + "logps/chosen": -0.8641712665557861, + "logps/rejected": -11.602714538574219, + "loss": 2.9104, + "nll_loss": 2.903456687927246, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08641713857650757, + "rewards/margins": 1.0738542079925537, + "rewards/rejected": -1.160271406173706, + "step": 1318 + }, + { + "epoch": 0.8205287713841368, + "grad_norm": 0.5295370221138, + "learning_rate": 1.7025e-05, + "log_odds_chosen": 9.117376327514648, + "log_odds_ratio": -0.46799957752227783, + "logits/chosen": 0.22593256831169128, + "logits/rejected": 0.6932124495506287, + "logps/chosen": -1.3216909170150757, + "logps/rejected": -10.34849739074707, + "loss": 2.7903, + "nll_loss": 2.7435405254364014, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13216909766197205, + "rewards/margins": 0.9026807546615601, + "rewards/rejected": -1.0348498821258545, + "step": 1319 + }, + { + "epoch": 0.8211508553654744, + "grad_norm": 0.518757164478302, + "learning_rate": 1.7000000000000003e-05, + "log_odds_chosen": 12.372821807861328, + "log_odds_ratio": -0.2226611226797104, + "logits/chosen": 0.2919222414493561, + "logits/rejected": 0.8774810433387756, + "logps/chosen": -1.1752337217330933, + "logps/rejected": -13.215282440185547, + "loss": 3.1039, + "nll_loss": 3.0816164016723633, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11752337962388992, + "rewards/margins": 1.2040050029754639, + "rewards/rejected": -1.3215281963348389, + "step": 1320 + }, + { + "epoch": 0.8217729393468118, + "grad_norm": 0.4607633352279663, + "learning_rate": 1.6975000000000003e-05, + "log_odds_chosen": 11.539329528808594, + "log_odds_ratio": -0.10293762385845184, + "logits/chosen": 0.19818146526813507, + "logits/rejected": 1.1686244010925293, + "logps/chosen": -1.1125632524490356, + "logps/rejected": -12.267274856567383, + "loss": 2.5394, + "nll_loss": 2.5291550159454346, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11125633120536804, + "rewards/margins": 1.1154711246490479, + "rewards/rejected": -1.2267273664474487, + "step": 1321 + }, + { + "epoch": 0.8223950233281493, + "grad_norm": 0.5890390276908875, + "learning_rate": 1.6950000000000002e-05, + "log_odds_chosen": 5.568022727966309, + "log_odds_ratio": -0.10367351770401001, + "logits/chosen": 0.35675832629203796, + "logits/rejected": 0.8858349323272705, + "logps/chosen": -1.0774688720703125, + "logps/rejected": -6.139078617095947, + "loss": 2.6681, + "nll_loss": 2.6576852798461914, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10774689167737961, + "rewards/margins": 0.5061609745025635, + "rewards/rejected": -0.6139078736305237, + "step": 1322 + }, + { + "epoch": 0.8230171073094867, + "grad_norm": 0.9928566813468933, + "learning_rate": 1.6925e-05, + "log_odds_chosen": 12.515892028808594, + "log_odds_ratio": -0.06518105417490005, + "logits/chosen": 0.31150442361831665, + "logits/rejected": 1.2270563840866089, + "logps/chosen": -1.189500331878662, + "logps/rejected": -13.337007522583008, + "loss": 2.5602, + "nll_loss": 2.55371356010437, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1189500242471695, + "rewards/margins": 1.2147507667541504, + "rewards/rejected": -1.3337006568908691, + "step": 1323 + }, + { + "epoch": 0.8236391912908243, + "grad_norm": 0.7025359869003296, + "learning_rate": 1.69e-05, + "log_odds_chosen": 9.76104736328125, + "log_odds_ratio": -0.19300048053264618, + "logits/chosen": 0.28773853182792664, + "logits/rejected": 0.9788026809692383, + "logps/chosen": -1.1405911445617676, + "logps/rejected": -10.510811805725098, + "loss": 2.6634, + "nll_loss": 2.6441402435302734, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11405910551548004, + "rewards/margins": 0.9370220303535461, + "rewards/rejected": -1.0510811805725098, + "step": 1324 + }, + { + "epoch": 0.8242612752721618, + "grad_norm": 0.4265744686126709, + "learning_rate": 1.6875000000000004e-05, + "log_odds_chosen": 10.766847610473633, + "log_odds_ratio": -0.0524822473526001, + "logits/chosen": 0.2768998444080353, + "logits/rejected": 0.9822081327438354, + "logps/chosen": -1.0433452129364014, + "logps/rejected": -11.404045104980469, + "loss": 2.9502, + "nll_loss": 2.9449455738067627, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1043345257639885, + "rewards/margins": 1.0360699892044067, + "rewards/rejected": -1.140404462814331, + "step": 1325 + }, + { + "epoch": 0.8248833592534992, + "grad_norm": 0.587645947933197, + "learning_rate": 1.6850000000000003e-05, + "log_odds_chosen": 6.577492713928223, + "log_odds_ratio": -0.14230968058109283, + "logits/chosen": 0.030120672658085823, + "logits/rejected": 0.39918053150177, + "logps/chosen": -1.1130759716033936, + "logps/rejected": -7.328277587890625, + "loss": 2.5104, + "nll_loss": 2.4961977005004883, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11130759119987488, + "rewards/margins": 0.6215201616287231, + "rewards/rejected": -0.7328277826309204, + "step": 1326 + }, + { + "epoch": 0.8255054432348367, + "grad_norm": 0.5352939367294312, + "learning_rate": 1.6825000000000002e-05, + "log_odds_chosen": 12.196432113647461, + "log_odds_ratio": -0.1495496928691864, + "logits/chosen": 0.1835860013961792, + "logits/rejected": 1.0073965787887573, + "logps/chosen": -1.0454256534576416, + "logps/rejected": -12.764404296875, + "loss": 2.84, + "nll_loss": 2.8250114917755127, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1045425683259964, + "rewards/margins": 1.1718978881835938, + "rewards/rejected": -1.2764403820037842, + "step": 1327 + }, + { + "epoch": 0.8261275272161742, + "grad_norm": 0.4474393129348755, + "learning_rate": 1.6800000000000002e-05, + "log_odds_chosen": 17.64754295349121, + "log_odds_ratio": -0.001237626769579947, + "logits/chosen": 0.33965516090393066, + "logits/rejected": 1.2508803606033325, + "logps/chosen": -1.0787427425384521, + "logps/rejected": -18.28293228149414, + "loss": 3.2162, + "nll_loss": 3.2160513401031494, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10787426680326462, + "rewards/margins": 1.72041916847229, + "rewards/rejected": -1.8282933235168457, + "step": 1328 + }, + { + "epoch": 0.8267496111975117, + "grad_norm": 0.44476574659347534, + "learning_rate": 1.6775e-05, + "log_odds_chosen": 11.613157272338867, + "log_odds_ratio": -0.1723821610212326, + "logits/chosen": 0.345926433801651, + "logits/rejected": 1.0617916584014893, + "logps/chosen": -1.2812693119049072, + "logps/rejected": -12.575846672058105, + "loss": 3.3687, + "nll_loss": 3.3514435291290283, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12812693417072296, + "rewards/margins": 1.129457712173462, + "rewards/rejected": -1.257584571838379, + "step": 1329 + }, + { + "epoch": 0.8273716951788491, + "grad_norm": 0.6063930988311768, + "learning_rate": 1.675e-05, + "log_odds_chosen": 9.643231391906738, + "log_odds_ratio": -0.2011903077363968, + "logits/chosen": 0.338983416557312, + "logits/rejected": 1.080698013305664, + "logps/chosen": -1.4720268249511719, + "logps/rejected": -10.901077270507812, + "loss": 2.9527, + "nll_loss": 2.9325718879699707, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14720268547534943, + "rewards/margins": 0.9429050087928772, + "rewards/rejected": -1.0901076793670654, + "step": 1330 + }, + { + "epoch": 0.8279937791601866, + "grad_norm": 0.4499737620353699, + "learning_rate": 1.6725000000000003e-05, + "log_odds_chosen": 9.355201721191406, + "log_odds_ratio": -0.11042997241020203, + "logits/chosen": 0.1041167601943016, + "logits/rejected": 0.7633360624313354, + "logps/chosen": -1.1983859539031982, + "logps/rejected": -10.103107452392578, + "loss": 2.7849, + "nll_loss": 2.773890972137451, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11983861029148102, + "rewards/margins": 0.8904721140861511, + "rewards/rejected": -1.0103106498718262, + "step": 1331 + }, + { + "epoch": 0.8286158631415241, + "grad_norm": 0.46015751361846924, + "learning_rate": 1.6700000000000003e-05, + "log_odds_chosen": 10.06387996673584, + "log_odds_ratio": -0.05467294156551361, + "logits/chosen": 0.1695416122674942, + "logits/rejected": 0.6716436147689819, + "logps/chosen": -0.9862855076789856, + "logps/rejected": -10.507351875305176, + "loss": 2.9439, + "nll_loss": 2.938450336456299, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09862855821847916, + "rewards/margins": 0.9521065950393677, + "rewards/rejected": -1.0507352352142334, + "step": 1332 + }, + { + "epoch": 0.8292379471228616, + "grad_norm": 0.4361891746520996, + "learning_rate": 1.6675000000000002e-05, + "log_odds_chosen": 12.508257865905762, + "log_odds_ratio": -0.026293473318219185, + "logits/chosen": 0.2983800172805786, + "logits/rejected": 1.2380685806274414, + "logps/chosen": -1.0879460573196411, + "logps/rejected": -13.190286636352539, + "loss": 2.9347, + "nll_loss": 2.9320545196533203, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10879460722208023, + "rewards/margins": 1.2102341651916504, + "rewards/rejected": -1.3190287351608276, + "step": 1333 + }, + { + "epoch": 0.829860031104199, + "grad_norm": 0.4361533522605896, + "learning_rate": 1.665e-05, + "log_odds_chosen": 7.443196773529053, + "log_odds_ratio": -0.30054956674575806, + "logits/chosen": 0.3471451997756958, + "logits/rejected": 0.7988330125808716, + "logps/chosen": -1.1569123268127441, + "logps/rejected": -8.317983627319336, + "loss": 3.2357, + "nll_loss": 3.205623149871826, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11569122970104218, + "rewards/margins": 0.7161070704460144, + "rewards/rejected": -0.831798255443573, + "step": 1334 + }, + { + "epoch": 0.8304821150855366, + "grad_norm": 0.7759374976158142, + "learning_rate": 1.6625e-05, + "log_odds_chosen": 8.826431274414062, + "log_odds_ratio": -0.3530728220939636, + "logits/chosen": 0.21989214420318604, + "logits/rejected": 0.6294089555740356, + "logps/chosen": -1.4264800548553467, + "logps/rejected": -10.073540687561035, + "loss": 3.312, + "nll_loss": 3.2766780853271484, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14264801144599915, + "rewards/margins": 0.8647060394287109, + "rewards/rejected": -1.0073541402816772, + "step": 1335 + }, + { + "epoch": 0.831104199066874, + "grad_norm": 0.9619458317756653, + "learning_rate": 1.66e-05, + "log_odds_chosen": 7.348890781402588, + "log_odds_ratio": -0.2723667323589325, + "logits/chosen": 0.18200621008872986, + "logits/rejected": 0.7618054747581482, + "logps/chosen": -1.065969467163086, + "logps/rejected": -7.974705219268799, + "loss": 2.7613, + "nll_loss": 2.734013557434082, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1065969318151474, + "rewards/margins": 0.6908735632896423, + "rewards/rejected": -0.7974704504013062, + "step": 1336 + }, + { + "epoch": 0.8317262830482115, + "grad_norm": 0.5136169791221619, + "learning_rate": 1.6575000000000003e-05, + "log_odds_chosen": 7.726633071899414, + "log_odds_ratio": -0.2186044454574585, + "logits/chosen": 0.34979715943336487, + "logits/rejected": 0.7437832951545715, + "logps/chosen": -1.210748553276062, + "logps/rejected": -8.55751895904541, + "loss": 3.2908, + "nll_loss": 3.268960475921631, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1210748553276062, + "rewards/margins": 0.7346770167350769, + "rewards/rejected": -0.8557519316673279, + "step": 1337 + }, + { + "epoch": 0.832348367029549, + "grad_norm": 0.45952683687210083, + "learning_rate": 1.6550000000000002e-05, + "log_odds_chosen": 15.065681457519531, + "log_odds_ratio": -0.0024630618281662464, + "logits/chosen": 0.13940998911857605, + "logits/rejected": 0.7494243383407593, + "logps/chosen": -0.9125413298606873, + "logps/rejected": -15.253064155578613, + "loss": 2.8324, + "nll_loss": 2.8321423530578613, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09125413745641708, + "rewards/margins": 1.4340523481369019, + "rewards/rejected": -1.5253064632415771, + "step": 1338 + }, + { + "epoch": 0.8329704510108865, + "grad_norm": 0.6703150868415833, + "learning_rate": 1.6525000000000002e-05, + "log_odds_chosen": 10.943548202514648, + "log_odds_ratio": -0.11169109493494034, + "logits/chosen": 0.29511356353759766, + "logits/rejected": 0.77367103099823, + "logps/chosen": -1.0926419496536255, + "logps/rejected": -11.617069244384766, + "loss": 2.7409, + "nll_loss": 2.7297096252441406, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10926420241594315, + "rewards/margins": 1.0524427890777588, + "rewards/rejected": -1.1617070436477661, + "step": 1339 + }, + { + "epoch": 0.833592534992224, + "grad_norm": 0.5566127300262451, + "learning_rate": 1.65e-05, + "log_odds_chosen": 12.820943832397461, + "log_odds_ratio": -0.010184680111706257, + "logits/chosen": 0.16678117215633392, + "logits/rejected": 0.7523674964904785, + "logps/chosen": -0.9320427775382996, + "logps/rejected": -13.13653564453125, + "loss": 3.0966, + "nll_loss": 3.0955657958984375, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09320428222417831, + "rewards/margins": 1.2204493284225464, + "rewards/rejected": -1.313653588294983, + "step": 1340 + }, + { + "epoch": 0.8342146189735614, + "grad_norm": 1.249294400215149, + "learning_rate": 1.6475e-05, + "log_odds_chosen": 13.536139488220215, + "log_odds_ratio": -0.09148459136486053, + "logits/chosen": 0.17023460566997528, + "logits/rejected": 1.1541365385055542, + "logps/chosen": -1.294020652770996, + "logps/rejected": -14.549723625183105, + "loss": 2.466, + "nll_loss": 2.4568064212799072, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1294020712375641, + "rewards/margins": 1.3255703449249268, + "rewards/rejected": -1.454972267150879, + "step": 1341 + }, + { + "epoch": 0.8348367029548989, + "grad_norm": 0.34336620569229126, + "learning_rate": 1.645e-05, + "log_odds_chosen": 11.245887756347656, + "log_odds_ratio": -0.08879686892032623, + "logits/chosen": 0.4275479018688202, + "logits/rejected": 1.1122469902038574, + "logps/chosen": -1.0759177207946777, + "logps/rejected": -11.819448471069336, + "loss": 3.6854, + "nll_loss": 3.6764721870422363, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10759177803993225, + "rewards/margins": 1.0743530988693237, + "rewards/rejected": -1.1819448471069336, + "step": 1342 + }, + { + "epoch": 0.8354587869362364, + "grad_norm": 0.4405929446220398, + "learning_rate": 1.6425000000000003e-05, + "log_odds_chosen": 15.990028381347656, + "log_odds_ratio": -0.00034587905975058675, + "logits/chosen": 0.4133451581001282, + "logits/rejected": 1.1971858739852905, + "logps/chosen": -1.0017715692520142, + "logps/rejected": -16.374874114990234, + "loss": 3.6086, + "nll_loss": 3.6085314750671387, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10017715394496918, + "rewards/margins": 1.537310242652893, + "rewards/rejected": -1.6374874114990234, + "step": 1343 + }, + { + "epoch": 0.8360808709175739, + "grad_norm": 0.43593719601631165, + "learning_rate": 1.6400000000000002e-05, + "log_odds_chosen": 11.253133773803711, + "log_odds_ratio": -0.14229533076286316, + "logits/chosen": 0.22059793770313263, + "logits/rejected": 0.9180048108100891, + "logps/chosen": -0.9364933967590332, + "logps/rejected": -11.65854263305664, + "loss": 2.7832, + "nll_loss": 2.7689208984375, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09364934265613556, + "rewards/margins": 1.0722049474716187, + "rewards/rejected": -1.1658543348312378, + "step": 1344 + }, + { + "epoch": 0.8367029548989113, + "grad_norm": 6.538516044616699, + "learning_rate": 1.6375e-05, + "log_odds_chosen": 11.31826114654541, + "log_odds_ratio": -0.12284395098686218, + "logits/chosen": 0.3450809717178345, + "logits/rejected": 0.881148099899292, + "logps/chosen": -0.9467482566833496, + "logps/rejected": -11.267402648925781, + "loss": 3.2558, + "nll_loss": 3.243523597717285, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09467482566833496, + "rewards/margins": 1.0320653915405273, + "rewards/rejected": -1.1267402172088623, + "step": 1345 + }, + { + "epoch": 0.8373250388802488, + "grad_norm": 0.45118892192840576, + "learning_rate": 1.635e-05, + "log_odds_chosen": 8.51021957397461, + "log_odds_ratio": -0.24373793601989746, + "logits/chosen": 0.05161430686712265, + "logits/rejected": 0.45903950929641724, + "logps/chosen": -1.0309022665023804, + "logps/rejected": -9.030682563781738, + "loss": 2.5431, + "nll_loss": 2.518749713897705, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10309022665023804, + "rewards/margins": 0.7999781370162964, + "rewards/rejected": -0.9030683040618896, + "step": 1346 + }, + { + "epoch": 0.8379471228615863, + "grad_norm": 0.8182756900787354, + "learning_rate": 1.6325e-05, + "log_odds_chosen": 9.616655349731445, + "log_odds_ratio": -0.15306349098682404, + "logits/chosen": 0.20947861671447754, + "logits/rejected": 0.5957802534103394, + "logps/chosen": -1.1415512561798096, + "logps/rejected": -10.409021377563477, + "loss": 3.2896, + "nll_loss": 3.274308443069458, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11415514349937439, + "rewards/margins": 0.9267469644546509, + "rewards/rejected": -1.0409021377563477, + "step": 1347 + }, + { + "epoch": 0.8385692068429238, + "grad_norm": 0.5537295341491699, + "learning_rate": 1.63e-05, + "log_odds_chosen": 18.167945861816406, + "log_odds_ratio": -0.027182959020137787, + "logits/chosen": 0.30054154992103577, + "logits/rejected": 1.0188592672348022, + "logps/chosen": -0.8114546537399292, + "logps/rejected": -18.324691772460938, + "loss": 2.6675, + "nll_loss": 2.6647937297821045, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08114546537399292, + "rewards/margins": 1.7513236999511719, + "rewards/rejected": -1.83246910572052, + "step": 1348 + }, + { + "epoch": 0.8391912908242612, + "grad_norm": 0.7026792764663696, + "learning_rate": 1.6275000000000003e-05, + "log_odds_chosen": 14.551230430603027, + "log_odds_ratio": -0.09296683967113495, + "logits/chosen": 0.3387167453765869, + "logits/rejected": 0.8440542221069336, + "logps/chosen": -0.9055599570274353, + "logps/rejected": -14.753962516784668, + "loss": 3.3845, + "nll_loss": 3.3752224445343018, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09055599570274353, + "rewards/margins": 1.3848403692245483, + "rewards/rejected": -1.4753963947296143, + "step": 1349 + }, + { + "epoch": 0.8398133748055988, + "grad_norm": 0.4853743314743042, + "learning_rate": 1.6250000000000002e-05, + "log_odds_chosen": 15.019338607788086, + "log_odds_ratio": -0.0688743144273758, + "logits/chosen": 0.2787754237651825, + "logits/rejected": 1.2702405452728271, + "logps/chosen": -1.029766321182251, + "logps/rejected": -15.654651641845703, + "loss": 2.6953, + "nll_loss": 2.688436985015869, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10297662764787674, + "rewards/margins": 1.4624884128570557, + "rewards/rejected": -1.5654650926589966, + "step": 1350 + }, + { + "epoch": 0.8404354587869363, + "grad_norm": 1.4702285528182983, + "learning_rate": 1.6225e-05, + "log_odds_chosen": 7.845383167266846, + "log_odds_ratio": -0.0672604888677597, + "logits/chosen": 0.3774755001068115, + "logits/rejected": 0.8473259210586548, + "logps/chosen": -0.9018560647964478, + "logps/rejected": -8.014701843261719, + "loss": 3.4515, + "nll_loss": 3.4447741508483887, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09018560498952866, + "rewards/margins": 0.7112846374511719, + "rewards/rejected": -0.801470160484314, + "step": 1351 + }, + { + "epoch": 0.8410575427682737, + "grad_norm": 0.5535169839859009, + "learning_rate": 1.62e-05, + "log_odds_chosen": 14.512027740478516, + "log_odds_ratio": -0.09268511831760406, + "logits/chosen": 0.1820366382598877, + "logits/rejected": 0.9162260293960571, + "logps/chosen": -0.9260967969894409, + "logps/rejected": -14.837316513061523, + "loss": 3.0602, + "nll_loss": 3.050899028778076, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09260967373847961, + "rewards/margins": 1.3911221027374268, + "rewards/rejected": -1.4837316274642944, + "step": 1352 + }, + { + "epoch": 0.8416796267496112, + "grad_norm": 0.4414682388305664, + "learning_rate": 1.6175e-05, + "log_odds_chosen": 16.720481872558594, + "log_odds_ratio": -0.10064268112182617, + "logits/chosen": 0.27310311794281006, + "logits/rejected": 1.1602401733398438, + "logps/chosen": -1.3199316263198853, + "logps/rejected": -17.78493881225586, + "loss": 3.1636, + "nll_loss": 3.1535096168518066, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13199317455291748, + "rewards/margins": 1.646500825881958, + "rewards/rejected": -1.778494119644165, + "step": 1353 + }, + { + "epoch": 0.8423017107309487, + "grad_norm": 0.46305572986602783, + "learning_rate": 1.6150000000000003e-05, + "log_odds_chosen": 9.767788887023926, + "log_odds_ratio": -0.2507934272289276, + "logits/chosen": 0.15547390282154083, + "logits/rejected": 0.7884215116500854, + "logps/chosen": -1.4701690673828125, + "logps/rejected": -11.03095817565918, + "loss": 2.8695, + "nll_loss": 2.8444552421569824, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14701691269874573, + "rewards/margins": 0.9560788869857788, + "rewards/rejected": -1.1030957698822021, + "step": 1354 + }, + { + "epoch": 0.8429237947122862, + "grad_norm": 0.42249321937561035, + "learning_rate": 1.6125000000000002e-05, + "log_odds_chosen": 19.457439422607422, + "log_odds_ratio": -2.3841880647523794e-07, + "logits/chosen": 0.2563483417034149, + "logits/rejected": 1.0023006200790405, + "logps/chosen": -0.9900782108306885, + "logps/rejected": -19.732521057128906, + "loss": 3.4363, + "nll_loss": 3.4362709522247314, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09900782257318497, + "rewards/margins": 1.8742443323135376, + "rewards/rejected": -1.9732520580291748, + "step": 1355 + }, + { + "epoch": 0.8435458786936236, + "grad_norm": 0.5480783581733704, + "learning_rate": 1.6100000000000002e-05, + "log_odds_chosen": 12.938760757446289, + "log_odds_ratio": -0.20344708859920502, + "logits/chosen": 0.382476806640625, + "logits/rejected": 1.0104025602340698, + "logps/chosen": -0.9441956281661987, + "logps/rejected": -13.267868995666504, + "loss": 3.296, + "nll_loss": 3.2756340503692627, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09441956877708435, + "rewards/margins": 1.2323673963546753, + "rewards/rejected": -1.326786994934082, + "step": 1356 + }, + { + "epoch": 0.8441679626749611, + "grad_norm": 0.6357641220092773, + "learning_rate": 1.6075e-05, + "log_odds_chosen": 9.309688568115234, + "log_odds_ratio": -0.06144268810749054, + "logits/chosen": 0.19917547702789307, + "logits/rejected": 0.6526224613189697, + "logps/chosen": -0.8743935823440552, + "logps/rejected": -9.478578567504883, + "loss": 2.9935, + "nll_loss": 2.9873154163360596, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08743935823440552, + "rewards/margins": 0.8604185581207275, + "rewards/rejected": -0.9478578567504883, + "step": 1357 + }, + { + "epoch": 0.8447900466562986, + "grad_norm": 0.40837082266807556, + "learning_rate": 1.605e-05, + "log_odds_chosen": 12.638919830322266, + "log_odds_ratio": -0.1141565814614296, + "logits/chosen": 0.2843877375125885, + "logits/rejected": 1.202309250831604, + "logps/chosen": -1.1809484958648682, + "logps/rejected": -13.486981391906738, + "loss": 2.9087, + "nll_loss": 2.897247314453125, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11809486150741577, + "rewards/margins": 1.2306030988693237, + "rewards/rejected": -1.3486980199813843, + "step": 1358 + }, + { + "epoch": 0.8454121306376361, + "grad_norm": 0.619658350944519, + "learning_rate": 1.6025e-05, + "log_odds_chosen": 12.037456512451172, + "log_odds_ratio": -0.10075844824314117, + "logits/chosen": 0.3413699269294739, + "logits/rejected": 0.8576540946960449, + "logps/chosen": -0.8795664310455322, + "logps/rejected": -12.2918062210083, + "loss": 3.488, + "nll_loss": 3.477937698364258, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08795663714408875, + "rewards/margins": 1.1412239074707031, + "rewards/rejected": -1.2291805744171143, + "step": 1359 + }, + { + "epoch": 0.8460342146189735, + "grad_norm": 0.410817950963974, + "learning_rate": 1.6000000000000003e-05, + "log_odds_chosen": 10.435465812683105, + "log_odds_ratio": -0.2789941430091858, + "logits/chosen": 0.1286708265542984, + "logits/rejected": 0.44350332021713257, + "logps/chosen": -1.1692880392074585, + "logps/rejected": -11.24249267578125, + "loss": 2.9848, + "nll_loss": 2.9569311141967773, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11692880839109421, + "rewards/margins": 1.007320523262024, + "rewards/rejected": -1.1242493391036987, + "step": 1360 + }, + { + "epoch": 0.846656298600311, + "grad_norm": 0.5594186186790466, + "learning_rate": 1.5975000000000002e-05, + "log_odds_chosen": 11.055662155151367, + "log_odds_ratio": -0.04379449784755707, + "logits/chosen": 0.31934839487075806, + "logits/rejected": 1.1686406135559082, + "logps/chosen": -1.1824477910995483, + "logps/rejected": -11.840726852416992, + "loss": 2.8957, + "nll_loss": 2.8913650512695312, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11824478209018707, + "rewards/margins": 1.0658278465270996, + "rewards/rejected": -1.184072732925415, + "step": 1361 + }, + { + "epoch": 0.8472783825816486, + "grad_norm": 0.5316110253334045, + "learning_rate": 1.595e-05, + "log_odds_chosen": 5.959257125854492, + "log_odds_ratio": -0.056952349841594696, + "logits/chosen": 0.4386367201805115, + "logits/rejected": 0.5847421884536743, + "logps/chosen": -0.9345696568489075, + "logps/rejected": -6.30491304397583, + "loss": 3.2712, + "nll_loss": 3.2655532360076904, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09345697611570358, + "rewards/margins": 0.5370343923568726, + "rewards/rejected": -0.6304913759231567, + "step": 1362 + }, + { + "epoch": 0.847900466562986, + "grad_norm": 0.4794823229312897, + "learning_rate": 1.5925e-05, + "log_odds_chosen": 10.93669319152832, + "log_odds_ratio": -0.013783779926598072, + "logits/chosen": 0.20791217684745789, + "logits/rejected": 0.7548701763153076, + "logps/chosen": -1.2015867233276367, + "logps/rejected": -11.472372055053711, + "loss": 3.1463, + "nll_loss": 3.1448731422424316, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12015866488218307, + "rewards/margins": 1.0270785093307495, + "rewards/rejected": -1.1472373008728027, + "step": 1363 + }, + { + "epoch": 0.8485225505443235, + "grad_norm": 0.5277031660079956, + "learning_rate": 1.59e-05, + "log_odds_chosen": 8.793513298034668, + "log_odds_ratio": -0.3329012989997864, + "logits/chosen": 0.36735790967941284, + "logits/rejected": 0.7775804996490479, + "logps/chosen": -1.056816816329956, + "logps/rejected": -9.48435115814209, + "loss": 3.1065, + "nll_loss": 3.0732502937316895, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10568168014287949, + "rewards/margins": 0.8427534699440002, + "rewards/rejected": -0.9484350681304932, + "step": 1364 + }, + { + "epoch": 0.8491446345256609, + "grad_norm": 0.9272438287734985, + "learning_rate": 1.5875e-05, + "log_odds_chosen": 11.923362731933594, + "log_odds_ratio": -0.06839544326066971, + "logits/chosen": 0.14544972777366638, + "logits/rejected": 0.8777525424957275, + "logps/chosen": -1.2184803485870361, + "logps/rejected": -12.776731491088867, + "loss": 2.6382, + "nll_loss": 2.631315231323242, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12184803187847137, + "rewards/margins": 1.155825138092041, + "rewards/rejected": -1.2776731252670288, + "step": 1365 + }, + { + "epoch": 0.8497667185069985, + "grad_norm": 0.44855692982673645, + "learning_rate": 1.5850000000000002e-05, + "log_odds_chosen": 8.991267204284668, + "log_odds_ratio": -0.09500695019960403, + "logits/chosen": 0.3809604346752167, + "logits/rejected": 1.16422438621521, + "logps/chosen": -1.182137370109558, + "logps/rejected": -9.790534019470215, + "loss": 3.465, + "nll_loss": 3.455484390258789, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11821374297142029, + "rewards/margins": 0.8608396649360657, + "rewards/rejected": -0.9790534377098083, + "step": 1366 + }, + { + "epoch": 0.8503888024883359, + "grad_norm": 0.5001557469367981, + "learning_rate": 1.5825000000000002e-05, + "log_odds_chosen": 11.170446395874023, + "log_odds_ratio": -0.019392523914575577, + "logits/chosen": 0.18927381932735443, + "logits/rejected": 1.0548985004425049, + "logps/chosen": -1.0558704137802124, + "logps/rejected": -11.756693840026855, + "loss": 2.6287, + "nll_loss": 2.6267898082733154, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10558703541755676, + "rewards/margins": 1.0700823068618774, + "rewards/rejected": -1.1756694316864014, + "step": 1367 + }, + { + "epoch": 0.8510108864696734, + "grad_norm": 0.553524374961853, + "learning_rate": 1.58e-05, + "log_odds_chosen": 10.177559852600098, + "log_odds_ratio": -0.12094742059707642, + "logits/chosen": 0.17707903683185577, + "logits/rejected": 0.9806874394416809, + "logps/chosen": -1.1824595928192139, + "logps/rejected": -10.97883415222168, + "loss": 2.6865, + "nll_loss": 2.67435622215271, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11824595183134079, + "rewards/margins": 0.9796375036239624, + "rewards/rejected": -1.0978834629058838, + "step": 1368 + }, + { + "epoch": 0.8516329704510109, + "grad_norm": 0.6485047340393066, + "learning_rate": 1.5775e-05, + "log_odds_chosen": 10.440032958984375, + "log_odds_ratio": -0.2638293206691742, + "logits/chosen": 0.2368917465209961, + "logits/rejected": 1.0509949922561646, + "logps/chosen": -1.2486120462417603, + "logps/rejected": -11.450448989868164, + "loss": 2.642, + "nll_loss": 2.615628242492676, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1248612031340599, + "rewards/margins": 1.020183801651001, + "rewards/rejected": -1.1450450420379639, + "step": 1369 + }, + { + "epoch": 0.8522550544323484, + "grad_norm": 0.472089558839798, + "learning_rate": 1.575e-05, + "log_odds_chosen": 2.847292423248291, + "log_odds_ratio": -0.5115450620651245, + "logits/chosen": 0.1676526665687561, + "logits/rejected": 0.23850813508033752, + "logps/chosen": -1.163887619972229, + "logps/rejected": -3.8988606929779053, + "loss": 2.9222, + "nll_loss": 2.871020793914795, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11638876795768738, + "rewards/margins": 0.2734973132610321, + "rewards/rejected": -0.3898860514163971, + "step": 1370 + }, + { + "epoch": 0.8528771384136858, + "grad_norm": 1.902549386024475, + "learning_rate": 1.5725e-05, + "log_odds_chosen": 9.938411712646484, + "log_odds_ratio": -0.16199146211147308, + "logits/chosen": 0.32466983795166016, + "logits/rejected": 1.023469090461731, + "logps/chosen": -0.9341733455657959, + "logps/rejected": -10.43630599975586, + "loss": 3.0852, + "nll_loss": 3.0690159797668457, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09341733902692795, + "rewards/margins": 0.9502133131027222, + "rewards/rejected": -1.043630599975586, + "step": 1371 + }, + { + "epoch": 0.8534992223950233, + "grad_norm": 0.5687903761863708, + "learning_rate": 1.5700000000000002e-05, + "log_odds_chosen": 9.373761177062988, + "log_odds_ratio": -0.13499051332473755, + "logits/chosen": 0.2836533188819885, + "logits/rejected": 0.8441271781921387, + "logps/chosen": -0.9228062629699707, + "logps/rejected": -9.52553939819336, + "loss": 3.0099, + "nll_loss": 2.9963736534118652, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09228062629699707, + "rewards/margins": 0.8602732419967651, + "rewards/rejected": -0.9525538682937622, + "step": 1372 + }, + { + "epoch": 0.8541213063763609, + "grad_norm": 0.8326713442802429, + "learning_rate": 1.5675e-05, + "log_odds_chosen": 10.913453102111816, + "log_odds_ratio": -0.16819366812705994, + "logits/chosen": 0.3695370554924011, + "logits/rejected": 0.9171431064605713, + "logps/chosen": -1.0279958248138428, + "logps/rejected": -11.530767440795898, + "loss": 2.7988, + "nll_loss": 2.7819666862487793, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10279958695173264, + "rewards/margins": 1.0502771139144897, + "rewards/rejected": -1.1530767679214478, + "step": 1373 + }, + { + "epoch": 0.8547433903576983, + "grad_norm": 0.47954127192497253, + "learning_rate": 1.565e-05, + "log_odds_chosen": 11.381818771362305, + "log_odds_ratio": -0.08887384831905365, + "logits/chosen": 0.22594711184501648, + "logits/rejected": 1.1135444641113281, + "logps/chosen": -1.155029535293579, + "logps/rejected": -12.046524047851562, + "loss": 2.631, + "nll_loss": 2.622159004211426, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11550294607877731, + "rewards/margins": 1.0891494750976562, + "rewards/rejected": -1.2046523094177246, + "step": 1374 + }, + { + "epoch": 0.8553654743390358, + "grad_norm": 1.0418078899383545, + "learning_rate": 1.5625e-05, + "log_odds_chosen": 5.646578311920166, + "log_odds_ratio": -0.22757403552532196, + "logits/chosen": 0.3599763512611389, + "logits/rejected": 0.44046440720558167, + "logps/chosen": -0.9107646346092224, + "logps/rejected": -6.134728908538818, + "loss": 2.9993, + "nll_loss": 2.9765775203704834, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09107646346092224, + "rewards/margins": 0.522396445274353, + "rewards/rejected": -0.6134728789329529, + "step": 1375 + }, + { + "epoch": 0.8559875583203732, + "grad_norm": 1.584696650505066, + "learning_rate": 1.56e-05, + "log_odds_chosen": 15.953493118286133, + "log_odds_ratio": -0.13301169872283936, + "logits/chosen": 0.3697129189968109, + "logits/rejected": 1.2205836772918701, + "logps/chosen": -1.839653730392456, + "logps/rejected": -17.536806106567383, + "loss": 3.5137, + "nll_loss": 3.5003836154937744, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18396538496017456, + "rewards/margins": 1.5697153806686401, + "rewards/rejected": -1.75368070602417, + "step": 1376 + }, + { + "epoch": 0.8566096423017108, + "grad_norm": 0.394852876663208, + "learning_rate": 1.5575e-05, + "log_odds_chosen": 10.27212142944336, + "log_odds_ratio": -0.19856134057044983, + "logits/chosen": 0.3533197045326233, + "logits/rejected": 0.7798941135406494, + "logps/chosen": -0.886967658996582, + "logps/rejected": -10.664911270141602, + "loss": 3.3432, + "nll_loss": 3.3233642578125, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08869676291942596, + "rewards/margins": 0.9777944087982178, + "rewards/rejected": -1.0664911270141602, + "step": 1377 + }, + { + "epoch": 0.8572317262830482, + "grad_norm": 0.5107656121253967, + "learning_rate": 1.5550000000000002e-05, + "log_odds_chosen": 10.686027526855469, + "log_odds_ratio": -0.18517914414405823, + "logits/chosen": 0.13516952097415924, + "logits/rejected": 0.5872347950935364, + "logps/chosen": -0.9439507722854614, + "logps/rejected": -11.0865478515625, + "loss": 2.3468, + "nll_loss": 2.328307628631592, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09439507126808167, + "rewards/margins": 1.014259696006775, + "rewards/rejected": -1.1086547374725342, + "step": 1378 + }, + { + "epoch": 0.8578538102643857, + "grad_norm": 0.7838579416275024, + "learning_rate": 1.5525e-05, + "log_odds_chosen": 8.990863800048828, + "log_odds_ratio": -0.24506224691867828, + "logits/chosen": 0.3935107886791229, + "logits/rejected": 1.1674621105194092, + "logps/chosen": -1.3738632202148438, + "logps/rejected": -10.04948902130127, + "loss": 3.5415, + "nll_loss": 3.516964912414551, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13738630712032318, + "rewards/margins": 0.8675625920295715, + "rewards/rejected": -1.0049489736557007, + "step": 1379 + }, + { + "epoch": 0.8584758942457231, + "grad_norm": 0.5477966666221619, + "learning_rate": 1.55e-05, + "log_odds_chosen": 11.66675090789795, + "log_odds_ratio": -0.23476821184158325, + "logits/chosen": 0.27057936787605286, + "logits/rejected": 0.9670227766036987, + "logps/chosen": -1.2665765285491943, + "logps/rejected": -12.575981140136719, + "loss": 2.9679, + "nll_loss": 2.9444386959075928, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1266576498746872, + "rewards/margins": 1.1309404373168945, + "rewards/rejected": -1.2575981616973877, + "step": 1380 + }, + { + "epoch": 0.8590979782270607, + "grad_norm": 0.6186298727989197, + "learning_rate": 1.5475e-05, + "log_odds_chosen": 6.804127216339111, + "log_odds_ratio": -0.29192444682121277, + "logits/chosen": 0.4237322211265564, + "logits/rejected": 0.6319191455841064, + "logps/chosen": -1.1395926475524902, + "logps/rejected": -7.732236385345459, + "loss": 3.1793, + "nll_loss": 3.150071859359741, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11395925283432007, + "rewards/margins": 0.659264326095581, + "rewards/rejected": -0.7732236385345459, + "step": 1381 + }, + { + "epoch": 0.8597200622083981, + "grad_norm": 0.3818768262863159, + "learning_rate": 1.545e-05, + "log_odds_chosen": 15.892073631286621, + "log_odds_ratio": -0.08740878105163574, + "logits/chosen": 0.367888867855072, + "logits/rejected": 1.3535079956054688, + "logps/chosen": -1.0746917724609375, + "logps/rejected": -16.52185821533203, + "loss": 3.2333, + "nll_loss": 3.2245535850524902, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10746918618679047, + "rewards/margins": 1.544716715812683, + "rewards/rejected": -1.6521859169006348, + "step": 1382 + }, + { + "epoch": 0.8603421461897356, + "grad_norm": 0.5417101383209229, + "learning_rate": 1.5425000000000002e-05, + "log_odds_chosen": 15.340182304382324, + "log_odds_ratio": -0.05749611184000969, + "logits/chosen": 0.338359534740448, + "logits/rejected": 1.1421986818313599, + "logps/chosen": -0.9286530017852783, + "logps/rejected": -15.792999267578125, + "loss": 2.9544, + "nll_loss": 2.948611259460449, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09286530315876007, + "rewards/margins": 1.4864346981048584, + "rewards/rejected": -1.579300045967102, + "step": 1383 + }, + { + "epoch": 0.860964230171073, + "grad_norm": 0.7678720355033875, + "learning_rate": 1.54e-05, + "log_odds_chosen": 16.128339767456055, + "log_odds_ratio": -0.10341505706310272, + "logits/chosen": 0.2992256283760071, + "logits/rejected": 1.0462899208068848, + "logps/chosen": -1.1855686902999878, + "logps/rejected": -16.78167152404785, + "loss": 3.2203, + "nll_loss": 3.209942102432251, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11855687201023102, + "rewards/margins": 1.5596102476119995, + "rewards/rejected": -1.6781671047210693, + "step": 1384 + }, + { + "epoch": 0.8615863141524106, + "grad_norm": 0.5416978597640991, + "learning_rate": 1.5375e-05, + "log_odds_chosen": 9.19207763671875, + "log_odds_ratio": -0.09668044000864029, + "logits/chosen": 0.30826884508132935, + "logits/rejected": 1.0236417055130005, + "logps/chosen": -1.0911591053009033, + "logps/rejected": -9.906354904174805, + "loss": 2.6784, + "nll_loss": 2.6686997413635254, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10911591351032257, + "rewards/margins": 0.8815194964408875, + "rewards/rejected": -0.9906355142593384, + "step": 1385 + }, + { + "epoch": 0.862208398133748, + "grad_norm": 0.41511037945747375, + "learning_rate": 1.535e-05, + "log_odds_chosen": 13.163442611694336, + "log_odds_ratio": -0.0031957889441400766, + "logits/chosen": 0.3195352554321289, + "logits/rejected": 1.319638967514038, + "logps/chosen": -1.1745014190673828, + "logps/rejected": -13.899215698242188, + "loss": 3.3087, + "nll_loss": 3.3083767890930176, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11745014786720276, + "rewards/margins": 1.272471308708191, + "rewards/rejected": -1.3899214267730713, + "step": 1386 + }, + { + "epoch": 0.8628304821150855, + "grad_norm": 0.52939373254776, + "learning_rate": 1.5325e-05, + "log_odds_chosen": 15.242445945739746, + "log_odds_ratio": -0.08322323113679886, + "logits/chosen": 0.16968780755996704, + "logits/rejected": 1.290441870689392, + "logps/chosen": -1.0485494136810303, + "logps/rejected": -15.906908988952637, + "loss": 2.486, + "nll_loss": 2.4777145385742188, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10485494136810303, + "rewards/margins": 1.4858359098434448, + "rewards/rejected": -1.5906908512115479, + "step": 1387 + }, + { + "epoch": 0.8634525660964231, + "grad_norm": 0.5325446724891663, + "learning_rate": 1.53e-05, + "log_odds_chosen": 15.141090393066406, + "log_odds_ratio": -0.21355324983596802, + "logits/chosen": 0.35596466064453125, + "logits/rejected": 1.269651174545288, + "logps/chosen": -1.2448396682739258, + "logps/rejected": -16.096210479736328, + "loss": 3.0043, + "nll_loss": 2.982938766479492, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12448396533727646, + "rewards/margins": 1.4851369857788086, + "rewards/rejected": -1.6096210479736328, + "step": 1388 + }, + { + "epoch": 0.8640746500777605, + "grad_norm": 0.6094441413879395, + "learning_rate": 1.5275000000000002e-05, + "log_odds_chosen": 14.076630592346191, + "log_odds_ratio": -0.19824470579624176, + "logits/chosen": 0.4293285608291626, + "logits/rejected": 1.2163474559783936, + "logps/chosen": -1.2386701107025146, + "logps/rejected": -15.064802169799805, + "loss": 3.1363, + "nll_loss": 3.1165122985839844, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12386702001094818, + "rewards/margins": 1.382613182067871, + "rewards/rejected": -1.5064802169799805, + "step": 1389 + }, + { + "epoch": 0.864696734059098, + "grad_norm": 0.4554082155227661, + "learning_rate": 1.525e-05, + "log_odds_chosen": 14.69717025756836, + "log_odds_ratio": -0.10522916167974472, + "logits/chosen": 0.3967858552932739, + "logits/rejected": 1.06642746925354, + "logps/chosen": -0.7577130794525146, + "logps/rejected": -14.894484519958496, + "loss": 3.5846, + "nll_loss": 3.5740511417388916, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.07577130943536758, + "rewards/margins": 1.4136772155761719, + "rewards/rejected": -1.4894484281539917, + "step": 1390 + }, + { + "epoch": 0.8653188180404354, + "grad_norm": 0.4598383605480194, + "learning_rate": 1.5225e-05, + "log_odds_chosen": 17.62496566772461, + "log_odds_ratio": -2.831222616350715e-07, + "logits/chosen": 0.29642823338508606, + "logits/rejected": 1.3097069263458252, + "logps/chosen": -1.1783454418182373, + "logps/rejected": -18.41707992553711, + "loss": 3.0947, + "nll_loss": 3.0946526527404785, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11783455312252045, + "rewards/margins": 1.7238733768463135, + "rewards/rejected": -1.8417079448699951, + "step": 1391 + }, + { + "epoch": 0.865940902021773, + "grad_norm": 0.45408785343170166, + "learning_rate": 1.52e-05, + "log_odds_chosen": 11.176227569580078, + "log_odds_ratio": -0.11154050379991531, + "logits/chosen": 0.11151541024446487, + "logits/rejected": 0.945254921913147, + "logps/chosen": -1.101738452911377, + "logps/rejected": -11.896114349365234, + "loss": 2.6586, + "nll_loss": 2.647481918334961, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11017385125160217, + "rewards/margins": 1.0794377326965332, + "rewards/rejected": -1.1896114349365234, + "step": 1392 + }, + { + "epoch": 0.8665629860031104, + "grad_norm": 0.38992127776145935, + "learning_rate": 1.5175e-05, + "log_odds_chosen": 16.277109146118164, + "log_odds_ratio": -0.00011801975051639602, + "logits/chosen": 0.32067984342575073, + "logits/rejected": 1.577821969985962, + "logps/chosen": -1.2991023063659668, + "logps/rejected": -17.236865997314453, + "loss": 3.2608, + "nll_loss": 3.260770797729492, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12991024553775787, + "rewards/margins": 1.5937763452529907, + "rewards/rejected": -1.723686695098877, + "step": 1393 + }, + { + "epoch": 0.8671850699844479, + "grad_norm": 0.6620752811431885, + "learning_rate": 1.515e-05, + "log_odds_chosen": 8.851555824279785, + "log_odds_ratio": -0.09632323682308197, + "logits/chosen": 0.25070205330848694, + "logits/rejected": 0.5107638835906982, + "logps/chosen": -0.8554275035858154, + "logps/rejected": -9.137062072753906, + "loss": 3.2416, + "nll_loss": 3.2320029735565186, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08554275333881378, + "rewards/margins": 0.8281635046005249, + "rewards/rejected": -0.9137061834335327, + "step": 1394 + }, + { + "epoch": 0.8678071539657853, + "grad_norm": 0.8303456902503967, + "learning_rate": 1.5125e-05, + "log_odds_chosen": 16.057336807250977, + "log_odds_ratio": -0.00020326176309026778, + "logits/chosen": 0.24806132912635803, + "logits/rejected": 1.1189615726470947, + "logps/chosen": -1.0641580820083618, + "logps/rejected": -16.5771484375, + "loss": 3.136, + "nll_loss": 3.1359386444091797, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10641580075025558, + "rewards/margins": 1.551298975944519, + "rewards/rejected": -1.6577147245407104, + "step": 1395 + }, + { + "epoch": 0.8684292379471229, + "grad_norm": 0.6153732538223267, + "learning_rate": 1.51e-05, + "log_odds_chosen": 8.939095497131348, + "log_odds_ratio": -0.13549737632274628, + "logits/chosen": -0.030933350324630737, + "logits/rejected": 0.6015803813934326, + "logps/chosen": -1.3215835094451904, + "logps/rejected": -9.989289283752441, + "loss": 2.6042, + "nll_loss": 2.5906641483306885, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1321583390235901, + "rewards/margins": 0.8667706251144409, + "rewards/rejected": -0.9989289045333862, + "step": 1396 + }, + { + "epoch": 0.8690513219284604, + "grad_norm": 0.546840250492096, + "learning_rate": 1.5075e-05, + "log_odds_chosen": 13.476724624633789, + "log_odds_ratio": -0.120454341173172, + "logits/chosen": 0.33575960993766785, + "logits/rejected": 1.0339983701705933, + "logps/chosen": -1.5418827533721924, + "logps/rejected": -14.785687446594238, + "loss": 3.1669, + "nll_loss": 3.1548702716827393, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15418827533721924, + "rewards/margins": 1.3243803977966309, + "rewards/rejected": -1.4785687923431396, + "step": 1397 + }, + { + "epoch": 0.8696734059097978, + "grad_norm": 0.4228728413581848, + "learning_rate": 1.505e-05, + "log_odds_chosen": 13.623929977416992, + "log_odds_ratio": -0.10882086306810379, + "logits/chosen": 0.24190104007720947, + "logits/rejected": 0.7488356828689575, + "logps/chosen": -1.416288137435913, + "logps/rejected": -14.760062217712402, + "loss": 3.3408, + "nll_loss": 3.32993221282959, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14162881672382355, + "rewards/margins": 1.3343775272369385, + "rewards/rejected": -1.476006269454956, + "step": 1398 + }, + { + "epoch": 0.8702954898911353, + "grad_norm": 2.191300630569458, + "learning_rate": 1.5025000000000001e-05, + "log_odds_chosen": 8.226001739501953, + "log_odds_ratio": -0.5000618696212769, + "logits/chosen": 0.060526616871356964, + "logits/rejected": 0.4570578932762146, + "logps/chosen": -1.5750854015350342, + "logps/rejected": -9.476434707641602, + "loss": 2.8869, + "nll_loss": 2.836855888366699, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15750853717327118, + "rewards/margins": 0.7901349067687988, + "rewards/rejected": -0.9476434588432312, + "step": 1399 + }, + { + "epoch": 0.8709175738724728, + "grad_norm": 0.4874240756034851, + "learning_rate": 1.5e-05, + "log_odds_chosen": 10.55546760559082, + "log_odds_ratio": -0.17145481705665588, + "logits/chosen": 0.24024158716201782, + "logits/rejected": 1.0146418809890747, + "logps/chosen": -1.0970916748046875, + "logps/rejected": -11.358686447143555, + "loss": 2.9935, + "nll_loss": 2.9763572216033936, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10970918089151382, + "rewards/margins": 1.026159405708313, + "rewards/rejected": -1.1358685493469238, + "step": 1400 + }, + { + "epoch": 0.8715396578538103, + "grad_norm": 0.6221321225166321, + "learning_rate": 1.4975e-05, + "log_odds_chosen": 10.754582405090332, + "log_odds_ratio": -0.47946444153785706, + "logits/chosen": 0.2002388834953308, + "logits/rejected": 0.7997665405273438, + "logps/chosen": -1.3535938262939453, + "logps/rejected": -11.846307754516602, + "loss": 3.3366, + "nll_loss": 3.288651943206787, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13535939157009125, + "rewards/margins": 1.0492714643478394, + "rewards/rejected": -1.1846308708190918, + "step": 1401 + }, + { + "epoch": 0.8721617418351477, + "grad_norm": 0.5414087176322937, + "learning_rate": 1.4950000000000001e-05, + "log_odds_chosen": 12.587039947509766, + "log_odds_ratio": -0.2860456109046936, + "logits/chosen": 0.10869123786687851, + "logits/rejected": 0.6310420036315918, + "logps/chosen": -1.1774928569793701, + "logps/rejected": -13.448193550109863, + "loss": 2.6884, + "nll_loss": 2.659811496734619, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11774928122758865, + "rewards/margins": 1.2270700931549072, + "rewards/rejected": -1.3448193073272705, + "step": 1402 + }, + { + "epoch": 0.8727838258164852, + "grad_norm": 0.6045001149177551, + "learning_rate": 1.4925e-05, + "log_odds_chosen": 9.989900588989258, + "log_odds_ratio": -0.20460441708564758, + "logits/chosen": 0.31068581342697144, + "logits/rejected": 0.7650317549705505, + "logps/chosen": -1.0151677131652832, + "logps/rejected": -10.629185676574707, + "loss": 3.2557, + "nll_loss": 3.2352700233459473, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10151677578687668, + "rewards/margins": 0.9614019393920898, + "rewards/rejected": -1.0629186630249023, + "step": 1403 + }, + { + "epoch": 0.8734059097978227, + "grad_norm": 0.5089529752731323, + "learning_rate": 1.49e-05, + "log_odds_chosen": 8.479745864868164, + "log_odds_ratio": -0.24054519832134247, + "logits/chosen": 0.08574174344539642, + "logits/rejected": 0.42926856875419617, + "logps/chosen": -0.8743036985397339, + "logps/rejected": -8.764272689819336, + "loss": 2.8783, + "nll_loss": 2.8542098999023438, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08743037283420563, + "rewards/margins": 0.7889969348907471, + "rewards/rejected": -0.8764272928237915, + "step": 1404 + }, + { + "epoch": 0.8740279937791602, + "grad_norm": 0.4408000707626343, + "learning_rate": 1.4875e-05, + "log_odds_chosen": 15.424028396606445, + "log_odds_ratio": -0.1469992995262146, + "logits/chosen": 0.41973283886909485, + "logits/rejected": 1.132185935974121, + "logps/chosen": -1.2691795825958252, + "logps/rejected": -16.375694274902344, + "loss": 3.9522, + "nll_loss": 3.937525749206543, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12691795825958252, + "rewards/margins": 1.5106515884399414, + "rewards/rejected": -1.637569546699524, + "step": 1405 + }, + { + "epoch": 0.8746500777604976, + "grad_norm": 0.5824654698371887, + "learning_rate": 1.485e-05, + "log_odds_chosen": 10.513412475585938, + "log_odds_ratio": -0.2729822099208832, + "logits/chosen": 0.22236670553684235, + "logits/rejected": 1.3081042766571045, + "logps/chosen": -1.083417296409607, + "logps/rejected": -11.281618118286133, + "loss": 2.7582, + "nll_loss": 2.730898141860962, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10834173113107681, + "rewards/margins": 1.0198200941085815, + "rewards/rejected": -1.1281617879867554, + "step": 1406 + }, + { + "epoch": 0.8752721617418352, + "grad_norm": 15.712387084960938, + "learning_rate": 1.4825e-05, + "log_odds_chosen": 12.111908912658691, + "log_odds_ratio": -0.07724400609731674, + "logits/chosen": 0.30688637495040894, + "logits/rejected": 1.0119655132293701, + "logps/chosen": -1.4687293767929077, + "logps/rejected": -13.193370819091797, + "loss": 3.3589, + "nll_loss": 3.3511881828308105, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14687293767929077, + "rewards/margins": 1.1724642515182495, + "rewards/rejected": -1.3193371295928955, + "step": 1407 + }, + { + "epoch": 0.8758942457231726, + "grad_norm": 0.5521935820579529, + "learning_rate": 1.48e-05, + "log_odds_chosen": 17.45698356628418, + "log_odds_ratio": -0.01309966016560793, + "logits/chosen": 0.3117651343345642, + "logits/rejected": 0.9942529201507568, + "logps/chosen": -0.940118670463562, + "logps/rejected": -17.889507293701172, + "loss": 2.691, + "nll_loss": 2.689737319946289, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09401187300682068, + "rewards/margins": 1.6949388980865479, + "rewards/rejected": -1.7889506816864014, + "step": 1408 + }, + { + "epoch": 0.8765163297045101, + "grad_norm": 0.42602846026420593, + "learning_rate": 1.4775e-05, + "log_odds_chosen": 16.168874740600586, + "log_odds_ratio": -0.10667488723993301, + "logits/chosen": 0.25878027081489563, + "logits/rejected": 1.3639453649520874, + "logps/chosen": -1.0515450239181519, + "logps/rejected": -16.774581909179688, + "loss": 2.7998, + "nll_loss": 2.7891736030578613, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10515449941158295, + "rewards/margins": 1.5723037719726562, + "rewards/rejected": -1.67745840549469, + "step": 1409 + }, + { + "epoch": 0.8771384136858476, + "grad_norm": 0.4654007852077484, + "learning_rate": 1.475e-05, + "log_odds_chosen": 13.330875396728516, + "log_odds_ratio": -0.21977800130844116, + "logits/chosen": 0.4110845923423767, + "logits/rejected": 0.8825340867042542, + "logps/chosen": -0.8441970348358154, + "logps/rejected": -13.64559555053711, + "loss": 3.4113, + "nll_loss": 3.3893280029296875, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08441969752311707, + "rewards/margins": 1.2801399230957031, + "rewards/rejected": -1.3645596504211426, + "step": 1410 + }, + { + "epoch": 0.8777604976671851, + "grad_norm": 0.8660902380943298, + "learning_rate": 1.4725e-05, + "log_odds_chosen": 14.066307067871094, + "log_odds_ratio": -0.031187426298856735, + "logits/chosen": 0.20439361035823822, + "logits/rejected": 1.059504508972168, + "logps/chosen": -1.189511775970459, + "logps/rejected": -14.74959659576416, + "loss": 2.3309, + "nll_loss": 2.327756404876709, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11895118653774261, + "rewards/margins": 1.356008529663086, + "rewards/rejected": -1.4749597311019897, + "step": 1411 + }, + { + "epoch": 0.8783825816485226, + "grad_norm": 0.4791586995124817, + "learning_rate": 1.47e-05, + "log_odds_chosen": 9.729087829589844, + "log_odds_ratio": -0.11456810683012009, + "logits/chosen": 0.2344791293144226, + "logits/rejected": 0.9708684682846069, + "logps/chosen": -1.1141091585159302, + "logps/rejected": -10.431741714477539, + "loss": 2.9354, + "nll_loss": 2.9239559173583984, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11141092330217361, + "rewards/margins": 0.9317631721496582, + "rewards/rejected": -1.043174147605896, + "step": 1412 + }, + { + "epoch": 0.87900466562986, + "grad_norm": 0.4437407851219177, + "learning_rate": 1.4675e-05, + "log_odds_chosen": 17.249923706054688, + "log_odds_ratio": -0.10913048684597015, + "logits/chosen": 0.25980401039123535, + "logits/rejected": 1.4403678178787231, + "logps/chosen": -1.3583660125732422, + "logps/rejected": -18.27130126953125, + "loss": 2.9153, + "nll_loss": 2.9043660163879395, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1358366161584854, + "rewards/margins": 1.691293716430664, + "rewards/rejected": -1.8271300792694092, + "step": 1413 + }, + { + "epoch": 0.8796267496111975, + "grad_norm": 0.47129300236701965, + "learning_rate": 1.465e-05, + "log_odds_chosen": 15.95350170135498, + "log_odds_ratio": -0.07945066690444946, + "logits/chosen": 0.4288966655731201, + "logits/rejected": 1.349419116973877, + "logps/chosen": -1.078054666519165, + "logps/rejected": -16.625701904296875, + "loss": 3.1376, + "nll_loss": 3.129685878753662, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10780548304319382, + "rewards/margins": 1.554764747619629, + "rewards/rejected": -1.6625701189041138, + "step": 1414 + }, + { + "epoch": 0.880248833592535, + "grad_norm": 0.5274661183357239, + "learning_rate": 1.4625e-05, + "log_odds_chosen": 5.400740146636963, + "log_odds_ratio": -0.28576767444610596, + "logits/chosen": 0.2634659707546234, + "logits/rejected": 0.4704267382621765, + "logps/chosen": -1.1760218143463135, + "logps/rejected": -6.290962219238281, + "loss": 3.1959, + "nll_loss": 3.1673409938812256, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1176021620631218, + "rewards/margins": 0.5114940404891968, + "rewards/rejected": -0.6290962100028992, + "step": 1415 + }, + { + "epoch": 0.8808709175738725, + "grad_norm": 0.6360839009284973, + "learning_rate": 1.4599999999999999e-05, + "log_odds_chosen": 18.659954071044922, + "log_odds_ratio": -0.0560433454811573, + "logits/chosen": 0.1060413271188736, + "logits/rejected": 1.236191749572754, + "logps/chosen": -1.053462266921997, + "logps/rejected": -19.213502883911133, + "loss": 2.3897, + "nll_loss": 2.3841118812561035, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10534623265266418, + "rewards/margins": 1.8160041570663452, + "rewards/rejected": -1.921350359916687, + "step": 1416 + }, + { + "epoch": 0.8814930015552099, + "grad_norm": 0.5016162395477295, + "learning_rate": 1.4575e-05, + "log_odds_chosen": 14.632455825805664, + "log_odds_ratio": -0.18493463099002838, + "logits/chosen": 0.29079532623291016, + "logits/rejected": 1.2937943935394287, + "logps/chosen": -1.1794633865356445, + "logps/rejected": -15.487001419067383, + "loss": 2.7321, + "nll_loss": 2.7136242389678955, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11794634163379669, + "rewards/margins": 1.4307537078857422, + "rewards/rejected": -1.548699975013733, + "step": 1417 + }, + { + "epoch": 0.8821150855365474, + "grad_norm": 0.47031769156455994, + "learning_rate": 1.455e-05, + "log_odds_chosen": 15.577923774719238, + "log_odds_ratio": -0.004490715451538563, + "logits/chosen": 0.2111625373363495, + "logits/rejected": 1.3336447477340698, + "logps/chosen": -1.0458276271820068, + "logps/rejected": -16.042156219482422, + "loss": 2.5581, + "nll_loss": 2.557619333267212, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1045827716588974, + "rewards/margins": 1.4996328353881836, + "rewards/rejected": -1.6042156219482422, + "step": 1418 + }, + { + "epoch": 0.882737169517885, + "grad_norm": 0.6436307430267334, + "learning_rate": 1.4524999999999999e-05, + "log_odds_chosen": 6.930571556091309, + "log_odds_ratio": -0.13519811630249023, + "logits/chosen": 0.1770489364862442, + "logits/rejected": 0.7756469249725342, + "logps/chosen": -1.2851059436798096, + "logps/rejected": -7.893743991851807, + "loss": 2.6715, + "nll_loss": 2.6580209732055664, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12851059436798096, + "rewards/margins": 0.6608638763427734, + "rewards/rejected": -0.7893744111061096, + "step": 1419 + }, + { + "epoch": 0.8833592534992224, + "grad_norm": 0.7085882425308228, + "learning_rate": 1.45e-05, + "log_odds_chosen": 14.094657897949219, + "log_odds_ratio": -0.16490913927555084, + "logits/chosen": 0.2271670699119568, + "logits/rejected": 1.1130411624908447, + "logps/chosen": -1.4116222858428955, + "logps/rejected": -15.165432929992676, + "loss": 2.8172, + "nll_loss": 2.8007044792175293, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1411622315645218, + "rewards/margins": 1.3753809928894043, + "rewards/rejected": -1.5165432691574097, + "step": 1420 + }, + { + "epoch": 0.8839813374805598, + "grad_norm": 0.48786014318466187, + "learning_rate": 1.4475e-05, + "log_odds_chosen": 13.460868835449219, + "log_odds_ratio": -0.013222498819231987, + "logits/chosen": 0.34067630767822266, + "logits/rejected": 1.1033310890197754, + "logps/chosen": -1.1284339427947998, + "logps/rejected": -14.182600975036621, + "loss": 3.205, + "nll_loss": 3.203630208969116, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11284340173006058, + "rewards/margins": 1.3054168224334717, + "rewards/rejected": -1.418260097503662, + "step": 1421 + }, + { + "epoch": 0.8846034214618974, + "grad_norm": 0.5590962767601013, + "learning_rate": 1.4449999999999999e-05, + "log_odds_chosen": 6.180453300476074, + "log_odds_ratio": -0.3521609902381897, + "logits/chosen": 0.30372947454452515, + "logits/rejected": 0.859440803527832, + "logps/chosen": -1.206611156463623, + "logps/rejected": -7.186957359313965, + "loss": 2.8295, + "nll_loss": 2.7942943572998047, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12066111713647842, + "rewards/margins": 0.5980346202850342, + "rewards/rejected": -0.7186957597732544, + "step": 1422 + }, + { + "epoch": 0.8852255054432349, + "grad_norm": 0.5221461057662964, + "learning_rate": 1.4425e-05, + "log_odds_chosen": 10.76657485961914, + "log_odds_ratio": -0.20132054388523102, + "logits/chosen": 0.19903302192687988, + "logits/rejected": 0.5170689225196838, + "logps/chosen": -1.120729923248291, + "logps/rejected": -11.55020523071289, + "loss": 3.1357, + "nll_loss": 3.1155667304992676, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11207298934459686, + "rewards/margins": 1.0429476499557495, + "rewards/rejected": -1.1550207138061523, + "step": 1423 + }, + { + "epoch": 0.8858475894245723, + "grad_norm": 0.5751407742500305, + "learning_rate": 1.44e-05, + "log_odds_chosen": 15.834976196289062, + "log_odds_ratio": -0.13728603720664978, + "logits/chosen": 0.3189513683319092, + "logits/rejected": 1.3567453622817993, + "logps/chosen": -1.0502710342407227, + "logps/rejected": -16.521549224853516, + "loss": 3.1854, + "nll_loss": 3.1716909408569336, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10502710938453674, + "rewards/margins": 1.5471278429031372, + "rewards/rejected": -1.652154803276062, + "step": 1424 + }, + { + "epoch": 0.8864696734059098, + "grad_norm": 0.5363021492958069, + "learning_rate": 1.4374999999999999e-05, + "log_odds_chosen": 19.794139862060547, + "log_odds_ratio": -0.0894283652305603, + "logits/chosen": 0.1790408045053482, + "logits/rejected": 1.2223554849624634, + "logps/chosen": -1.062679648399353, + "logps/rejected": -20.486034393310547, + "loss": 2.3991, + "nll_loss": 2.390169382095337, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10626795887947083, + "rewards/margins": 1.9423353672027588, + "rewards/rejected": -2.0486035346984863, + "step": 1425 + }, + { + "epoch": 0.8870917573872473, + "grad_norm": 0.4193936288356781, + "learning_rate": 1.435e-05, + "log_odds_chosen": 19.63629913330078, + "log_odds_ratio": -0.014150548726320267, + "logits/chosen": 0.20590832829475403, + "logits/rejected": 1.3159328699111938, + "logps/chosen": -1.1567211151123047, + "logps/rejected": -20.181039810180664, + "loss": 3.0019, + "nll_loss": 3.0004405975341797, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11567211151123047, + "rewards/margins": 1.9024319648742676, + "rewards/rejected": -2.018104076385498, + "step": 1426 + }, + { + "epoch": 0.8877138413685848, + "grad_norm": 3.831439256668091, + "learning_rate": 1.4325e-05, + "log_odds_chosen": 7.438900470733643, + "log_odds_ratio": -0.2074524164199829, + "logits/chosen": 0.35610508918762207, + "logits/rejected": 0.6404019594192505, + "logps/chosen": -0.8985580205917358, + "logps/rejected": -7.916787624359131, + "loss": 3.1508, + "nll_loss": 3.130077838897705, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08985579758882523, + "rewards/margins": 0.7018229961395264, + "rewards/rejected": -0.791678786277771, + "step": 1427 + }, + { + "epoch": 0.8883359253499222, + "grad_norm": 1.092878818511963, + "learning_rate": 1.43e-05, + "log_odds_chosen": 18.112590789794922, + "log_odds_ratio": -0.06469640880823135, + "logits/chosen": 0.5067091584205627, + "logits/rejected": 1.4463820457458496, + "logps/chosen": -1.6346746683120728, + "logps/rejected": -19.401138305664062, + "loss": 3.9286, + "nll_loss": 3.922130584716797, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.16346746683120728, + "rewards/margins": 1.776646614074707, + "rewards/rejected": -1.9401140213012695, + "step": 1428 + }, + { + "epoch": 0.8889580093312597, + "grad_norm": 0.40182310342788696, + "learning_rate": 1.4275e-05, + "log_odds_chosen": 17.829360961914062, + "log_odds_ratio": -4.7570942115271464e-05, + "logits/chosen": 0.4751187562942505, + "logits/rejected": 1.7462925910949707, + "logps/chosen": -1.5762360095977783, + "logps/rejected": -19.148487091064453, + "loss": 3.6646, + "nll_loss": 3.6646087169647217, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15762358903884888, + "rewards/margins": 1.7572250366210938, + "rewards/rejected": -1.9148486852645874, + "step": 1429 + }, + { + "epoch": 0.8895800933125972, + "grad_norm": 0.4239805340766907, + "learning_rate": 1.4249999999999999e-05, + "log_odds_chosen": 20.32266616821289, + "log_odds_ratio": -0.15288175642490387, + "logits/chosen": 0.2174326777458191, + "logits/rejected": 1.30299711227417, + "logps/chosen": -1.0801613330841064, + "logps/rejected": -21.01744270324707, + "loss": 2.8315, + "nll_loss": 2.8161635398864746, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10801612585783005, + "rewards/margins": 1.9937282800674438, + "rewards/rejected": -2.1017441749572754, + "step": 1430 + }, + { + "epoch": 0.8902021772939347, + "grad_norm": 0.4994823634624481, + "learning_rate": 1.4225e-05, + "log_odds_chosen": 15.963156700134277, + "log_odds_ratio": -0.03521762415766716, + "logits/chosen": 0.191191628575325, + "logits/rejected": 1.2609500885009766, + "logps/chosen": -1.1631869077682495, + "logps/rejected": -16.678741455078125, + "loss": 2.6632, + "nll_loss": 2.659717559814453, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11631868779659271, + "rewards/margins": 1.5515555143356323, + "rewards/rejected": -1.6678742170333862, + "step": 1431 + }, + { + "epoch": 0.8908242612752721, + "grad_norm": 0.5873495936393738, + "learning_rate": 1.42e-05, + "log_odds_chosen": 14.923879623413086, + "log_odds_ratio": -0.00032400363124907017, + "logits/chosen": 0.14456267654895782, + "logits/rejected": 0.8725841045379639, + "logps/chosen": -1.1998927593231201, + "logps/rejected": -15.70447826385498, + "loss": 2.725, + "nll_loss": 2.7249717712402344, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11998927593231201, + "rewards/margins": 1.4504585266113281, + "rewards/rejected": -1.5704479217529297, + "step": 1432 + }, + { + "epoch": 0.8914463452566096, + "grad_norm": 0.6210421919822693, + "learning_rate": 1.4174999999999999e-05, + "log_odds_chosen": 11.426855087280273, + "log_odds_ratio": -0.1443122774362564, + "logits/chosen": 0.34937727451324463, + "logits/rejected": 1.107904076576233, + "logps/chosen": -1.5372898578643799, + "logps/rejected": -12.699395179748535, + "loss": 3.2649, + "nll_loss": 3.2504353523254395, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15372899174690247, + "rewards/margins": 1.1162105798721313, + "rewards/rejected": -1.269939661026001, + "step": 1433 + }, + { + "epoch": 0.8920684292379472, + "grad_norm": 0.6488243937492371, + "learning_rate": 1.415e-05, + "log_odds_chosen": 11.816390037536621, + "log_odds_ratio": -0.08364453911781311, + "logits/chosen": 0.2236700803041458, + "logits/rejected": 1.4047526121139526, + "logps/chosen": -1.4130889177322388, + "logps/rejected": -12.905828475952148, + "loss": 2.6891, + "nll_loss": 2.680753469467163, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14130890369415283, + "rewards/margins": 1.1492741107940674, + "rewards/rejected": -1.2905830144882202, + "step": 1434 + }, + { + "epoch": 0.8926905132192846, + "grad_norm": 7.441209316253662, + "learning_rate": 1.4125e-05, + "log_odds_chosen": 9.631667137145996, + "log_odds_ratio": -0.13277006149291992, + "logits/chosen": 0.20559415221214294, + "logits/rejected": 0.3517681062221527, + "logps/chosen": -1.7672669887542725, + "logps/rejected": -10.936025619506836, + "loss": 2.9401, + "nll_loss": 2.9268393516540527, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17672669887542725, + "rewards/margins": 0.9168758988380432, + "rewards/rejected": -1.0936025381088257, + "step": 1435 + }, + { + "epoch": 0.8933125972006221, + "grad_norm": 0.5252163410186768, + "learning_rate": 1.4099999999999999e-05, + "log_odds_chosen": 18.1024112701416, + "log_odds_ratio": -0.03475270792841911, + "logits/chosen": 0.1778755635023117, + "logits/rejected": 0.9054905772209167, + "logps/chosen": -1.1439323425292969, + "logps/rejected": -18.793231964111328, + "loss": 2.8542, + "nll_loss": 2.8507204055786133, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11439324170351028, + "rewards/margins": 1.764930248260498, + "rewards/rejected": -1.8793234825134277, + "step": 1436 + }, + { + "epoch": 0.8939346811819595, + "grad_norm": 0.4643388092517853, + "learning_rate": 1.4075e-05, + "log_odds_chosen": 11.433347702026367, + "log_odds_ratio": -0.13411425054073334, + "logits/chosen": 0.26451680064201355, + "logits/rejected": 1.064178705215454, + "logps/chosen": -0.990769624710083, + "logps/rejected": -11.884706497192383, + "loss": 2.7966, + "nll_loss": 2.7832188606262207, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09907696396112442, + "rewards/margins": 1.0893937349319458, + "rewards/rejected": -1.188470721244812, + "step": 1437 + }, + { + "epoch": 0.8945567651632971, + "grad_norm": 0.5520986318588257, + "learning_rate": 1.4050000000000003e-05, + "log_odds_chosen": 10.105766296386719, + "log_odds_ratio": -0.1801670640707016, + "logits/chosen": 0.12006427347660065, + "logits/rejected": 0.7376315593719482, + "logps/chosen": -1.650545597076416, + "logps/rejected": -11.550055503845215, + "loss": 2.5541, + "nll_loss": 2.5360946655273438, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1650545597076416, + "rewards/margins": 0.9899511337280273, + "rewards/rejected": -1.155005693435669, + "step": 1438 + }, + { + "epoch": 0.8951788491446345, + "grad_norm": 0.5198318362236023, + "learning_rate": 1.4025000000000002e-05, + "log_odds_chosen": 7.641714096069336, + "log_odds_ratio": -0.25751960277557373, + "logits/chosen": 0.29090097546577454, + "logits/rejected": 0.6141082048416138, + "logps/chosen": -1.2502827644348145, + "logps/rejected": -8.630484580993652, + "loss": 3.1904, + "nll_loss": 3.164672374725342, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12502828240394592, + "rewards/margins": 0.738020122051239, + "rewards/rejected": -0.8630484342575073, + "step": 1439 + }, + { + "epoch": 0.895800933125972, + "grad_norm": 0.4290095865726471, + "learning_rate": 1.4000000000000001e-05, + "log_odds_chosen": 14.732014656066895, + "log_odds_ratio": -0.08160438388586044, + "logits/chosen": 0.12874333560466766, + "logits/rejected": 0.9056077003479004, + "logps/chosen": -1.228428602218628, + "logps/rejected": -15.536664009094238, + "loss": 3.2002, + "nll_loss": 3.192028045654297, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12284286320209503, + "rewards/margins": 1.4308236837387085, + "rewards/rejected": -1.5536664724349976, + "step": 1440 + }, + { + "epoch": 0.8964230171073095, + "grad_norm": 0.30635496973991394, + "learning_rate": 1.3975000000000003e-05, + "log_odds_chosen": 14.656373023986816, + "log_odds_ratio": -0.0002228428202215582, + "logits/chosen": 0.23041866719722748, + "logits/rejected": 1.2936317920684814, + "logps/chosen": -0.9796774387359619, + "logps/rejected": -14.984707832336426, + "loss": 2.9882, + "nll_loss": 2.98820161819458, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0979677364230156, + "rewards/margins": 1.400503158569336, + "rewards/rejected": -1.4984709024429321, + "step": 1441 + }, + { + "epoch": 0.897045101088647, + "grad_norm": 0.41320523619651794, + "learning_rate": 1.3950000000000002e-05, + "log_odds_chosen": 10.99651050567627, + "log_odds_ratio": -0.32272082567214966, + "logits/chosen": 0.19428414106369019, + "logits/rejected": 0.7030611634254456, + "logps/chosen": -1.1404513120651245, + "logps/rejected": -11.859764099121094, + "loss": 3.2682, + "nll_loss": 3.235926628112793, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11404513567686081, + "rewards/margins": 1.0719313621520996, + "rewards/rejected": -1.1859763860702515, + "step": 1442 + }, + { + "epoch": 0.8976671850699844, + "grad_norm": 0.7050619721412659, + "learning_rate": 1.3925000000000001e-05, + "log_odds_chosen": 11.484809875488281, + "log_odds_ratio": -0.15941309928894043, + "logits/chosen": 0.35777807235717773, + "logits/rejected": 1.1093180179595947, + "logps/chosen": -1.2923328876495361, + "logps/rejected": -12.462808609008789, + "loss": 3.3092, + "nll_loss": 3.293220043182373, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12923328578472137, + "rewards/margins": 1.1170475482940674, + "rewards/rejected": -1.2462809085845947, + "step": 1443 + }, + { + "epoch": 0.8982892690513219, + "grad_norm": 0.412223219871521, + "learning_rate": 1.3900000000000002e-05, + "log_odds_chosen": 18.409934997558594, + "log_odds_ratio": -0.0002995289978571236, + "logits/chosen": 0.2523375451564789, + "logits/rejected": 1.2306113243103027, + "logps/chosen": -1.0025357007980347, + "logps/rejected": -18.848125457763672, + "loss": 3.369, + "nll_loss": 3.368983745574951, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10025357455015182, + "rewards/margins": 1.7845587730407715, + "rewards/rejected": -1.884812355041504, + "step": 1444 + }, + { + "epoch": 0.8989113530326595, + "grad_norm": 0.3040986955165863, + "learning_rate": 1.3875000000000002e-05, + "log_odds_chosen": 17.37039566040039, + "log_odds_ratio": -1.7285432249991572e-06, + "logits/chosen": 0.5640264749526978, + "logits/rejected": 1.9091010093688965, + "logps/chosen": -0.8681442737579346, + "logps/rejected": -17.551462173461914, + "loss": 4.1642, + "nll_loss": 4.164191246032715, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08681442588567734, + "rewards/margins": 1.6683318614959717, + "rewards/rejected": -1.7551462650299072, + "step": 1445 + }, + { + "epoch": 0.8995334370139969, + "grad_norm": 0.7804985046386719, + "learning_rate": 1.3850000000000001e-05, + "log_odds_chosen": 12.528520584106445, + "log_odds_ratio": -0.036599528044462204, + "logits/chosen": 0.31422773003578186, + "logits/rejected": 1.119031310081482, + "logps/chosen": -1.1160674095153809, + "logps/rejected": -13.23311996459961, + "loss": 2.9005, + "nll_loss": 2.896796703338623, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11160673946142197, + "rewards/margins": 1.211705207824707, + "rewards/rejected": -1.3233120441436768, + "step": 1446 + }, + { + "epoch": 0.9001555209953344, + "grad_norm": 0.41449618339538574, + "learning_rate": 1.3825000000000002e-05, + "log_odds_chosen": 11.29874038696289, + "log_odds_ratio": -0.19807516038417816, + "logits/chosen": 0.07104424387216568, + "logits/rejected": 0.4711911380290985, + "logps/chosen": -1.0898693799972534, + "logps/rejected": -12.04837703704834, + "loss": 3.2759, + "nll_loss": 3.256134271621704, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10898693650960922, + "rewards/margins": 1.0958508253097534, + "rewards/rejected": -1.2048377990722656, + "step": 1447 + }, + { + "epoch": 0.9007776049766718, + "grad_norm": 0.42537832260131836, + "learning_rate": 1.3800000000000002e-05, + "log_odds_chosen": 14.00430965423584, + "log_odds_ratio": -0.10201773792505264, + "logits/chosen": 0.2881453037261963, + "logits/rejected": 1.2576179504394531, + "logps/chosen": -0.884018063545227, + "logps/rejected": -14.224514961242676, + "loss": 3.2637, + "nll_loss": 3.2534520626068115, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08840180933475494, + "rewards/margins": 1.3340497016906738, + "rewards/rejected": -1.4224514961242676, + "step": 1448 + }, + { + "epoch": 0.9013996889580094, + "grad_norm": 0.42243319749832153, + "learning_rate": 1.3775000000000001e-05, + "log_odds_chosen": 11.295740127563477, + "log_odds_ratio": -0.10844042152166367, + "logits/chosen": 0.16659203171730042, + "logits/rejected": 0.9894870519638062, + "logps/chosen": -1.0629875659942627, + "logps/rejected": -11.810334205627441, + "loss": 3.1585, + "nll_loss": 3.1476120948791504, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10629875957965851, + "rewards/margins": 1.0747345685958862, + "rewards/rejected": -1.1810333728790283, + "step": 1449 + }, + { + "epoch": 0.9020217729393468, + "grad_norm": 0.5875496864318848, + "learning_rate": 1.3750000000000002e-05, + "log_odds_chosen": 10.801142692565918, + "log_odds_ratio": -0.17056904733181, + "logits/chosen": 0.09089596569538116, + "logits/rejected": 0.5544472932815552, + "logps/chosen": -1.0837565660476685, + "logps/rejected": -11.519688606262207, + "loss": 2.8123, + "nll_loss": 2.795261859893799, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1083756536245346, + "rewards/margins": 1.043593168258667, + "rewards/rejected": -1.1519688367843628, + "step": 1450 + }, + { + "epoch": 0.9026438569206843, + "grad_norm": 0.6801373362541199, + "learning_rate": 1.3725000000000002e-05, + "log_odds_chosen": 11.216224670410156, + "log_odds_ratio": -0.11024151742458344, + "logits/chosen": 0.15584243834018707, + "logits/rejected": 0.7682511210441589, + "logps/chosen": -1.1805226802825928, + "logps/rejected": -12.04312801361084, + "loss": 3.2438, + "nll_loss": 3.2327301502227783, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11805225908756256, + "rewards/margins": 1.0862605571746826, + "rewards/rejected": -1.204312801361084, + "step": 1451 + }, + { + "epoch": 0.9032659409020217, + "grad_norm": 1.5156279802322388, + "learning_rate": 1.3700000000000001e-05, + "log_odds_chosen": 11.422454833984375, + "log_odds_ratio": -0.11768678575754166, + "logits/chosen": 0.23056858777999878, + "logits/rejected": 1.1006563901901245, + "logps/chosen": -1.2227720022201538, + "logps/rejected": -12.296416282653809, + "loss": 3.1485, + "nll_loss": 3.1367223262786865, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12227719277143478, + "rewards/margins": 1.107364535331726, + "rewards/rejected": -1.2296416759490967, + "step": 1452 + }, + { + "epoch": 0.9038880248833593, + "grad_norm": 0.4438559114933014, + "learning_rate": 1.3675000000000002e-05, + "log_odds_chosen": 21.953859329223633, + "log_odds_ratio": -1.4901162970204496e-08, + "logits/chosen": 0.14678457379341125, + "logits/rejected": 0.9667971730232239, + "logps/chosen": -0.9780452251434326, + "logps/rejected": -22.29878044128418, + "loss": 2.7227, + "nll_loss": 2.7226831912994385, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09780453145503998, + "rewards/margins": 2.132073402404785, + "rewards/rejected": -2.2298779487609863, + "step": 1453 + }, + { + "epoch": 0.9045101088646967, + "grad_norm": 2.3538661003112793, + "learning_rate": 1.3650000000000001e-05, + "log_odds_chosen": 12.744718551635742, + "log_odds_ratio": -0.06637068092823029, + "logits/chosen": 0.18035347759723663, + "logits/rejected": 0.8537945747375488, + "logps/chosen": -2.0617189407348633, + "logps/rejected": -14.523418426513672, + "loss": 3.229, + "nll_loss": 3.2223703861236572, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.206171914935112, + "rewards/margins": 1.246169924736023, + "rewards/rejected": -1.4523417949676514, + "step": 1454 + }, + { + "epoch": 0.9051321928460342, + "grad_norm": 0.5291186571121216, + "learning_rate": 1.3625e-05, + "log_odds_chosen": 9.619067192077637, + "log_odds_ratio": -0.1370728313922882, + "logits/chosen": 0.10796351730823517, + "logits/rejected": 0.5130314826965332, + "logps/chosen": -0.9000706672668457, + "logps/rejected": -9.873458862304688, + "loss": 2.6361, + "nll_loss": 2.622385025024414, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09000707417726517, + "rewards/margins": 0.8973388671875, + "rewards/rejected": -0.9873458743095398, + "step": 1455 + }, + { + "epoch": 0.9057542768273716, + "grad_norm": 0.5473167896270752, + "learning_rate": 1.3600000000000002e-05, + "log_odds_chosen": 9.86450481414795, + "log_odds_ratio": -0.08004651963710785, + "logits/chosen": 0.2445843517780304, + "logits/rejected": 0.9183000326156616, + "logps/chosen": -1.064620018005371, + "logps/rejected": -10.451854705810547, + "loss": 3.2033, + "nll_loss": 3.1953063011169434, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10646200180053711, + "rewards/margins": 0.9387235641479492, + "rewards/rejected": -1.0451855659484863, + "step": 1456 + }, + { + "epoch": 0.9063763608087092, + "grad_norm": 0.4545001685619354, + "learning_rate": 1.3575000000000001e-05, + "log_odds_chosen": 14.232173919677734, + "log_odds_ratio": -0.15462376177310944, + "logits/chosen": 0.3833097517490387, + "logits/rejected": 1.2270677089691162, + "logps/chosen": -1.1023833751678467, + "logps/rejected": -14.765386581420898, + "loss": 3.3058, + "nll_loss": 3.2903385162353516, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11023833602666855, + "rewards/margins": 1.366300344467163, + "rewards/rejected": -1.4765386581420898, + "step": 1457 + }, + { + "epoch": 0.9069984447900467, + "grad_norm": 0.41073378920555115, + "learning_rate": 1.3550000000000002e-05, + "log_odds_chosen": 7.300911903381348, + "log_odds_ratio": -0.1350908875465393, + "logits/chosen": 0.18523235619068146, + "logits/rejected": 0.4215852618217468, + "logps/chosen": -0.9286333918571472, + "logps/rejected": -7.772511959075928, + "loss": 3.1842, + "nll_loss": 3.170691728591919, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09286333620548248, + "rewards/margins": 0.6843879222869873, + "rewards/rejected": -0.7772511839866638, + "step": 1458 + }, + { + "epoch": 0.9076205287713841, + "grad_norm": 0.499830961227417, + "learning_rate": 1.3525000000000002e-05, + "log_odds_chosen": 16.171920776367188, + "log_odds_ratio": -0.0024008448235690594, + "logits/chosen": 0.30481821298599243, + "logits/rejected": 1.3369604349136353, + "logps/chosen": -0.9462791681289673, + "logps/rejected": -16.467857360839844, + "loss": 3.1955, + "nll_loss": 3.19523286819458, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09462791681289673, + "rewards/margins": 1.5521578788757324, + "rewards/rejected": -1.6467856168746948, + "step": 1459 + }, + { + "epoch": 0.9082426127527217, + "grad_norm": 0.44492849707603455, + "learning_rate": 1.3500000000000001e-05, + "log_odds_chosen": 10.069293975830078, + "log_odds_ratio": -0.0921773836016655, + "logits/chosen": 0.3066140413284302, + "logits/rejected": 0.8817154765129089, + "logps/chosen": -1.0714751482009888, + "logps/rejected": -10.760387420654297, + "loss": 3.5663, + "nll_loss": 3.5571231842041016, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10714752227067947, + "rewards/margins": 0.9688910841941833, + "rewards/rejected": -1.0760385990142822, + "step": 1460 + }, + { + "epoch": 0.9088646967340591, + "grad_norm": 0.6452760696411133, + "learning_rate": 1.3475000000000002e-05, + "log_odds_chosen": 12.372976303100586, + "log_odds_ratio": -0.22037629783153534, + "logits/chosen": 0.24817940592765808, + "logits/rejected": 0.922889232635498, + "logps/chosen": -1.1425799131393433, + "logps/rejected": -13.200763702392578, + "loss": 2.7866, + "nll_loss": 2.764538288116455, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11425799131393433, + "rewards/margins": 1.2058184146881104, + "rewards/rejected": -1.3200764656066895, + "step": 1461 + }, + { + "epoch": 0.9094867807153966, + "grad_norm": 0.6373099684715271, + "learning_rate": 1.3450000000000002e-05, + "log_odds_chosen": 10.688899040222168, + "log_odds_ratio": -0.2276204526424408, + "logits/chosen": 0.5093823075294495, + "logits/rejected": 1.0521379709243774, + "logps/chosen": -1.4107290506362915, + "logps/rejected": -11.85671615600586, + "loss": 4.0253, + "nll_loss": 4.002525329589844, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14107291400432587, + "rewards/margins": 1.0445988178253174, + "rewards/rejected": -1.1856716871261597, + "step": 1462 + }, + { + "epoch": 0.910108864696734, + "grad_norm": 0.5296677947044373, + "learning_rate": 1.3425000000000001e-05, + "log_odds_chosen": 12.754611015319824, + "log_odds_ratio": -0.19718533754348755, + "logits/chosen": 0.3532048165798187, + "logits/rejected": 1.3405009508132935, + "logps/chosen": -1.1448729038238525, + "logps/rejected": -13.596270561218262, + "loss": 3.2044, + "nll_loss": 3.1846399307250977, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11448729038238525, + "rewards/margins": 1.2451398372650146, + "rewards/rejected": -1.3596272468566895, + "step": 1463 + }, + { + "epoch": 0.9107309486780716, + "grad_norm": 0.9786263108253479, + "learning_rate": 1.3400000000000002e-05, + "log_odds_chosen": 7.895408630371094, + "log_odds_ratio": -0.18677686154842377, + "logits/chosen": 0.09396594762802124, + "logits/rejected": 0.8790769577026367, + "logps/chosen": -1.0056233406066895, + "logps/rejected": -8.473982810974121, + "loss": 2.7747, + "nll_loss": 2.755993366241455, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10056232661008835, + "rewards/margins": 0.7468359470367432, + "rewards/rejected": -0.8473982810974121, + "step": 1464 + }, + { + "epoch": 0.911353032659409, + "grad_norm": 2.532768726348877, + "learning_rate": 1.3375000000000002e-05, + "log_odds_chosen": 16.206634521484375, + "log_odds_ratio": -0.0901508703827858, + "logits/chosen": 0.29881221055984497, + "logits/rejected": 1.2665278911590576, + "logps/chosen": -1.5193629264831543, + "logps/rejected": -17.421382904052734, + "loss": 3.2096, + "nll_loss": 3.200589895248413, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15193629264831543, + "rewards/margins": 1.5902019739151, + "rewards/rejected": -1.7421382665634155, + "step": 1465 + }, + { + "epoch": 0.9119751166407465, + "grad_norm": 0.45200738310813904, + "learning_rate": 1.3350000000000001e-05, + "log_odds_chosen": 12.099109649658203, + "log_odds_ratio": -0.10622071474790573, + "logits/chosen": 0.27904149889945984, + "logits/rejected": 1.2529817819595337, + "logps/chosen": -1.0653222799301147, + "logps/rejected": -12.664470672607422, + "loss": 3.0189, + "nll_loss": 3.0083017349243164, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10653223097324371, + "rewards/margins": 1.1599149703979492, + "rewards/rejected": -1.2664470672607422, + "step": 1466 + }, + { + "epoch": 0.9125972006220839, + "grad_norm": 0.5577512383460999, + "learning_rate": 1.3325000000000002e-05, + "log_odds_chosen": 14.484386444091797, + "log_odds_ratio": -0.007911762222647667, + "logits/chosen": 0.1263856291770935, + "logits/rejected": 0.8978151082992554, + "logps/chosen": -0.9192382097244263, + "logps/rejected": -14.804035186767578, + "loss": 2.5146, + "nll_loss": 2.513773202896118, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09192381799221039, + "rewards/margins": 1.3884797096252441, + "rewards/rejected": -1.4804035425186157, + "step": 1467 + }, + { + "epoch": 0.9132192846034215, + "grad_norm": 0.4941674768924713, + "learning_rate": 1.3300000000000001e-05, + "log_odds_chosen": 10.110100746154785, + "log_odds_ratio": -0.03399426490068436, + "logits/chosen": 0.22827720642089844, + "logits/rejected": 0.8136681914329529, + "logps/chosen": -0.9263241291046143, + "logps/rejected": -10.246597290039062, + "loss": 3.2691, + "nll_loss": 3.265674352645874, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09263241291046143, + "rewards/margins": 0.932027280330658, + "rewards/rejected": -1.0246597528457642, + "step": 1468 + }, + { + "epoch": 0.913841368584759, + "grad_norm": 0.6259835362434387, + "learning_rate": 1.3275e-05, + "log_odds_chosen": 11.36493968963623, + "log_odds_ratio": -0.19162523746490479, + "logits/chosen": 0.19190458953380585, + "logits/rejected": 0.771049976348877, + "logps/chosen": -1.0270004272460938, + "logps/rejected": -11.900120735168457, + "loss": 2.7219, + "nll_loss": 2.70273494720459, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10270003974437714, + "rewards/margins": 1.0873119831085205, + "rewards/rejected": -1.1900120973587036, + "step": 1469 + }, + { + "epoch": 0.9144634525660964, + "grad_norm": 0.8540546298027039, + "learning_rate": 1.3250000000000002e-05, + "log_odds_chosen": 9.284761428833008, + "log_odds_ratio": -0.2318931221961975, + "logits/chosen": 0.22063320875167847, + "logits/rejected": 0.7904773354530334, + "logps/chosen": -1.284113883972168, + "logps/rejected": -10.228588104248047, + "loss": 2.8488, + "nll_loss": 2.825575828552246, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1284113973379135, + "rewards/margins": 0.8944474458694458, + "rewards/rejected": -1.0228588581085205, + "step": 1470 + }, + { + "epoch": 0.9150855365474339, + "grad_norm": 0.5292484760284424, + "learning_rate": 1.3225000000000001e-05, + "log_odds_chosen": 12.626777648925781, + "log_odds_ratio": -0.13767273724079132, + "logits/chosen": 0.2530801594257355, + "logits/rejected": 1.1952565908432007, + "logps/chosen": -0.9651281833648682, + "logps/rejected": -13.066086769104004, + "loss": 2.8352, + "nll_loss": 2.821428060531616, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0965128242969513, + "rewards/margins": 1.2100958824157715, + "rewards/rejected": -1.3066086769104004, + "step": 1471 + }, + { + "epoch": 0.9157076205287714, + "grad_norm": 0.737404465675354, + "learning_rate": 1.32e-05, + "log_odds_chosen": 14.638701438903809, + "log_odds_ratio": -0.0027568021323531866, + "logits/chosen": 0.38527604937553406, + "logits/rejected": 1.2940354347229004, + "logps/chosen": -1.0043842792510986, + "logps/rejected": -15.046712875366211, + "loss": 2.9281, + "nll_loss": 2.9278013706207275, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1004384309053421, + "rewards/margins": 1.4042328596115112, + "rewards/rejected": -1.5046712160110474, + "step": 1472 + }, + { + "epoch": 0.9163297045101089, + "grad_norm": 0.4254592955112457, + "learning_rate": 1.3175000000000002e-05, + "log_odds_chosen": 18.022205352783203, + "log_odds_ratio": -0.002449390245601535, + "logits/chosen": 0.45590126514434814, + "logits/rejected": 1.8061012029647827, + "logps/chosen": -1.1624892950057983, + "logps/rejected": -18.69601058959961, + "loss": 3.6024, + "nll_loss": 3.6021316051483154, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11624892801046371, + "rewards/margins": 1.753352165222168, + "rewards/rejected": -1.8696010112762451, + "step": 1473 + }, + { + "epoch": 0.9169517884914463, + "grad_norm": 0.4656004309654236, + "learning_rate": 1.3150000000000001e-05, + "log_odds_chosen": 11.069246292114258, + "log_odds_ratio": -0.14720577001571655, + "logits/chosen": 0.1729264259338379, + "logits/rejected": 0.8423563241958618, + "logps/chosen": -1.3754249811172485, + "logps/rejected": -12.108327865600586, + "loss": 2.8642, + "nll_loss": 2.8495240211486816, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1375425010919571, + "rewards/margins": 1.073290228843689, + "rewards/rejected": -1.2108327150344849, + "step": 1474 + }, + { + "epoch": 0.9175738724727839, + "grad_norm": 0.8528266549110413, + "learning_rate": 1.3125e-05, + "log_odds_chosen": 10.443950653076172, + "log_odds_ratio": -0.059229776263237, + "logits/chosen": 0.3430590033531189, + "logits/rejected": 1.0942491292953491, + "logps/chosen": -1.181295394897461, + "logps/rejected": -11.18059253692627, + "loss": 3.3527, + "nll_loss": 3.3467843532562256, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11812953650951385, + "rewards/margins": 0.9999297261238098, + "rewards/rejected": -1.1180591583251953, + "step": 1475 + }, + { + "epoch": 0.9181959564541213, + "grad_norm": 0.748715877532959, + "learning_rate": 1.3100000000000002e-05, + "log_odds_chosen": 9.176325798034668, + "log_odds_ratio": -0.19434534013271332, + "logits/chosen": 0.27273157238960266, + "logits/rejected": 0.9127519130706787, + "logps/chosen": -1.1760427951812744, + "logps/rejected": -9.9922513961792, + "loss": 2.9926, + "nll_loss": 2.973123550415039, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11760427802801132, + "rewards/margins": 0.8816208839416504, + "rewards/rejected": -0.9992251396179199, + "step": 1476 + }, + { + "epoch": 0.9188180404354588, + "grad_norm": 0.4457939863204956, + "learning_rate": 1.3075000000000001e-05, + "log_odds_chosen": 17.727825164794922, + "log_odds_ratio": -0.0003742209810297936, + "logits/chosen": 0.489930659532547, + "logits/rejected": 1.601247787475586, + "logps/chosen": -1.2831439971923828, + "logps/rejected": -18.507240295410156, + "loss": 3.5756, + "nll_loss": 3.575566530227661, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12831439077854156, + "rewards/margins": 1.722409725189209, + "rewards/rejected": -1.8507239818572998, + "step": 1477 + }, + { + "epoch": 0.9194401244167962, + "grad_norm": 0.5372123718261719, + "learning_rate": 1.305e-05, + "log_odds_chosen": 5.822672367095947, + "log_odds_ratio": -0.24347680807113647, + "logits/chosen": 0.2464301735162735, + "logits/rejected": 0.44997262954711914, + "logps/chosen": -1.2049652338027954, + "logps/rejected": -6.674422740936279, + "loss": 3.2726, + "nll_loss": 3.2482857704162598, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12049652636051178, + "rewards/margins": 0.5469457507133484, + "rewards/rejected": -0.6674423217773438, + "step": 1478 + }, + { + "epoch": 0.9200622083981338, + "grad_norm": 0.4896005094051361, + "learning_rate": 1.3025000000000002e-05, + "log_odds_chosen": 12.5404052734375, + "log_odds_ratio": -0.08044345676898956, + "logits/chosen": 0.38424739241600037, + "logits/rejected": 1.3379318714141846, + "logps/chosen": -0.9839893579483032, + "logps/rejected": -13.034258842468262, + "loss": 3.1035, + "nll_loss": 3.0955049991607666, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09839893877506256, + "rewards/margins": 1.2050269842147827, + "rewards/rejected": -1.3034260272979736, + "step": 1479 + }, + { + "epoch": 0.9206842923794712, + "grad_norm": 0.5063160061836243, + "learning_rate": 1.3000000000000001e-05, + "log_odds_chosen": 9.515791893005371, + "log_odds_ratio": -0.16696563363075256, + "logits/chosen": 0.36880922317504883, + "logits/rejected": 0.9227983355522156, + "logps/chosen": -1.0066584348678589, + "logps/rejected": -10.174430847167969, + "loss": 3.3245, + "nll_loss": 3.3077735900878906, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10066584497690201, + "rewards/margins": 0.9167773127555847, + "rewards/rejected": -1.017443060874939, + "step": 1480 + }, + { + "epoch": 0.9213063763608087, + "grad_norm": 0.4576911926269531, + "learning_rate": 1.2975e-05, + "log_odds_chosen": 12.466102600097656, + "log_odds_ratio": -0.18402725458145142, + "logits/chosen": 0.13083387911319733, + "logits/rejected": 0.53689044713974, + "logps/chosen": -1.0055042505264282, + "logps/rejected": -13.049692153930664, + "loss": 2.6695, + "nll_loss": 2.6510913372039795, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10055042803287506, + "rewards/margins": 1.2044188976287842, + "rewards/rejected": -1.304969310760498, + "step": 1481 + }, + { + "epoch": 0.9219284603421461, + "grad_norm": 0.46238452196121216, + "learning_rate": 1.2950000000000001e-05, + "log_odds_chosen": 15.337545394897461, + "log_odds_ratio": -0.03438110649585724, + "logits/chosen": 0.4146786034107208, + "logits/rejected": 1.442423701286316, + "logps/chosen": -1.1937940120697021, + "logps/rejected": -16.170305252075195, + "loss": 3.3299, + "nll_loss": 3.3265042304992676, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11937940120697021, + "rewards/margins": 1.4976511001586914, + "rewards/rejected": -1.6170305013656616, + "step": 1482 + }, + { + "epoch": 0.9225505443234837, + "grad_norm": 1.6496787071228027, + "learning_rate": 1.2925e-05, + "log_odds_chosen": 17.128173828125, + "log_odds_ratio": -8.068131137406453e-05, + "logits/chosen": 0.49006718397140503, + "logits/rejected": 1.4875874519348145, + "logps/chosen": -1.2655510902404785, + "logps/rejected": -18.002466201782227, + "loss": 3.4641, + "nll_loss": 3.46408748626709, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12655511498451233, + "rewards/margins": 1.6736915111541748, + "rewards/rejected": -1.8002467155456543, + "step": 1483 + }, + { + "epoch": 0.9231726283048212, + "grad_norm": 0.839210569858551, + "learning_rate": 1.29e-05, + "log_odds_chosen": 16.212020874023438, + "log_odds_ratio": -0.006270325277000666, + "logits/chosen": 0.256135493516922, + "logits/rejected": 1.3664355278015137, + "logps/chosen": -1.0374066829681396, + "logps/rejected": -16.69953155517578, + "loss": 2.8523, + "nll_loss": 2.851687431335449, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10374066978693008, + "rewards/margins": 1.5662124156951904, + "rewards/rejected": -1.6699531078338623, + "step": 1484 + }, + { + "epoch": 0.9237947122861586, + "grad_norm": 0.5375116467475891, + "learning_rate": 1.2875000000000001e-05, + "log_odds_chosen": 15.11935043334961, + "log_odds_ratio": -0.010710510425269604, + "logits/chosen": 0.23992419242858887, + "logits/rejected": 1.610540509223938, + "logps/chosen": -1.131101131439209, + "logps/rejected": -15.822179794311523, + "loss": 2.3641, + "nll_loss": 2.3630285263061523, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11311011761426926, + "rewards/margins": 1.469107985496521, + "rewards/rejected": -1.582218050956726, + "step": 1485 + }, + { + "epoch": 0.9244167962674961, + "grad_norm": 0.6357943415641785, + "learning_rate": 1.285e-05, + "log_odds_chosen": 5.553439140319824, + "log_odds_ratio": -0.39662641286849976, + "logits/chosen": 0.21328821778297424, + "logits/rejected": 0.5546770095825195, + "logps/chosen": -1.0445785522460938, + "logps/rejected": -6.2532267570495605, + "loss": 2.487, + "nll_loss": 2.4473471641540527, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10445785522460938, + "rewards/margins": 0.5208648443222046, + "rewards/rejected": -0.625322699546814, + "step": 1486 + }, + { + "epoch": 0.9250388802488336, + "grad_norm": 1.35960054397583, + "learning_rate": 1.2825000000000002e-05, + "log_odds_chosen": 13.23503303527832, + "log_odds_ratio": -0.026235153898596764, + "logits/chosen": 0.04117313399910927, + "logits/rejected": 0.4238998293876648, + "logps/chosen": -1.3150159120559692, + "logps/rejected": -14.009586334228516, + "loss": 2.9444, + "nll_loss": 2.9417426586151123, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13150160014629364, + "rewards/margins": 1.2694569826126099, + "rewards/rejected": -1.4009586572647095, + "step": 1487 + }, + { + "epoch": 0.9256609642301711, + "grad_norm": 0.8810969591140747, + "learning_rate": 1.2800000000000001e-05, + "log_odds_chosen": 13.211524963378906, + "log_odds_ratio": -0.08693347871303558, + "logits/chosen": 0.29597651958465576, + "logits/rejected": 1.2714580297470093, + "logps/chosen": -1.328726053237915, + "logps/rejected": -14.09039306640625, + "loss": 3.231, + "nll_loss": 3.2223141193389893, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1328725963830948, + "rewards/margins": 1.2761669158935547, + "rewards/rejected": -1.4090393781661987, + "step": 1488 + }, + { + "epoch": 0.9262830482115085, + "grad_norm": 0.6031063199043274, + "learning_rate": 1.2775e-05, + "log_odds_chosen": 13.652146339416504, + "log_odds_ratio": -0.0523035004734993, + "logits/chosen": 0.3231468200683594, + "logits/rejected": 0.9991359710693359, + "logps/chosen": -1.0230048894882202, + "logps/rejected": -14.191951751708984, + "loss": 3.2951, + "nll_loss": 3.2898614406585693, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1023004874587059, + "rewards/margins": 1.3168946504592896, + "rewards/rejected": -1.4191951751708984, + "step": 1489 + }, + { + "epoch": 0.926905132192846, + "grad_norm": 0.627905547618866, + "learning_rate": 1.2750000000000002e-05, + "log_odds_chosen": 10.043725967407227, + "log_odds_ratio": -0.15319031476974487, + "logits/chosen": 0.11043369770050049, + "logits/rejected": 0.7919861674308777, + "logps/chosen": -1.0833916664123535, + "logps/rejected": -10.649333953857422, + "loss": 2.8962, + "nll_loss": 2.8808720111846924, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10833916813135147, + "rewards/margins": 0.9565942883491516, + "rewards/rejected": -1.0649335384368896, + "step": 1490 + }, + { + "epoch": 0.9275272161741835, + "grad_norm": 0.4083629548549652, + "learning_rate": 1.2725000000000001e-05, + "log_odds_chosen": 14.011482238769531, + "log_odds_ratio": -0.10505443811416626, + "logits/chosen": 0.3975568115711212, + "logits/rejected": 1.282463550567627, + "logps/chosen": -0.7236562967300415, + "logps/rejected": -14.066780090332031, + "loss": 2.9918, + "nll_loss": 2.9812498092651367, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.07236562669277191, + "rewards/margins": 1.3343124389648438, + "rewards/rejected": -1.4066780805587769, + "step": 1491 + }, + { + "epoch": 0.928149300155521, + "grad_norm": 0.524310827255249, + "learning_rate": 1.27e-05, + "log_odds_chosen": 11.368038177490234, + "log_odds_ratio": -0.09671325981616974, + "logits/chosen": 0.35482871532440186, + "logits/rejected": 1.0728152990341187, + "logps/chosen": -1.3038160800933838, + "logps/rejected": -12.34891128540039, + "loss": 3.4129, + "nll_loss": 3.403229236602783, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13038161396980286, + "rewards/margins": 1.1045094728469849, + "rewards/rejected": -1.2348910570144653, + "step": 1492 + }, + { + "epoch": 0.9287713841368584, + "grad_norm": 0.5173730850219727, + "learning_rate": 1.2675000000000001e-05, + "log_odds_chosen": 12.411164283752441, + "log_odds_ratio": -0.0007505936082452536, + "logits/chosen": 0.20201897621154785, + "logits/rejected": 0.9776585102081299, + "logps/chosen": -1.2841124534606934, + "logps/rejected": -13.26869010925293, + "loss": 3.0104, + "nll_loss": 3.0102906227111816, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12841124832630157, + "rewards/margins": 1.1984577178955078, + "rewards/rejected": -1.326869010925293, + "step": 1493 + }, + { + "epoch": 0.929393468118196, + "grad_norm": 1.3060460090637207, + "learning_rate": 1.2650000000000001e-05, + "log_odds_chosen": 8.517346382141113, + "log_odds_ratio": -0.004937875550240278, + "logits/chosen": 0.31674104928970337, + "logits/rejected": 0.7935144901275635, + "logps/chosen": -1.203063726425171, + "logps/rejected": -9.244138717651367, + "loss": 3.5697, + "nll_loss": 3.5692033767700195, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12030638009309769, + "rewards/margins": 0.8041075468063354, + "rewards/rejected": -0.9244139194488525, + "step": 1494 + }, + { + "epoch": 0.9300155520995335, + "grad_norm": 0.758243978023529, + "learning_rate": 1.2625e-05, + "log_odds_chosen": 16.914310455322266, + "log_odds_ratio": -5.289989076118218e-06, + "logits/chosen": 0.4182242453098297, + "logits/rejected": 1.7866361141204834, + "logps/chosen": -0.9813098907470703, + "logps/rejected": -17.347728729248047, + "loss": 3.1011, + "nll_loss": 3.101064920425415, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09813098609447479, + "rewards/margins": 1.6366418600082397, + "rewards/rejected": -1.7347729206085205, + "step": 1495 + }, + { + "epoch": 0.9306376360808709, + "grad_norm": 0.61381995677948, + "learning_rate": 1.2600000000000001e-05, + "log_odds_chosen": 4.939112186431885, + "log_odds_ratio": -0.23473593592643738, + "logits/chosen": 0.18245118856430054, + "logits/rejected": 0.4139311611652374, + "logps/chosen": -1.11427903175354, + "logps/rejected": -5.520164966583252, + "loss": 3.0184, + "nll_loss": 2.9949193000793457, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1114279106259346, + "rewards/margins": 0.4405885636806488, + "rewards/rejected": -0.5520164966583252, + "step": 1496 + }, + { + "epoch": 0.9312597200622084, + "grad_norm": 0.6340899467468262, + "learning_rate": 1.2575e-05, + "log_odds_chosen": 5.411050796508789, + "log_odds_ratio": -0.32791846990585327, + "logits/chosen": 0.20967331528663635, + "logits/rejected": 0.4660225510597229, + "logps/chosen": -1.0537598133087158, + "logps/rejected": -6.0242204666137695, + "loss": 3.0802, + "nll_loss": 3.047390937805176, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1053759828209877, + "rewards/margins": 0.4970460534095764, + "rewards/rejected": -0.6024219989776611, + "step": 1497 + }, + { + "epoch": 0.9318818040435459, + "grad_norm": 0.5436956286430359, + "learning_rate": 1.255e-05, + "log_odds_chosen": 8.004091262817383, + "log_odds_ratio": -0.1258280873298645, + "logits/chosen": 0.11888009309768677, + "logits/rejected": 0.5211424827575684, + "logps/chosen": -1.5225926637649536, + "logps/rejected": -9.303966522216797, + "loss": 2.7858, + "nll_loss": 2.7731993198394775, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15225926041603088, + "rewards/margins": 0.7781373858451843, + "rewards/rejected": -0.9303966760635376, + "step": 1498 + }, + { + "epoch": 0.9325038880248834, + "grad_norm": 0.577427864074707, + "learning_rate": 1.2525000000000001e-05, + "log_odds_chosen": 9.300687789916992, + "log_odds_ratio": -0.15734805166721344, + "logits/chosen": 0.2356635332107544, + "logits/rejected": 1.1234387159347534, + "logps/chosen": -1.3030877113342285, + "logps/rejected": -10.311586380004883, + "loss": 2.6674, + "nll_loss": 2.6516411304473877, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13030877709388733, + "rewards/margins": 0.9008498191833496, + "rewards/rejected": -1.031158685684204, + "step": 1499 + }, + { + "epoch": 0.9331259720062208, + "grad_norm": 0.4966254234313965, + "learning_rate": 1.25e-05, + "log_odds_chosen": 7.163057327270508, + "log_odds_ratio": -0.11469916999340057, + "logits/chosen": 0.2907523810863495, + "logits/rejected": 0.8035860657691956, + "logps/chosen": -1.073452353477478, + "logps/rejected": -7.86415958404541, + "loss": 3.1622, + "nll_loss": 3.150710344314575, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10734523087739944, + "rewards/margins": 0.6790707111358643, + "rewards/rejected": -0.7864159345626831, + "step": 1500 + }, + { + "epoch": 0.9337480559875583, + "grad_norm": 0.7652081251144409, + "learning_rate": 1.2475e-05, + "log_odds_chosen": 8.80681324005127, + "log_odds_ratio": -0.10107364505529404, + "logits/chosen": 0.19027899205684662, + "logits/rejected": 0.9780977964401245, + "logps/chosen": -1.2092275619506836, + "logps/rejected": -9.490230560302734, + "loss": 2.8116, + "nll_loss": 2.8014516830444336, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1209227591753006, + "rewards/margins": 0.828100323677063, + "rewards/rejected": -0.9490230679512024, + "step": 1501 + }, + { + "epoch": 0.9343701399688958, + "grad_norm": 0.5554780960083008, + "learning_rate": 1.2450000000000001e-05, + "log_odds_chosen": 5.736096382141113, + "log_odds_ratio": -0.10775266587734222, + "logits/chosen": 0.21648985147476196, + "logits/rejected": 0.7037409543991089, + "logps/chosen": -1.4072825908660889, + "logps/rejected": -6.900873184204102, + "loss": 2.9205, + "nll_loss": 2.909726858139038, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14072826504707336, + "rewards/margins": 0.549359142780304, + "rewards/rejected": -0.6900873184204102, + "step": 1502 + }, + { + "epoch": 0.9349922239502333, + "grad_norm": 0.4990847110748291, + "learning_rate": 1.2425e-05, + "log_odds_chosen": 6.844992160797119, + "log_odds_ratio": -0.12505614757537842, + "logits/chosen": 0.1515367031097412, + "logits/rejected": 0.5935308933258057, + "logps/chosen": -1.2481999397277832, + "logps/rejected": -7.804927825927734, + "loss": 2.8519, + "nll_loss": 2.8393449783325195, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12482000142335892, + "rewards/margins": 0.6556727886199951, + "rewards/rejected": -0.7804927825927734, + "step": 1503 + }, + { + "epoch": 0.9356143079315707, + "grad_norm": 0.5404874682426453, + "learning_rate": 1.24e-05, + "log_odds_chosen": 12.516252517700195, + "log_odds_ratio": -0.001035528490319848, + "logits/chosen": 0.2302742600440979, + "logits/rejected": 0.9458526372909546, + "logps/chosen": -1.1124039888381958, + "logps/rejected": -13.01448917388916, + "loss": 2.7871, + "nll_loss": 2.786956548690796, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11124040186405182, + "rewards/margins": 1.1902084350585938, + "rewards/rejected": -1.3014488220214844, + "step": 1504 + }, + { + "epoch": 0.9362363919129082, + "grad_norm": 0.4964461326599121, + "learning_rate": 1.2375000000000001e-05, + "log_odds_chosen": 7.724780559539795, + "log_odds_ratio": -0.2758060395717621, + "logits/chosen": 0.26488885283470154, + "logits/rejected": 0.8047075867652893, + "logps/chosen": -1.2019445896148682, + "logps/rejected": -8.662567138671875, + "loss": 3.0769, + "nll_loss": 3.0493555068969727, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1201944649219513, + "rewards/margins": 0.7460623383522034, + "rewards/rejected": -0.8662567734718323, + "step": 1505 + }, + { + "epoch": 0.9368584758942458, + "grad_norm": 0.6931914687156677, + "learning_rate": 1.235e-05, + "log_odds_chosen": 10.359264373779297, + "log_odds_ratio": -0.2229677140712738, + "logits/chosen": 0.22140294313430786, + "logits/rejected": 0.8083517551422119, + "logps/chosen": -1.0726377964019775, + "logps/rejected": -11.14098834991455, + "loss": 2.9101, + "nll_loss": 2.8877878189086914, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10726377367973328, + "rewards/margins": 1.0068351030349731, + "rewards/rejected": -1.1140987873077393, + "step": 1506 + }, + { + "epoch": 0.9374805598755832, + "grad_norm": 0.5076847076416016, + "learning_rate": 1.2325e-05, + "log_odds_chosen": 13.57852840423584, + "log_odds_ratio": -0.00017591772484593093, + "logits/chosen": 0.15400853753089905, + "logits/rejected": 1.0751701593399048, + "logps/chosen": -1.0404856204986572, + "logps/rejected": -14.068706512451172, + "loss": 2.4864, + "nll_loss": 2.486356019973755, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10404857248067856, + "rewards/margins": 1.3028221130371094, + "rewards/rejected": -1.4068706035614014, + "step": 1507 + }, + { + "epoch": 0.9381026438569207, + "grad_norm": 0.5365278720855713, + "learning_rate": 1.23e-05, + "log_odds_chosen": 10.816153526306152, + "log_odds_ratio": -0.12300407141447067, + "logits/chosen": 0.0691407173871994, + "logits/rejected": 0.763458251953125, + "logps/chosen": -1.367710828781128, + "logps/rejected": -11.853177070617676, + "loss": 2.67, + "nll_loss": 2.6577091217041016, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1367710828781128, + "rewards/margins": 1.0485467910766602, + "rewards/rejected": -1.1853177547454834, + "step": 1508 + }, + { + "epoch": 0.9387247278382581, + "grad_norm": 1.6353999376296997, + "learning_rate": 1.2275e-05, + "log_odds_chosen": 6.172170639038086, + "log_odds_ratio": -0.28141510486602783, + "logits/chosen": 0.2383582592010498, + "logits/rejected": 0.6284301280975342, + "logps/chosen": -1.2594974040985107, + "logps/rejected": -6.96533203125, + "loss": 3.2839, + "nll_loss": 3.2557425498962402, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12594974040985107, + "rewards/margins": 0.5705834627151489, + "rewards/rejected": -0.696533203125, + "step": 1509 + }, + { + "epoch": 0.9393468118195957, + "grad_norm": 0.8588716387748718, + "learning_rate": 1.225e-05, + "log_odds_chosen": 4.53616189956665, + "log_odds_ratio": -0.2243345081806183, + "logits/chosen": 0.2073875516653061, + "logits/rejected": 0.4629060626029968, + "logps/chosen": -1.5986669063568115, + "logps/rejected": -5.792633533477783, + "loss": 2.6652, + "nll_loss": 2.642760753631592, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15986669063568115, + "rewards/margins": 0.41939669847488403, + "rewards/rejected": -0.5792633891105652, + "step": 1510 + }, + { + "epoch": 0.9399688958009331, + "grad_norm": 0.9089362621307373, + "learning_rate": 1.2225e-05, + "log_odds_chosen": 8.19037914276123, + "log_odds_ratio": -0.11289818584918976, + "logits/chosen": 0.18413937091827393, + "logits/rejected": 0.6798813939094543, + "logps/chosen": -0.737865686416626, + "logps/rejected": -7.826382160186768, + "loss": 2.6895, + "nll_loss": 2.6782355308532715, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07378657162189484, + "rewards/margins": 0.70885169506073, + "rewards/rejected": -0.7826382517814636, + "step": 1511 + }, + { + "epoch": 0.9405909797822706, + "grad_norm": 0.5411819815635681, + "learning_rate": 1.22e-05, + "log_odds_chosen": 10.123170852661133, + "log_odds_ratio": -0.16777871549129486, + "logits/chosen": 0.19700467586517334, + "logits/rejected": 0.7284948825836182, + "logps/chosen": -1.0894380807876587, + "logps/rejected": -10.83730411529541, + "loss": 2.4656, + "nll_loss": 2.4488000869750977, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10894380509853363, + "rewards/margins": 0.974786639213562, + "rewards/rejected": -1.0837304592132568, + "step": 1512 + }, + { + "epoch": 0.9412130637636081, + "grad_norm": 0.5812392234802246, + "learning_rate": 1.2175e-05, + "log_odds_chosen": 14.376922607421875, + "log_odds_ratio": -0.020723722875118256, + "logits/chosen": 0.3078254163265228, + "logits/rejected": 1.2554408311843872, + "logps/chosen": -0.8478683233261108, + "logps/rejected": -14.51792049407959, + "loss": 2.8554, + "nll_loss": 2.853318214416504, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08478683233261108, + "rewards/margins": 1.3670051097869873, + "rewards/rejected": -1.4517920017242432, + "step": 1513 + }, + { + "epoch": 0.9418351477449456, + "grad_norm": 7.773168087005615, + "learning_rate": 1.215e-05, + "log_odds_chosen": 10.952980041503906, + "log_odds_ratio": -0.2803260087966919, + "logits/chosen": 0.21798907220363617, + "logits/rejected": 0.8301116228103638, + "logps/chosen": -1.5885159969329834, + "logps/rejected": -12.118048667907715, + "loss": 3.4017, + "nll_loss": 3.373647928237915, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15885160863399506, + "rewards/margins": 1.0529532432556152, + "rewards/rejected": -1.2118048667907715, + "step": 1514 + }, + { + "epoch": 0.942457231726283, + "grad_norm": 0.5161781907081604, + "learning_rate": 1.2125e-05, + "log_odds_chosen": 10.635643005371094, + "log_odds_ratio": -0.16009806096553802, + "logits/chosen": 0.3003097176551819, + "logits/rejected": 0.7898467779159546, + "logps/chosen": -1.16111159324646, + "logps/rejected": -11.478269577026367, + "loss": 3.3575, + "nll_loss": 3.3414993286132812, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.116111159324646, + "rewards/margins": 1.0317158699035645, + "rewards/rejected": -1.1478270292282104, + "step": 1515 + }, + { + "epoch": 0.9430793157076205, + "grad_norm": 0.5859395861625671, + "learning_rate": 1.2100000000000001e-05, + "log_odds_chosen": 14.82717514038086, + "log_odds_ratio": -0.0002996036200784147, + "logits/chosen": 0.31332436203956604, + "logits/rejected": 1.3550702333450317, + "logps/chosen": -1.2290230989456177, + "logps/rejected": -15.66615104675293, + "loss": 2.9716, + "nll_loss": 2.9715206623077393, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12290230393409729, + "rewards/margins": 1.443712830543518, + "rewards/rejected": -1.566615104675293, + "step": 1516 + }, + { + "epoch": 0.943701399688958, + "grad_norm": 0.8364419341087341, + "learning_rate": 1.2075e-05, + "log_odds_chosen": 7.173739910125732, + "log_odds_ratio": -0.07395786046981812, + "logits/chosen": 0.307338148355484, + "logits/rejected": 0.6798582077026367, + "logps/chosen": -0.8903729915618896, + "logps/rejected": -7.38499641418457, + "loss": 3.2096, + "nll_loss": 3.2022242546081543, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08903729170560837, + "rewards/margins": 0.6494623422622681, + "rewards/rejected": -0.738499641418457, + "step": 1517 + }, + { + "epoch": 0.9443234836702955, + "grad_norm": 0.7112438678741455, + "learning_rate": 1.205e-05, + "log_odds_chosen": 4.342818260192871, + "log_odds_ratio": -0.16208398342132568, + "logits/chosen": 0.19038252532482147, + "logits/rejected": 0.6972472071647644, + "logps/chosen": -1.3774667978286743, + "logps/rejected": -5.464968204498291, + "loss": 2.9031, + "nll_loss": 2.886842727661133, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1377466768026352, + "rewards/margins": 0.40875014662742615, + "rewards/rejected": -0.5464968681335449, + "step": 1518 + }, + { + "epoch": 0.944945567651633, + "grad_norm": 0.5884023904800415, + "learning_rate": 1.2025000000000001e-05, + "log_odds_chosen": 11.338025093078613, + "log_odds_ratio": -0.02644185721874237, + "logits/chosen": 0.18361350893974304, + "logits/rejected": 0.8677639365196228, + "logps/chosen": -1.2472485303878784, + "logps/rejected": -12.216198921203613, + "loss": 2.9414, + "nll_loss": 2.938786506652832, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1247248575091362, + "rewards/margins": 1.0968950986862183, + "rewards/rejected": -1.2216198444366455, + "step": 1519 + }, + { + "epoch": 0.9455676516329704, + "grad_norm": 0.416962206363678, + "learning_rate": 1.2e-05, + "log_odds_chosen": 12.717013359069824, + "log_odds_ratio": -0.007837346754968166, + "logits/chosen": 0.2448866069316864, + "logits/rejected": 1.115902304649353, + "logps/chosen": -1.300870418548584, + "logps/rejected": -13.522640228271484, + "loss": 3.1906, + "nll_loss": 3.1898019313812256, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13008704781532288, + "rewards/margins": 1.2221770286560059, + "rewards/rejected": -1.352264165878296, + "step": 1520 + }, + { + "epoch": 0.946189735614308, + "grad_norm": 0.6918087601661682, + "learning_rate": 1.1975e-05, + "log_odds_chosen": 10.162681579589844, + "log_odds_ratio": -0.16703368723392487, + "logits/chosen": 0.1385897994041443, + "logits/rejected": 0.8961490988731384, + "logps/chosen": -1.1439799070358276, + "logps/rejected": -11.012479782104492, + "loss": 2.7063, + "nll_loss": 2.689603090286255, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11439798772335052, + "rewards/margins": 0.9868499040603638, + "rewards/rejected": -1.1012479066848755, + "step": 1521 + }, + { + "epoch": 0.9468118195956454, + "grad_norm": 0.849907398223877, + "learning_rate": 1.195e-05, + "log_odds_chosen": 13.232062339782715, + "log_odds_ratio": -0.00012399072875268757, + "logits/chosen": 0.3761923015117645, + "logits/rejected": 1.0816802978515625, + "logps/chosen": -1.2260112762451172, + "logps/rejected": -13.891904830932617, + "loss": 3.7704, + "nll_loss": 3.770425796508789, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12260112911462784, + "rewards/margins": 1.2665894031524658, + "rewards/rejected": -1.3891905546188354, + "step": 1522 + }, + { + "epoch": 0.9474339035769829, + "grad_norm": 1.2110743522644043, + "learning_rate": 1.1925e-05, + "log_odds_chosen": 14.117300033569336, + "log_odds_ratio": -0.07730953395366669, + "logits/chosen": 0.18281584978103638, + "logits/rejected": 1.4821653366088867, + "logps/chosen": -1.16435968875885, + "logps/rejected": -14.846202850341797, + "loss": 2.2272, + "nll_loss": 2.2194743156433105, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1164359599351883, + "rewards/margins": 1.3681843280792236, + "rewards/rejected": -1.4846203327178955, + "step": 1523 + }, + { + "epoch": 0.9480559875583203, + "grad_norm": 0.4888734519481659, + "learning_rate": 1.19e-05, + "log_odds_chosen": 12.100495338439941, + "log_odds_ratio": -0.26559317111968994, + "logits/chosen": 0.33780574798583984, + "logits/rejected": 0.9818902015686035, + "logps/chosen": -1.1453903913497925, + "logps/rejected": -12.870171546936035, + "loss": 3.0814, + "nll_loss": 3.0548579692840576, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11453904211521149, + "rewards/margins": 1.1724780797958374, + "rewards/rejected": -1.2870171070098877, + "step": 1524 + }, + { + "epoch": 0.9486780715396579, + "grad_norm": 3.656186103820801, + "learning_rate": 1.1875e-05, + "log_odds_chosen": 12.780552864074707, + "log_odds_ratio": -0.1394210159778595, + "logits/chosen": 0.3146355152130127, + "logits/rejected": 1.1511363983154297, + "logps/chosen": -1.582490086555481, + "logps/rejected": -14.108150482177734, + "loss": 3.215, + "nll_loss": 3.201098918914795, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15824902057647705, + "rewards/margins": 1.2525660991668701, + "rewards/rejected": -1.4108150005340576, + "step": 1525 + }, + { + "epoch": 0.9493001555209953, + "grad_norm": 0.49954620003700256, + "learning_rate": 1.185e-05, + "log_odds_chosen": 10.924458503723145, + "log_odds_ratio": -0.07372286170721054, + "logits/chosen": 0.2594726085662842, + "logits/rejected": 0.8973318338394165, + "logps/chosen": -0.8566200733184814, + "logps/rejected": -11.07498550415039, + "loss": 3.4027, + "nll_loss": 3.395279884338379, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08566200733184814, + "rewards/margins": 1.021836519241333, + "rewards/rejected": -1.1074985265731812, + "step": 1526 + }, + { + "epoch": 0.9499222395023328, + "grad_norm": 0.37401673197746277, + "learning_rate": 1.1825e-05, + "log_odds_chosen": 12.93400764465332, + "log_odds_ratio": -0.10499030351638794, + "logits/chosen": 0.04626879468560219, + "logits/rejected": 0.5351424217224121, + "logps/chosen": -1.0168579816818237, + "logps/rejected": -13.485580444335938, + "loss": 3.0433, + "nll_loss": 3.032796859741211, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10168579965829849, + "rewards/margins": 1.2468721866607666, + "rewards/rejected": -1.3485580682754517, + "step": 1527 + }, + { + "epoch": 0.9505443234836704, + "grad_norm": 0.49836209416389465, + "learning_rate": 1.18e-05, + "log_odds_chosen": 11.764126777648926, + "log_odds_ratio": -0.043174177408218384, + "logits/chosen": 0.18523219227790833, + "logits/rejected": 0.8043461441993713, + "logps/chosen": -1.0149867534637451, + "logps/rejected": -12.14633846282959, + "loss": 3.1922, + "nll_loss": 3.187835931777954, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10149867087602615, + "rewards/margins": 1.1131352186203003, + "rewards/rejected": -1.2146339416503906, + "step": 1528 + }, + { + "epoch": 0.9511664074650078, + "grad_norm": 2.499340295791626, + "learning_rate": 1.1775e-05, + "log_odds_chosen": 14.100419998168945, + "log_odds_ratio": -0.029255589470267296, + "logits/chosen": 0.21268950402736664, + "logits/rejected": 0.9389889240264893, + "logps/chosen": -1.1754329204559326, + "logps/rejected": -14.794118881225586, + "loss": 3.0132, + "nll_loss": 3.0102486610412598, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1175433024764061, + "rewards/margins": 1.3618686199188232, + "rewards/rejected": -1.4794118404388428, + "step": 1529 + }, + { + "epoch": 0.9517884914463453, + "grad_norm": 0.5740556716918945, + "learning_rate": 1.175e-05, + "log_odds_chosen": 9.988779067993164, + "log_odds_ratio": -0.02954120934009552, + "logits/chosen": 0.12227550148963928, + "logits/rejected": 0.5920629501342773, + "logps/chosen": -1.3234734535217285, + "logps/rejected": -10.88769817352295, + "loss": 3.0954, + "nll_loss": 3.0924153327941895, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13234734535217285, + "rewards/margins": 0.9564225077629089, + "rewards/rejected": -1.0887699127197266, + "step": 1530 + }, + { + "epoch": 0.9524105754276827, + "grad_norm": 0.3808734714984894, + "learning_rate": 1.1725e-05, + "log_odds_chosen": 13.617674827575684, + "log_odds_ratio": -0.0005064024007879198, + "logits/chosen": 0.14402905106544495, + "logits/rejected": 0.8458138704299927, + "logps/chosen": -1.1474123001098633, + "logps/rejected": -14.359789848327637, + "loss": 2.9651, + "nll_loss": 2.9650535583496094, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11474122852087021, + "rewards/margins": 1.3212378025054932, + "rewards/rejected": -1.4359790086746216, + "step": 1531 + }, + { + "epoch": 0.9530326594090203, + "grad_norm": 0.594470739364624, + "learning_rate": 1.1700000000000001e-05, + "log_odds_chosen": 10.063478469848633, + "log_odds_ratio": -0.008701215498149395, + "logits/chosen": 0.2576248049736023, + "logits/rejected": 1.087363600730896, + "logps/chosen": -1.083966851234436, + "logps/rejected": -10.723621368408203, + "loss": 2.9443, + "nll_loss": 2.9434709548950195, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10839669406414032, + "rewards/margins": 0.9639653563499451, + "rewards/rejected": -1.0723621845245361, + "step": 1532 + }, + { + "epoch": 0.9536547433903577, + "grad_norm": 0.38040781021118164, + "learning_rate": 1.1675000000000001e-05, + "log_odds_chosen": 7.64688777923584, + "log_odds_ratio": -0.33788639307022095, + "logits/chosen": 0.09052865207195282, + "logits/rejected": 0.3300938010215759, + "logps/chosen": -1.3858916759490967, + "logps/rejected": -8.915044784545898, + "loss": 3.3453, + "nll_loss": 3.311525344848633, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13858917355537415, + "rewards/margins": 0.7529152631759644, + "rewards/rejected": -0.8915044665336609, + "step": 1533 + }, + { + "epoch": 0.9542768273716952, + "grad_norm": 0.540230929851532, + "learning_rate": 1.1650000000000002e-05, + "log_odds_chosen": 10.206024169921875, + "log_odds_ratio": -0.10539490729570389, + "logits/chosen": 0.17809538543224335, + "logits/rejected": 0.9211235642433167, + "logps/chosen": -1.1614084243774414, + "logps/rejected": -11.00497055053711, + "loss": 2.933, + "nll_loss": 2.922435760498047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11614084243774414, + "rewards/margins": 0.984356164932251, + "rewards/rejected": -1.1004970073699951, + "step": 1534 + }, + { + "epoch": 0.9548989113530326, + "grad_norm": 0.5923968553543091, + "learning_rate": 1.1625000000000001e-05, + "log_odds_chosen": 8.06085205078125, + "log_odds_ratio": -0.10900537669658661, + "logits/chosen": 0.09346047043800354, + "logits/rejected": 0.6258691549301147, + "logps/chosen": -1.2285250425338745, + "logps/rejected": -8.991416931152344, + "loss": 2.5072, + "nll_loss": 2.4963459968566895, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12285250425338745, + "rewards/margins": 0.7762892842292786, + "rewards/rejected": -0.899141788482666, + "step": 1535 + }, + { + "epoch": 0.9555209953343702, + "grad_norm": 0.576630711555481, + "learning_rate": 1.16e-05, + "log_odds_chosen": 11.273231506347656, + "log_odds_ratio": -0.01995195262134075, + "logits/chosen": 0.1091519147157669, + "logits/rejected": 0.8646926879882812, + "logps/chosen": -1.2671400308609009, + "logps/rejected": -12.117172241210938, + "loss": 3.0024, + "nll_loss": 3.0004241466522217, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12671400606632233, + "rewards/margins": 1.085003137588501, + "rewards/rejected": -1.211717128753662, + "step": 1536 + }, + { + "epoch": 0.9561430793157076, + "grad_norm": 0.5668447613716125, + "learning_rate": 1.1575000000000002e-05, + "log_odds_chosen": 8.51119327545166, + "log_odds_ratio": -0.1863744705915451, + "logits/chosen": 0.22063475847244263, + "logits/rejected": 1.0275760889053345, + "logps/chosen": -1.0437042713165283, + "logps/rejected": -9.248046875, + "loss": 2.7686, + "nll_loss": 2.7500054836273193, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10437043011188507, + "rewards/margins": 0.8204342722892761, + "rewards/rejected": -0.9248047471046448, + "step": 1537 + }, + { + "epoch": 0.9567651632970451, + "grad_norm": 0.7183383703231812, + "learning_rate": 1.1550000000000001e-05, + "log_odds_chosen": 9.048027038574219, + "log_odds_ratio": -0.18893525004386902, + "logits/chosen": 0.14682061970233917, + "logits/rejected": 0.5823942422866821, + "logps/chosen": -1.291245937347412, + "logps/rejected": -10.022661209106445, + "loss": 3.2216, + "nll_loss": 3.2026634216308594, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12912458181381226, + "rewards/margins": 0.8731414675712585, + "rewards/rejected": -1.0022661685943604, + "step": 1538 + }, + { + "epoch": 0.9573872472783825, + "grad_norm": 0.7190625071525574, + "learning_rate": 1.1525e-05, + "log_odds_chosen": 11.10485553741455, + "log_odds_ratio": -0.03244490176439285, + "logits/chosen": 0.24327440559864044, + "logits/rejected": 0.8990823030471802, + "logps/chosen": -1.2052109241485596, + "logps/rejected": -11.896177291870117, + "loss": 2.7229, + "nll_loss": 2.7196097373962402, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12052108347415924, + "rewards/margins": 1.0690966844558716, + "rewards/rejected": -1.1896177530288696, + "step": 1539 + }, + { + "epoch": 0.9580093312597201, + "grad_norm": 0.5160905718803406, + "learning_rate": 1.1500000000000002e-05, + "log_odds_chosen": 14.448993682861328, + "log_odds_ratio": -1.6958052583504468e-05, + "logits/chosen": 0.31216803193092346, + "logits/rejected": 1.264636516571045, + "logps/chosen": -1.1925134658813477, + "logps/rejected": -15.253753662109375, + "loss": 3.3065, + "nll_loss": 3.306514263153076, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11925134807825089, + "rewards/margins": 1.4061239957809448, + "rewards/rejected": -1.5253753662109375, + "step": 1540 + }, + { + "epoch": 0.9586314152410575, + "grad_norm": 0.43151193857192993, + "learning_rate": 1.1475000000000001e-05, + "log_odds_chosen": 10.120941162109375, + "log_odds_ratio": -0.06145137920975685, + "logits/chosen": 0.107419952750206, + "logits/rejected": 0.42087522149086, + "logps/chosen": -1.298701286315918, + "logps/rejected": -10.87364673614502, + "loss": 3.4746, + "nll_loss": 3.46848464012146, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12987013161182404, + "rewards/margins": 0.9574945569038391, + "rewards/rejected": -1.087364673614502, + "step": 1541 + }, + { + "epoch": 0.959253499222395, + "grad_norm": 0.513953447341919, + "learning_rate": 1.145e-05, + "log_odds_chosen": 7.542330265045166, + "log_odds_ratio": -0.3065045475959778, + "logits/chosen": 0.23044796288013458, + "logits/rejected": 0.6118386387825012, + "logps/chosen": -1.169670820236206, + "logps/rejected": -8.212411880493164, + "loss": 3.0168, + "nll_loss": 2.9861817359924316, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11696707457304001, + "rewards/margins": 0.7042741179466248, + "rewards/rejected": -0.8212412595748901, + "step": 1542 + }, + { + "epoch": 0.9598755832037325, + "grad_norm": 0.4111732840538025, + "learning_rate": 1.1425000000000002e-05, + "log_odds_chosen": 11.027361869812012, + "log_odds_ratio": -0.09346923232078552, + "logits/chosen": 0.35561326146125793, + "logits/rejected": 1.1297205686569214, + "logps/chosen": -1.0780565738677979, + "logps/rejected": -11.643434524536133, + "loss": 3.6231, + "nll_loss": 3.613704204559326, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10780565440654755, + "rewards/margins": 1.0565377473831177, + "rewards/rejected": -1.1643434762954712, + "step": 1543 + }, + { + "epoch": 0.96049766718507, + "grad_norm": 0.5090989470481873, + "learning_rate": 1.1400000000000001e-05, + "log_odds_chosen": 4.659493446350098, + "log_odds_ratio": -0.24689598381519318, + "logits/chosen": 0.09714489430189133, + "logits/rejected": 0.3079416751861572, + "logps/chosen": -1.0464566946029663, + "logps/rejected": -5.345902442932129, + "loss": 2.9323, + "nll_loss": 2.9075896739959717, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10464566946029663, + "rewards/margins": 0.42994457483291626, + "rewards/rejected": -0.5345902442932129, + "step": 1544 + }, + { + "epoch": 0.9611197511664075, + "grad_norm": 0.6913397312164307, + "learning_rate": 1.1375e-05, + "log_odds_chosen": 12.19678020477295, + "log_odds_ratio": -0.07039018720388412, + "logits/chosen": 0.13496407866477966, + "logits/rejected": 0.8670356273651123, + "logps/chosen": -0.9649935960769653, + "logps/rejected": -12.593484878540039, + "loss": 2.7044, + "nll_loss": 2.6974096298217773, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09649936109781265, + "rewards/margins": 1.1628491878509521, + "rewards/rejected": -1.2593486309051514, + "step": 1545 + }, + { + "epoch": 0.9617418351477449, + "grad_norm": 0.6813112497329712, + "learning_rate": 1.1350000000000001e-05, + "log_odds_chosen": 11.80770492553711, + "log_odds_ratio": -0.06004830822348595, + "logits/chosen": 0.1646079421043396, + "logits/rejected": 0.7866946458816528, + "logps/chosen": -1.2272096872329712, + "logps/rejected": -12.65283203125, + "loss": 3.0013, + "nll_loss": 2.9952850341796875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12272097170352936, + "rewards/margins": 1.1425621509552002, + "rewards/rejected": -1.265283226966858, + "step": 1546 + }, + { + "epoch": 0.9623639191290825, + "grad_norm": 0.5108241438865662, + "learning_rate": 1.1325e-05, + "log_odds_chosen": 5.958439826965332, + "log_odds_ratio": -0.4140869081020355, + "logits/chosen": 0.26702117919921875, + "logits/rejected": 0.5210127234458923, + "logps/chosen": -1.2928879261016846, + "logps/rejected": -6.994579315185547, + "loss": 3.3799, + "nll_loss": 3.3384571075439453, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12928879261016846, + "rewards/margins": 0.5701691508293152, + "rewards/rejected": -0.6994579434394836, + "step": 1547 + }, + { + "epoch": 0.9629860031104199, + "grad_norm": 0.5286838412284851, + "learning_rate": 1.13e-05, + "log_odds_chosen": 14.024574279785156, + "log_odds_ratio": -0.00039988019852899015, + "logits/chosen": 0.3011244535446167, + "logits/rejected": 1.2122119665145874, + "logps/chosen": -1.0042245388031006, + "logps/rejected": -14.499638557434082, + "loss": 3.0475, + "nll_loss": 3.0474681854248047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1004224568605423, + "rewards/margins": 1.349541425704956, + "rewards/rejected": -1.4499640464782715, + "step": 1548 + }, + { + "epoch": 0.9636080870917574, + "grad_norm": 0.4677494168281555, + "learning_rate": 1.1275000000000001e-05, + "log_odds_chosen": 9.42393970489502, + "log_odds_ratio": -0.16956661641597748, + "logits/chosen": 0.11604233086109161, + "logits/rejected": 0.7357518076896667, + "logps/chosen": -1.2022645473480225, + "logps/rejected": -10.252437591552734, + "loss": 2.8647, + "nll_loss": 2.847747802734375, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12022645771503448, + "rewards/margins": 0.9050173759460449, + "rewards/rejected": -1.0252437591552734, + "step": 1549 + }, + { + "epoch": 0.9642301710730948, + "grad_norm": 0.5168280601501465, + "learning_rate": 1.125e-05, + "log_odds_chosen": 11.79471206665039, + "log_odds_ratio": -0.08363199234008789, + "logits/chosen": 0.3251190185546875, + "logits/rejected": 1.2006672620773315, + "logps/chosen": -1.3963546752929688, + "logps/rejected": -12.919671058654785, + "loss": 3.4929, + "nll_loss": 3.484511137008667, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13963547348976135, + "rewards/margins": 1.1523315906524658, + "rewards/rejected": -1.2919670343399048, + "step": 1550 + }, + { + "epoch": 0.9648522550544324, + "grad_norm": 0.6232737898826599, + "learning_rate": 1.1225e-05, + "log_odds_chosen": 12.68300724029541, + "log_odds_ratio": -0.0002065250009763986, + "logits/chosen": 0.12053397297859192, + "logits/rejected": 0.9833630323410034, + "logps/chosen": -1.0209791660308838, + "logps/rejected": -13.13711166381836, + "loss": 2.3546, + "nll_loss": 2.3545565605163574, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10209790617227554, + "rewards/margins": 1.2116132974624634, + "rewards/rejected": -1.313711166381836, + "step": 1551 + }, + { + "epoch": 0.9654743390357698, + "grad_norm": 1.0204579830169678, + "learning_rate": 1.1200000000000001e-05, + "log_odds_chosen": 13.328527450561523, + "log_odds_ratio": -0.2011832892894745, + "logits/chosen": 0.3819844722747803, + "logits/rejected": 1.1638801097869873, + "logps/chosen": -1.0102115869522095, + "logps/rejected": -13.841870307922363, + "loss": 3.1434, + "nll_loss": 3.12326717376709, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10102115571498871, + "rewards/margins": 1.2831659317016602, + "rewards/rejected": -1.3841872215270996, + "step": 1552 + }, + { + "epoch": 0.9660964230171073, + "grad_norm": 3.3021206855773926, + "learning_rate": 1.1175e-05, + "log_odds_chosen": 14.08477783203125, + "log_odds_ratio": -0.26379579305648804, + "logits/chosen": 0.2694425880908966, + "logits/rejected": 1.4157636165618896, + "logps/chosen": -1.4726241827011108, + "logps/rejected": -15.221651077270508, + "loss": 2.3611, + "nll_loss": 2.3346829414367676, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14726242423057556, + "rewards/margins": 1.3749027252197266, + "rewards/rejected": -1.5221651792526245, + "step": 1553 + }, + { + "epoch": 0.9667185069984447, + "grad_norm": 0.6348780393600464, + "learning_rate": 1.115e-05, + "log_odds_chosen": 11.649886131286621, + "log_odds_ratio": -0.09239879250526428, + "logits/chosen": 0.21169120073318481, + "logits/rejected": 0.7421719431877136, + "logps/chosen": -1.2626230716705322, + "logps/rejected": -12.575725555419922, + "loss": 2.7963, + "nll_loss": 2.7870941162109375, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12626230716705322, + "rewards/margins": 1.131310224533081, + "rewards/rejected": -1.2575724124908447, + "step": 1554 + }, + { + "epoch": 0.9673405909797823, + "grad_norm": 0.572074830532074, + "learning_rate": 1.1125000000000001e-05, + "log_odds_chosen": 10.015185356140137, + "log_odds_ratio": -0.017020372673869133, + "logits/chosen": 0.1622290313243866, + "logits/rejected": 0.8837634325027466, + "logps/chosen": -1.1480190753936768, + "logps/rejected": -10.764152526855469, + "loss": 2.786, + "nll_loss": 2.7842512130737305, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11480191349983215, + "rewards/margins": 0.9616133570671082, + "rewards/rejected": -1.0764153003692627, + "step": 1555 + }, + { + "epoch": 0.9679626749611198, + "grad_norm": 0.5456128120422363, + "learning_rate": 1.11e-05, + "log_odds_chosen": 15.245088577270508, + "log_odds_ratio": -0.00036605659988708794, + "logits/chosen": 0.27020353078842163, + "logits/rejected": 1.1084659099578857, + "logps/chosen": -1.1513752937316895, + "logps/rejected": -15.948476791381836, + "loss": 2.9191, + "nll_loss": 2.9190452098846436, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11513753235340118, + "rewards/margins": 1.4797101020812988, + "rewards/rejected": -1.5948476791381836, + "step": 1556 + }, + { + "epoch": 0.9685847589424572, + "grad_norm": 0.6812815070152283, + "learning_rate": 1.1075e-05, + "log_odds_chosen": 9.595189094543457, + "log_odds_ratio": -0.07183755934238434, + "logits/chosen": 0.3203434646129608, + "logits/rejected": 0.9297034740447998, + "logps/chosen": -1.0930052995681763, + "logps/rejected": -10.203285217285156, + "loss": 3.3591, + "nll_loss": 3.3519504070281982, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10930053144693375, + "rewards/margins": 0.9110280275344849, + "rewards/rejected": -1.0203286409378052, + "step": 1557 + }, + { + "epoch": 0.9692068429237947, + "grad_norm": 0.4690392315387726, + "learning_rate": 1.1050000000000001e-05, + "log_odds_chosen": 5.883632659912109, + "log_odds_ratio": -0.3575781583786011, + "logits/chosen": 0.06924352049827576, + "logits/rejected": 0.5914300680160522, + "logps/chosen": -1.129926323890686, + "logps/rejected": -6.685489177703857, + "loss": 2.4541, + "nll_loss": 2.4183237552642822, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11299263685941696, + "rewards/margins": 0.5555562973022461, + "rewards/rejected": -0.6685488820075989, + "step": 1558 + }, + { + "epoch": 0.9698289269051322, + "grad_norm": 0.4620547890663147, + "learning_rate": 1.1025e-05, + "log_odds_chosen": 15.391680717468262, + "log_odds_ratio": -0.07981463521718979, + "logits/chosen": 0.2869001030921936, + "logits/rejected": 1.361629843711853, + "logps/chosen": -1.096318006515503, + "logps/rejected": -16.13008689880371, + "loss": 3.1539, + "nll_loss": 3.145958185195923, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10963179171085358, + "rewards/margins": 1.5033769607543945, + "rewards/rejected": -1.6130086183547974, + "step": 1559 + }, + { + "epoch": 0.9704510108864697, + "grad_norm": 0.517099916934967, + "learning_rate": 1.1000000000000001e-05, + "log_odds_chosen": 11.813902854919434, + "log_odds_ratio": -0.07760776579380035, + "logits/chosen": 0.2125309407711029, + "logits/rejected": 1.057563304901123, + "logps/chosen": -1.2820537090301514, + "logps/rejected": -12.755212783813477, + "loss": 2.7792, + "nll_loss": 2.7714877128601074, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12820537388324738, + "rewards/margins": 1.1473159790039062, + "rewards/rejected": -1.2755212783813477, + "step": 1560 + }, + { + "epoch": 0.9710730948678071, + "grad_norm": 0.5570861101150513, + "learning_rate": 1.0975e-05, + "log_odds_chosen": 8.667329788208008, + "log_odds_ratio": -0.28669866919517517, + "logits/chosen": 0.26910680532455444, + "logits/rejected": 0.5824862718582153, + "logps/chosen": -1.1450307369232178, + "logps/rejected": -9.458759307861328, + "loss": 3.2066, + "nll_loss": 3.177957057952881, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11450307071208954, + "rewards/margins": 0.831372857093811, + "rewards/rejected": -0.945875883102417, + "step": 1561 + }, + { + "epoch": 0.9716951788491446, + "grad_norm": 0.6233094930648804, + "learning_rate": 1.095e-05, + "log_odds_chosen": 12.726680755615234, + "log_odds_ratio": -0.0005210311501286924, + "logits/chosen": 0.2549572288990021, + "logits/rejected": 1.0368318557739258, + "logps/chosen": -1.2574390172958374, + "logps/rejected": -13.563409805297852, + "loss": 2.9092, + "nll_loss": 2.909170627593994, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12574389576911926, + "rewards/margins": 1.2305971384048462, + "rewards/rejected": -1.356341004371643, + "step": 1562 + }, + { + "epoch": 0.9723172628304821, + "grad_norm": 7.476173400878906, + "learning_rate": 1.0925000000000001e-05, + "log_odds_chosen": 7.137988567352295, + "log_odds_ratio": -0.15849797427654266, + "logits/chosen": 0.4429250657558441, + "logits/rejected": 1.059818983078003, + "logps/chosen": -1.839691400527954, + "logps/rejected": -8.487317085266113, + "loss": 3.1695, + "nll_loss": 3.1536335945129395, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1839691400527954, + "rewards/margins": 0.6647626161575317, + "rewards/rejected": -0.8487317562103271, + "step": 1563 + }, + { + "epoch": 0.9729393468118196, + "grad_norm": 1.7156760692596436, + "learning_rate": 1.09e-05, + "log_odds_chosen": 10.09739875793457, + "log_odds_ratio": -0.14037665724754333, + "logits/chosen": 0.16592925786972046, + "logits/rejected": 0.832611083984375, + "logps/chosen": -0.9634124636650085, + "logps/rejected": -10.481101036071777, + "loss": 2.9348, + "nll_loss": 2.920729637145996, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09634124487638474, + "rewards/margins": 0.9517688751220703, + "rewards/rejected": -1.0481101274490356, + "step": 1564 + }, + { + "epoch": 0.973561430793157, + "grad_norm": 0.5390524864196777, + "learning_rate": 1.0875e-05, + "log_odds_chosen": 9.242631912231445, + "log_odds_ratio": -0.10577996075153351, + "logits/chosen": 0.3823816776275635, + "logits/rejected": 0.9803093671798706, + "logps/chosen": -1.0878280401229858, + "logps/rejected": -9.865617752075195, + "loss": 3.7807, + "nll_loss": 3.7701072692871094, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1087828055024147, + "rewards/margins": 0.8777790069580078, + "rewards/rejected": -0.9865618348121643, + "step": 1565 + }, + { + "epoch": 0.9741835147744946, + "grad_norm": 0.5683154463768005, + "learning_rate": 1.0850000000000001e-05, + "log_odds_chosen": 10.887935638427734, + "log_odds_ratio": -0.12982094287872314, + "logits/chosen": 0.12701228260993958, + "logits/rejected": 0.630905270576477, + "logps/chosen": -0.969216525554657, + "logps/rejected": -11.376995086669922, + "loss": 2.6706, + "nll_loss": 2.657618284225464, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0969216525554657, + "rewards/margins": 1.0407779216766357, + "rewards/rejected": -1.1376994848251343, + "step": 1566 + }, + { + "epoch": 0.9748055987558321, + "grad_norm": 0.5054221749305725, + "learning_rate": 1.0825e-05, + "log_odds_chosen": 11.538697242736816, + "log_odds_ratio": -0.2012937068939209, + "logits/chosen": 0.23306933045387268, + "logits/rejected": 1.1021910905838013, + "logps/chosen": -1.2251908779144287, + "logps/rejected": -12.437653541564941, + "loss": 2.932, + "nll_loss": 2.911904811859131, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1225190982222557, + "rewards/margins": 1.121246099472046, + "rewards/rejected": -1.2437653541564941, + "step": 1567 + }, + { + "epoch": 0.9754276827371695, + "grad_norm": 0.3712461590766907, + "learning_rate": 1.08e-05, + "log_odds_chosen": 10.93287181854248, + "log_odds_ratio": -0.0026042910758405924, + "logits/chosen": 0.28485745191574097, + "logits/rejected": 0.8299000263214111, + "logps/chosen": -1.1558926105499268, + "logps/rejected": -11.693628311157227, + "loss": 3.2769, + "nll_loss": 3.276618480682373, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11558926105499268, + "rewards/margins": 1.0537735223770142, + "rewards/rejected": -1.1693627834320068, + "step": 1568 + }, + { + "epoch": 0.976049766718507, + "grad_norm": 0.468656986951828, + "learning_rate": 1.0775000000000001e-05, + "log_odds_chosen": 8.820526123046875, + "log_odds_ratio": -0.1020168736577034, + "logits/chosen": 0.27156370878219604, + "logits/rejected": 0.9939742684364319, + "logps/chosen": -1.0126519203186035, + "logps/rejected": -9.478734016418457, + "loss": 3.0389, + "nll_loss": 3.0287251472473145, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10126519948244095, + "rewards/margins": 0.8466081619262695, + "rewards/rejected": -0.9478734135627747, + "step": 1569 + }, + { + "epoch": 0.9766718506998445, + "grad_norm": 0.5117287635803223, + "learning_rate": 1.075e-05, + "log_odds_chosen": 6.064488410949707, + "log_odds_ratio": -0.2216080129146576, + "logits/chosen": 0.32926154136657715, + "logits/rejected": 0.6299622654914856, + "logps/chosen": -0.9756649136543274, + "logps/rejected": -6.43527364730835, + "loss": 3.2928, + "nll_loss": 3.2706661224365234, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09756650030612946, + "rewards/margins": 0.5459609031677246, + "rewards/rejected": -0.6435273885726929, + "step": 1570 + }, + { + "epoch": 0.977293934681182, + "grad_norm": 0.4976355731487274, + "learning_rate": 1.0725e-05, + "log_odds_chosen": 8.256917953491211, + "log_odds_ratio": -0.1852550506591797, + "logits/chosen": 0.40325844287872314, + "logits/rejected": 0.9676214456558228, + "logps/chosen": -1.2901231050491333, + "logps/rejected": -9.301626205444336, + "loss": 3.2676, + "nll_loss": 3.249091386795044, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1290123164653778, + "rewards/margins": 0.8011503219604492, + "rewards/rejected": -0.9301626682281494, + "step": 1571 + }, + { + "epoch": 0.9779160186625194, + "grad_norm": 0.8282569646835327, + "learning_rate": 1.0700000000000001e-05, + "log_odds_chosen": 9.258617401123047, + "log_odds_ratio": -0.026124443858861923, + "logits/chosen": 0.21136893332004547, + "logits/rejected": 0.524401068687439, + "logps/chosen": -0.92659592628479, + "logps/rejected": -9.56185245513916, + "loss": 2.7815, + "nll_loss": 2.77886700630188, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0926596000790596, + "rewards/margins": 0.8635256886482239, + "rewards/rejected": -0.9561852812767029, + "step": 1572 + }, + { + "epoch": 0.9785381026438569, + "grad_norm": 0.4185151159763336, + "learning_rate": 1.0675e-05, + "log_odds_chosen": 13.6192045211792, + "log_odds_ratio": -0.1079566702246666, + "logits/chosen": 0.3801066279411316, + "logits/rejected": 1.0820777416229248, + "logps/chosen": -1.1457598209381104, + "logps/rejected": -14.35219669342041, + "loss": 3.5128, + "nll_loss": 3.5020086765289307, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11457598209381104, + "rewards/margins": 1.320643663406372, + "rewards/rejected": -1.4352197647094727, + "step": 1573 + }, + { + "epoch": 0.9791601866251944, + "grad_norm": 0.5058881640434265, + "learning_rate": 1.065e-05, + "log_odds_chosen": 10.972328186035156, + "log_odds_ratio": -0.002450345316901803, + "logits/chosen": 0.11459196358919144, + "logits/rejected": 0.7418378591537476, + "logps/chosen": -1.306053638458252, + "logps/rejected": -11.894512176513672, + "loss": 2.7975, + "nll_loss": 2.7972939014434814, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13060535490512848, + "rewards/margins": 1.0588459968566895, + "rewards/rejected": -1.1894512176513672, + "step": 1574 + }, + { + "epoch": 0.9797822706065319, + "grad_norm": 0.5295330882072449, + "learning_rate": 1.0625e-05, + "log_odds_chosen": 10.171019554138184, + "log_odds_ratio": -0.1479872316122055, + "logits/chosen": 0.3536169230937958, + "logits/rejected": 0.7485109567642212, + "logps/chosen": -1.1546906232833862, + "logps/rejected": -10.923442840576172, + "loss": 3.8514, + "nll_loss": 3.8366503715515137, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1154690533876419, + "rewards/margins": 0.9768753051757812, + "rewards/rejected": -1.0923444032669067, + "step": 1575 + }, + { + "epoch": 0.9804043545878693, + "grad_norm": 0.4251069724559784, + "learning_rate": 1.06e-05, + "log_odds_chosen": 12.614507675170898, + "log_odds_ratio": -0.007318359334021807, + "logits/chosen": 0.33328309655189514, + "logits/rejected": 1.0371602773666382, + "logps/chosen": -1.0035243034362793, + "logps/rejected": -13.143770217895508, + "loss": 3.0478, + "nll_loss": 3.047060489654541, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.100352443754673, + "rewards/margins": 1.214024543762207, + "rewards/rejected": -1.3143770694732666, + "step": 1576 + }, + { + "epoch": 0.9810264385692068, + "grad_norm": 0.7450897693634033, + "learning_rate": 1.0575e-05, + "log_odds_chosen": 12.139310836791992, + "log_odds_ratio": -0.014527878724038601, + "logits/chosen": 0.2485947608947754, + "logits/rejected": 0.7911143898963928, + "logps/chosen": -1.5825238227844238, + "logps/rejected": -13.46920394897461, + "loss": 3.0352, + "nll_loss": 3.0337603092193604, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15825238823890686, + "rewards/margins": 1.1886680126190186, + "rewards/rejected": -1.346920371055603, + "step": 1577 + }, + { + "epoch": 0.9816485225505444, + "grad_norm": 0.4823170602321625, + "learning_rate": 1.055e-05, + "log_odds_chosen": 10.599369049072266, + "log_odds_ratio": -0.01910674385726452, + "logits/chosen": 0.18403150141239166, + "logits/rejected": 0.7430771589279175, + "logps/chosen": -0.907859742641449, + "logps/rejected": -10.833913803100586, + "loss": 2.9747, + "nll_loss": 2.972804546356201, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09078596532344818, + "rewards/margins": 0.9926053881645203, + "rewards/rejected": -1.0833913087844849, + "step": 1578 + }, + { + "epoch": 0.9822706065318818, + "grad_norm": 0.5032903552055359, + "learning_rate": 1.0525e-05, + "log_odds_chosen": 11.860549926757812, + "log_odds_ratio": -0.000851090473588556, + "logits/chosen": 0.2981140613555908, + "logits/rejected": 1.1328045129776, + "logps/chosen": -1.4232183694839478, + "logps/rejected": -12.965503692626953, + "loss": 3.3613, + "nll_loss": 3.3611791133880615, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14232183992862701, + "rewards/margins": 1.1542284488677979, + "rewards/rejected": -1.2965502738952637, + "step": 1579 + }, + { + "epoch": 0.9828926905132193, + "grad_norm": 0.42979589104652405, + "learning_rate": 1.05e-05, + "log_odds_chosen": 11.368293762207031, + "log_odds_ratio": -0.0016658775275573134, + "logits/chosen": 0.2528938949108124, + "logits/rejected": 0.9410756826400757, + "logps/chosen": -1.0155589580535889, + "logps/rejected": -11.831025123596191, + "loss": 3.2585, + "nll_loss": 3.258355140686035, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10155589878559113, + "rewards/margins": 1.081546664237976, + "rewards/rejected": -1.1831024885177612, + "step": 1580 + }, + { + "epoch": 0.9835147744945568, + "grad_norm": 0.7552335858345032, + "learning_rate": 1.0475e-05, + "log_odds_chosen": 8.832204818725586, + "log_odds_ratio": -0.027881065383553505, + "logits/chosen": 0.3224768042564392, + "logits/rejected": 0.8275183439254761, + "logps/chosen": -1.3035496473312378, + "logps/rejected": -9.811277389526367, + "loss": 3.2479, + "nll_loss": 3.245144844055176, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13035497069358826, + "rewards/margins": 0.8507728576660156, + "rewards/rejected": -0.9811277389526367, + "step": 1581 + }, + { + "epoch": 0.9841368584758943, + "grad_norm": 0.4854262173175812, + "learning_rate": 1.045e-05, + "log_odds_chosen": 7.1624650955200195, + "log_odds_ratio": -0.15542778372764587, + "logits/chosen": 0.3523561358451843, + "logits/rejected": 0.9019551277160645, + "logps/chosen": -0.9577276110649109, + "logps/rejected": -7.694332599639893, + "loss": 2.7196, + "nll_loss": 2.7040822505950928, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09577276557683945, + "rewards/margins": 0.6736605763435364, + "rewards/rejected": -0.7694332599639893, + "step": 1582 + }, + { + "epoch": 0.9847589424572317, + "grad_norm": 0.48357895016670227, + "learning_rate": 1.0425e-05, + "log_odds_chosen": 11.84089469909668, + "log_odds_ratio": -0.08281046897172928, + "logits/chosen": 0.2278730422258377, + "logits/rejected": 1.1716654300689697, + "logps/chosen": -1.1539627313613892, + "logps/rejected": -12.60505485534668, + "loss": 2.3173, + "nll_loss": 2.3089795112609863, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11539626121520996, + "rewards/margins": 1.1451092958450317, + "rewards/rejected": -1.2605055570602417, + "step": 1583 + }, + { + "epoch": 0.9853810264385692, + "grad_norm": 0.5850421190261841, + "learning_rate": 1.04e-05, + "log_odds_chosen": 11.96889591217041, + "log_odds_ratio": -0.26381194591522217, + "logits/chosen": 0.2205938845872879, + "logits/rejected": 1.0089349746704102, + "logps/chosen": -1.2542933225631714, + "logps/rejected": -12.704479217529297, + "loss": 2.6266, + "nll_loss": 2.600229501724243, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12542933225631714, + "rewards/margins": 1.1450186967849731, + "rewards/rejected": -1.2704479694366455, + "step": 1584 + }, + { + "epoch": 0.9860031104199067, + "grad_norm": 0.5196158289909363, + "learning_rate": 1.0375e-05, + "log_odds_chosen": 13.077522277832031, + "log_odds_ratio": -0.0018111845711246133, + "logits/chosen": 0.3096368610858917, + "logits/rejected": 1.080397129058838, + "logps/chosen": -1.0235284566879272, + "logps/rejected": -13.350191116333008, + "loss": 2.9722, + "nll_loss": 2.972025156021118, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10235285013914108, + "rewards/margins": 1.232666254043579, + "rewards/rejected": -1.3350191116333008, + "step": 1585 + }, + { + "epoch": 0.9866251944012442, + "grad_norm": 0.5081434845924377, + "learning_rate": 1.035e-05, + "log_odds_chosen": 10.091049194335938, + "log_odds_ratio": -0.05019140988588333, + "logits/chosen": 0.17959453165531158, + "logits/rejected": 0.8621308207511902, + "logps/chosen": -1.0767017602920532, + "logps/rejected": -10.69836139678955, + "loss": 2.9533, + "nll_loss": 2.9482641220092773, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10767018049955368, + "rewards/margins": 0.9621660113334656, + "rewards/rejected": -1.069836139678955, + "step": 1586 + }, + { + "epoch": 0.9872472783825816, + "grad_norm": 1.4921205043792725, + "learning_rate": 1.0325e-05, + "log_odds_chosen": 15.034688949584961, + "log_odds_ratio": -5.1558799896156415e-06, + "logits/chosen": 0.35980913043022156, + "logits/rejected": 1.0580722093582153, + "logps/chosen": -1.868407130241394, + "logps/rejected": -16.496301651000977, + "loss": 3.3692, + "nll_loss": 3.3691837787628174, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1868406981229782, + "rewards/margins": 1.462789535522461, + "rewards/rejected": -1.6496303081512451, + "step": 1587 + }, + { + "epoch": 0.9878693623639191, + "grad_norm": 1.2328771352767944, + "learning_rate": 1.03e-05, + "log_odds_chosen": 12.517577171325684, + "log_odds_ratio": -0.020710619166493416, + "logits/chosen": 0.340537965297699, + "logits/rejected": 1.3173099756240845, + "logps/chosen": -0.9290058016777039, + "logps/rejected": -12.619564056396484, + "loss": 3.0733, + "nll_loss": 3.0711874961853027, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0929005891084671, + "rewards/margins": 1.1690558195114136, + "rewards/rejected": -1.2619564533233643, + "step": 1588 + }, + { + "epoch": 0.9884914463452567, + "grad_norm": 0.639462411403656, + "learning_rate": 1.0275e-05, + "log_odds_chosen": 14.48855209350586, + "log_odds_ratio": -3.144158654322382e-06, + "logits/chosen": 0.29837891459465027, + "logits/rejected": 1.3534728288650513, + "logps/chosen": -0.8880980014801025, + "logps/rejected": -14.57780933380127, + "loss": 2.6912, + "nll_loss": 2.691150188446045, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08880980312824249, + "rewards/margins": 1.3689711093902588, + "rewards/rejected": -1.4577808380126953, + "step": 1589 + }, + { + "epoch": 0.9891135303265941, + "grad_norm": 0.5558017492294312, + "learning_rate": 1.025e-05, + "log_odds_chosen": 11.420799255371094, + "log_odds_ratio": -0.1718229502439499, + "logits/chosen": 0.4150314927101135, + "logits/rejected": 1.6155272722244263, + "logps/chosen": -1.1975841522216797, + "logps/rejected": -12.336982727050781, + "loss": 2.9496, + "nll_loss": 2.932443380355835, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11975841224193573, + "rewards/margins": 1.1139397621154785, + "rewards/rejected": -1.2336982488632202, + "step": 1590 + }, + { + "epoch": 0.9897356143079316, + "grad_norm": 0.4650667905807495, + "learning_rate": 1.0225e-05, + "log_odds_chosen": 11.10346508026123, + "log_odds_ratio": -0.0070058186538517475, + "logits/chosen": 0.4163108170032501, + "logits/rejected": 1.0532922744750977, + "logps/chosen": -1.1275982856750488, + "logps/rejected": -11.808972358703613, + "loss": 3.3, + "nll_loss": 3.2992758750915527, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11275982111692429, + "rewards/margins": 1.0681374073028564, + "rewards/rejected": -1.1808972358703613, + "step": 1591 + }, + { + "epoch": 0.990357698289269, + "grad_norm": 0.486665278673172, + "learning_rate": 1.02e-05, + "log_odds_chosen": 11.23182487487793, + "log_odds_ratio": -0.010925328359007835, + "logits/chosen": 0.1485457718372345, + "logits/rejected": 0.7938356995582581, + "logps/chosen": -1.0545248985290527, + "logps/rejected": -11.765100479125977, + "loss": 2.9406, + "nll_loss": 2.9395089149475098, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10545249283313751, + "rewards/margins": 1.0710575580596924, + "rewards/rejected": -1.1765100955963135, + "step": 1592 + }, + { + "epoch": 0.9909797822706066, + "grad_norm": 0.7251661419868469, + "learning_rate": 1.0175e-05, + "log_odds_chosen": 9.512269020080566, + "log_odds_ratio": -0.2862478494644165, + "logits/chosen": 0.3005995452404022, + "logits/rejected": 0.9812291264533997, + "logps/chosen": -0.9364309310913086, + "logps/rejected": -10.132664680480957, + "loss": 2.3115, + "nll_loss": 2.282909631729126, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09364309161901474, + "rewards/margins": 0.9196233749389648, + "rewards/rejected": -1.0132665634155273, + "step": 1593 + }, + { + "epoch": 0.991601866251944, + "grad_norm": 0.5410097241401672, + "learning_rate": 1.0150000000000001e-05, + "log_odds_chosen": 9.208051681518555, + "log_odds_ratio": -0.18747776746749878, + "logits/chosen": 0.2776232957839966, + "logits/rejected": 1.036055564880371, + "logps/chosen": -1.1870697736740112, + "logps/rejected": -9.9979887008667, + "loss": 2.9676, + "nll_loss": 2.948817729949951, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11870698630809784, + "rewards/margins": 0.8810918927192688, + "rewards/rejected": -0.9997988939285278, + "step": 1594 + }, + { + "epoch": 0.9922239502332815, + "grad_norm": 0.6100337505340576, + "learning_rate": 1.0125e-05, + "log_odds_chosen": 11.178662300109863, + "log_odds_ratio": -0.02708502486348152, + "logits/chosen": 0.26386797428131104, + "logits/rejected": 0.9037798047065735, + "logps/chosen": -0.9145346879959106, + "logps/rejected": -11.334199905395508, + "loss": 3.0249, + "nll_loss": 3.0221993923187256, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09145347774028778, + "rewards/margins": 1.0419665575027466, + "rewards/rejected": -1.1334201097488403, + "step": 1595 + }, + { + "epoch": 0.9928460342146189, + "grad_norm": 2.953352451324463, + "learning_rate": 1.0100000000000002e-05, + "log_odds_chosen": 9.112907409667969, + "log_odds_ratio": -0.1708381026983261, + "logits/chosen": 0.17483511567115784, + "logits/rejected": 0.6510817408561707, + "logps/chosen": -1.2610094547271729, + "logps/rejected": -10.078710556030273, + "loss": 2.7612, + "nll_loss": 2.744128465652466, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12610095739364624, + "rewards/margins": 0.881770133972168, + "rewards/rejected": -1.007871150970459, + "step": 1596 + }, + { + "epoch": 0.9934681181959565, + "grad_norm": 0.48717230558395386, + "learning_rate": 1.0075000000000001e-05, + "log_odds_chosen": 14.488679885864258, + "log_odds_ratio": -0.0001398213644279167, + "logits/chosen": 0.22382810711860657, + "logits/rejected": 1.1981775760650635, + "logps/chosen": -1.0615004301071167, + "logps/rejected": -14.9384765625, + "loss": 2.774, + "nll_loss": 2.7739906311035156, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10615004599094391, + "rewards/margins": 1.387697696685791, + "rewards/rejected": -1.4938478469848633, + "step": 1597 + }, + { + "epoch": 0.9940902021772939, + "grad_norm": 0.47258996963500977, + "learning_rate": 1.005e-05, + "log_odds_chosen": 9.835807800292969, + "log_odds_ratio": -0.24436575174331665, + "logits/chosen": 0.30663585662841797, + "logits/rejected": 0.9482704401016235, + "logps/chosen": -1.2437496185302734, + "logps/rejected": -10.829856872558594, + "loss": 2.9497, + "nll_loss": 2.9252965450286865, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12437495589256287, + "rewards/margins": 0.9586107730865479, + "rewards/rejected": -1.082985758781433, + "step": 1598 + }, + { + "epoch": 0.9947122861586314, + "grad_norm": 0.5687262415885925, + "learning_rate": 1.0025000000000001e-05, + "log_odds_chosen": 7.890524864196777, + "log_odds_ratio": -0.17712527513504028, + "logits/chosen": 0.2682409882545471, + "logits/rejected": 0.8357047438621521, + "logps/chosen": -1.0649604797363281, + "logps/rejected": -8.51159954071045, + "loss": 2.6346, + "nll_loss": 2.6168601512908936, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10649605840444565, + "rewards/margins": 0.7446639537811279, + "rewards/rejected": -0.8511600494384766, + "step": 1599 + }, + { + "epoch": 0.995334370139969, + "grad_norm": 0.6656171679496765, + "learning_rate": 1e-05, + "log_odds_chosen": 10.36136531829834, + "log_odds_ratio": -0.1039852499961853, + "logits/chosen": 0.3594803810119629, + "logits/rejected": 0.7507673501968384, + "logps/chosen": -0.8331475853919983, + "logps/rejected": -10.611698150634766, + "loss": 3.0173, + "nll_loss": 3.0068767070770264, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08331475406885147, + "rewards/margins": 0.9778551459312439, + "rewards/rejected": -1.0611698627471924, + "step": 1600 + }, + { + "epoch": 0.9959564541213064, + "grad_norm": 0.7820709943771362, + "learning_rate": 9.975e-06, + "log_odds_chosen": 8.559907913208008, + "log_odds_ratio": -0.19893547892570496, + "logits/chosen": 0.32968413829803467, + "logits/rejected": 0.9315425157546997, + "logps/chosen": -1.1777303218841553, + "logps/rejected": -9.3145112991333, + "loss": 3.0425, + "nll_loss": 3.0226306915283203, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11777302622795105, + "rewards/margins": 0.8136780261993408, + "rewards/rejected": -0.931451141834259, + "step": 1601 + }, + { + "epoch": 0.9965785381026439, + "grad_norm": 0.5365874767303467, + "learning_rate": 9.950000000000001e-06, + "log_odds_chosen": 4.336380481719971, + "log_odds_ratio": -0.31836003065109253, + "logits/chosen": -0.07907748222351074, + "logits/rejected": 0.1741887927055359, + "logps/chosen": -1.5812636613845825, + "logps/rejected": -5.768222808837891, + "loss": 2.5041, + "nll_loss": 2.4723105430603027, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1581263691186905, + "rewards/margins": 0.4186958968639374, + "rewards/rejected": -0.5768222808837891, + "step": 1602 + }, + { + "epoch": 0.9972006220839813, + "grad_norm": 0.6687927842140198, + "learning_rate": 9.925e-06, + "log_odds_chosen": 6.069998264312744, + "log_odds_ratio": -0.13015542924404144, + "logits/chosen": 0.20325936377048492, + "logits/rejected": 0.6469485759735107, + "logps/chosen": -1.2751343250274658, + "logps/rejected": -6.99819803237915, + "loss": 2.9466, + "nll_loss": 2.9335503578186035, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12751343846321106, + "rewards/margins": 0.5723063945770264, + "rewards/rejected": -0.699819803237915, + "step": 1603 + }, + { + "epoch": 0.9978227060653189, + "grad_norm": 0.4991750419139862, + "learning_rate": 9.900000000000002e-06, + "log_odds_chosen": 8.110227584838867, + "log_odds_ratio": -0.10920145362615585, + "logits/chosen": 0.1598237156867981, + "logits/rejected": 0.7098408341407776, + "logps/chosen": -1.1333831548690796, + "logps/rejected": -8.875372886657715, + "loss": 3.0029, + "nll_loss": 2.991992950439453, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11333832144737244, + "rewards/margins": 0.7741988897323608, + "rewards/rejected": -0.8875373005867004, + "step": 1604 + }, + { + "epoch": 0.9984447900466563, + "grad_norm": 0.4689289629459381, + "learning_rate": 9.875000000000001e-06, + "log_odds_chosen": 7.205236911773682, + "log_odds_ratio": -0.10720035433769226, + "logits/chosen": 0.09387461841106415, + "logits/rejected": 0.5457128882408142, + "logps/chosen": -1.4958709478378296, + "logps/rejected": -8.467511177062988, + "loss": 3.0375, + "nll_loss": 3.0267648696899414, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14958709478378296, + "rewards/margins": 0.697163999080658, + "rewards/rejected": -0.8467510938644409, + "step": 1605 + }, + { + "epoch": 0.9990668740279938, + "grad_norm": 0.5157873034477234, + "learning_rate": 9.85e-06, + "log_odds_chosen": 10.404451370239258, + "log_odds_ratio": -0.004838836379349232, + "logits/chosen": 0.23798084259033203, + "logits/rejected": 0.7781890034675598, + "logps/chosen": -0.9072043299674988, + "logps/rejected": -10.63504409790039, + "loss": 2.5904, + "nll_loss": 2.5898895263671875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09072043001651764, + "rewards/margins": 0.9727840423583984, + "rewards/rejected": -1.0635044574737549, + "step": 1606 + }, + { + "epoch": 0.9996889580093312, + "grad_norm": 0.45538413524627686, + "learning_rate": 9.825000000000002e-06, + "log_odds_chosen": 13.454652786254883, + "log_odds_ratio": -0.007265198975801468, + "logits/chosen": 0.23318925499916077, + "logits/rejected": 0.9881614446640015, + "logps/chosen": -0.8240679502487183, + "logps/rejected": -13.559259414672852, + "loss": 3.0339, + "nll_loss": 3.033188819885254, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08240679651498795, + "rewards/margins": 1.2735192775726318, + "rewards/rejected": -1.3559260368347168, + "step": 1607 + }, + { + "epoch": 1.0003110419906687, + "grad_norm": 1.1448850631713867, + "learning_rate": 9.800000000000001e-06, + "log_odds_chosen": 12.285888671875, + "log_odds_ratio": -0.0008950508199632168, + "logits/chosen": 0.1304558366537094, + "logits/rejected": 0.48669731616973877, + "logps/chosen": -1.2000110149383545, + "logps/rejected": -12.767168045043945, + "loss": 2.9909, + "nll_loss": 2.9908194541931152, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12000110745429993, + "rewards/margins": 1.1567156314849854, + "rewards/rejected": -1.2767168283462524, + "step": 1608 + }, + { + "epoch": 1.0009331259720062, + "grad_norm": 6.079453945159912, + "learning_rate": 9.775e-06, + "log_odds_chosen": 11.781765937805176, + "log_odds_ratio": -0.006075817160308361, + "logits/chosen": 0.31462544202804565, + "logits/rejected": 0.9736188650131226, + "logps/chosen": -1.4540257453918457, + "logps/rejected": -12.950994491577148, + "loss": 3.3169, + "nll_loss": 3.316250801086426, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14540256559848785, + "rewards/margins": 1.1496968269348145, + "rewards/rejected": -1.2950994968414307, + "step": 1609 + }, + { + "epoch": 1.0015552099533438, + "grad_norm": 0.5772498846054077, + "learning_rate": 9.750000000000002e-06, + "log_odds_chosen": 9.219070434570312, + "log_odds_ratio": -0.03137453272938728, + "logits/chosen": 0.2179422527551651, + "logits/rejected": 0.3960098624229431, + "logps/chosen": -0.7287307977676392, + "logps/rejected": -8.83579158782959, + "loss": 2.9345, + "nll_loss": 2.9313859939575195, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0728730782866478, + "rewards/margins": 0.8107061386108398, + "rewards/rejected": -0.8835792541503906, + "step": 1610 + }, + { + "epoch": 1.0021772939346811, + "grad_norm": 0.5609044432640076, + "learning_rate": 9.725000000000001e-06, + "log_odds_chosen": 11.617526054382324, + "log_odds_ratio": -0.17162840068340302, + "logits/chosen": 0.2871357798576355, + "logits/rejected": 1.2228782176971436, + "logps/chosen": -1.0836530923843384, + "logps/rejected": -12.375179290771484, + "loss": 2.9787, + "nll_loss": 2.96148943901062, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10836531966924667, + "rewards/margins": 1.129152536392212, + "rewards/rejected": -1.2375179529190063, + "step": 1611 + }, + { + "epoch": 1.0027993779160187, + "grad_norm": 0.5066659450531006, + "learning_rate": 9.7e-06, + "log_odds_chosen": 6.8533430099487305, + "log_odds_ratio": -0.28324294090270996, + "logits/chosen": 0.35354509949684143, + "logits/rejected": 0.6426160335540771, + "logps/chosen": -1.046383023262024, + "logps/rejected": -7.608695983886719, + "loss": 3.3284, + "nll_loss": 3.3001086711883545, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10463830083608627, + "rewards/margins": 0.6562313437461853, + "rewards/rejected": -0.7608696222305298, + "step": 1612 + }, + { + "epoch": 1.003421461897356, + "grad_norm": 0.5004162192344666, + "learning_rate": 9.675000000000001e-06, + "log_odds_chosen": 11.484716415405273, + "log_odds_ratio": -0.09374061226844788, + "logits/chosen": 0.2518472671508789, + "logits/rejected": 0.6501962542533875, + "logps/chosen": -0.8462555408477783, + "logps/rejected": -11.751243591308594, + "loss": 3.2331, + "nll_loss": 3.223677158355713, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08462555706501007, + "rewards/margins": 1.090498685836792, + "rewards/rejected": -1.1751242876052856, + "step": 1613 + }, + { + "epoch": 1.0040435458786936, + "grad_norm": 0.5626034736633301, + "learning_rate": 9.65e-06, + "log_odds_chosen": 7.137520790100098, + "log_odds_ratio": -0.25156933069229126, + "logits/chosen": 0.30671074986457825, + "logits/rejected": 0.664474606513977, + "logps/chosen": -1.1866908073425293, + "logps/rejected": -8.073833465576172, + "loss": 3.3325, + "nll_loss": 3.3073184490203857, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11866907775402069, + "rewards/margins": 0.6887142658233643, + "rewards/rejected": -0.8073834180831909, + "step": 1614 + }, + { + "epoch": 1.0046656298600312, + "grad_norm": 0.6039891242980957, + "learning_rate": 9.625e-06, + "log_odds_chosen": 10.04780387878418, + "log_odds_ratio": -0.017218245193362236, + "logits/chosen": 0.2603004276752472, + "logits/rejected": 0.9685711860656738, + "logps/chosen": -1.0688201189041138, + "logps/rejected": -10.475671768188477, + "loss": 2.7024, + "nll_loss": 2.7006781101226807, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1068820059299469, + "rewards/margins": 0.9406852722167969, + "rewards/rejected": -1.0475672483444214, + "step": 1615 + }, + { + "epoch": 1.0052877138413685, + "grad_norm": 0.5885332226753235, + "learning_rate": 9.600000000000001e-06, + "log_odds_chosen": 14.77972412109375, + "log_odds_ratio": -8.65639085532166e-05, + "logits/chosen": 0.2089068740606308, + "logits/rejected": 1.0688587427139282, + "logps/chosen": -1.0595755577087402, + "logps/rejected": -15.397711753845215, + "loss": 2.7162, + "nll_loss": 2.7161991596221924, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10595756024122238, + "rewards/margins": 1.4338136911392212, + "rewards/rejected": -1.5397711992263794, + "step": 1616 + }, + { + "epoch": 1.005909797822706, + "grad_norm": 0.6334537267684937, + "learning_rate": 9.575e-06, + "log_odds_chosen": 8.620237350463867, + "log_odds_ratio": -0.10149388015270233, + "logits/chosen": 0.31568121910095215, + "logits/rejected": 0.9553278088569641, + "logps/chosen": -1.2086679935455322, + "logps/rejected": -9.4962797164917, + "loss": 2.6799, + "nll_loss": 2.669752597808838, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1208668053150177, + "rewards/margins": 0.8287612199783325, + "rewards/rejected": -0.9496279954910278, + "step": 1617 + }, + { + "epoch": 1.0065318818040436, + "grad_norm": 0.4512050449848175, + "learning_rate": 9.55e-06, + "log_odds_chosen": 13.334916114807129, + "log_odds_ratio": -0.01727379858493805, + "logits/chosen": 0.27413877844810486, + "logits/rejected": 1.246893048286438, + "logps/chosen": -1.2072069644927979, + "logps/rejected": -14.180831909179688, + "loss": 3.3728, + "nll_loss": 3.3710615634918213, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12072070688009262, + "rewards/margins": 1.2973624467849731, + "rewards/rejected": -1.4180831909179688, + "step": 1618 + }, + { + "epoch": 1.007153965785381, + "grad_norm": 0.4606066048145294, + "learning_rate": 9.525000000000001e-06, + "log_odds_chosen": 9.140962600708008, + "log_odds_ratio": -0.22496956586837769, + "logits/chosen": 0.26907601952552795, + "logits/rejected": 0.7347286939620972, + "logps/chosen": -1.0311001539230347, + "logps/rejected": -9.772761344909668, + "loss": 3.3197, + "nll_loss": 3.2971792221069336, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10311002284288406, + "rewards/margins": 0.8741660714149475, + "rewards/rejected": -0.9772761464118958, + "step": 1619 + }, + { + "epoch": 1.0077760497667185, + "grad_norm": 0.5686437487602234, + "learning_rate": 9.5e-06, + "log_odds_chosen": 11.098395347595215, + "log_odds_ratio": -0.0006429707864299417, + "logits/chosen": 0.2674759030342102, + "logits/rejected": 0.975322961807251, + "logps/chosen": -0.975488543510437, + "logps/rejected": -11.32666015625, + "loss": 2.7744, + "nll_loss": 2.7743349075317383, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09754885733127594, + "rewards/margins": 1.0351171493530273, + "rewards/rejected": -1.1326658725738525, + "step": 1620 + }, + { + "epoch": 1.008398133748056, + "grad_norm": 0.5224622488021851, + "learning_rate": 9.475e-06, + "log_odds_chosen": 11.144240379333496, + "log_odds_ratio": -0.020334865897893906, + "logits/chosen": 0.16083793342113495, + "logits/rejected": 0.7533071041107178, + "logps/chosen": -1.4969186782836914, + "logps/rejected": -12.29748821258545, + "loss": 3.0359, + "nll_loss": 3.03383207321167, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1496918797492981, + "rewards/margins": 1.0800570249557495, + "rewards/rejected": -1.2297488451004028, + "step": 1621 + }, + { + "epoch": 1.0090202177293934, + "grad_norm": 0.5410659313201904, + "learning_rate": 9.450000000000001e-06, + "log_odds_chosen": 8.919403076171875, + "log_odds_ratio": -0.04832353815436363, + "logits/chosen": 0.11468247324228287, + "logits/rejected": 0.5462947487831116, + "logps/chosen": -0.9372066259384155, + "logps/rejected": -9.317145347595215, + "loss": 2.7276, + "nll_loss": 2.7227399349212646, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09372066706418991, + "rewards/margins": 0.8379939198493958, + "rewards/rejected": -0.931714653968811, + "step": 1622 + }, + { + "epoch": 1.009642301710731, + "grad_norm": 0.5279932022094727, + "learning_rate": 9.425e-06, + "log_odds_chosen": 9.845748901367188, + "log_odds_ratio": -0.22621092200279236, + "logits/chosen": 0.14059889316558838, + "logits/rejected": 0.6940497159957886, + "logps/chosen": -1.0277276039123535, + "logps/rejected": -10.441768646240234, + "loss": 2.6895, + "nll_loss": 2.666898727416992, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10277276486158371, + "rewards/margins": 0.9414041638374329, + "rewards/rejected": -1.0441769361495972, + "step": 1623 + }, + { + "epoch": 1.0102643856920683, + "grad_norm": 0.45512518286705017, + "learning_rate": 9.4e-06, + "log_odds_chosen": 8.7213716506958, + "log_odds_ratio": -0.22801584005355835, + "logits/chosen": 0.18221153318881989, + "logits/rejected": 0.6221237778663635, + "logps/chosen": -0.9007062911987305, + "logps/rejected": -9.128543853759766, + "loss": 3.0911, + "nll_loss": 3.0683000087738037, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09007062762975693, + "rewards/margins": 0.8227838277816772, + "rewards/rejected": -0.9128544330596924, + "step": 1624 + }, + { + "epoch": 1.010886469673406, + "grad_norm": 0.5499163269996643, + "learning_rate": 9.375000000000001e-06, + "log_odds_chosen": 13.863089561462402, + "log_odds_ratio": -0.00355136557482183, + "logits/chosen": 0.313961386680603, + "logits/rejected": 1.2714308500289917, + "logps/chosen": -1.0769063234329224, + "logps/rejected": -14.27375602722168, + "loss": 3.1019, + "nll_loss": 3.101551055908203, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10769063234329224, + "rewards/margins": 1.3196849822998047, + "rewards/rejected": -1.4273755550384521, + "step": 1625 + }, + { + "epoch": 1.0115085536547435, + "grad_norm": 10.339038848876953, + "learning_rate": 9.35e-06, + "log_odds_chosen": 12.912702560424805, + "log_odds_ratio": -0.0035233048256486654, + "logits/chosen": 0.3977145552635193, + "logits/rejected": 1.5618952512741089, + "logps/chosen": -1.3644357919692993, + "logps/rejected": -13.928369522094727, + "loss": 2.8382, + "nll_loss": 2.8378021717071533, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1364435851573944, + "rewards/margins": 1.2563934326171875, + "rewards/rejected": -1.3928370475769043, + "step": 1626 + }, + { + "epoch": 1.0121306376360808, + "grad_norm": 0.6409747004508972, + "learning_rate": 9.325e-06, + "log_odds_chosen": 7.80657958984375, + "log_odds_ratio": -0.14597001671791077, + "logits/chosen": 0.3135356307029724, + "logits/rejected": 0.9694831371307373, + "logps/chosen": -1.0715206861495972, + "logps/rejected": -8.544049263000488, + "loss": 3.0718, + "nll_loss": 3.057161331176758, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1071520671248436, + "rewards/margins": 0.7472528219223022, + "rewards/rejected": -0.8544048070907593, + "step": 1627 + }, + { + "epoch": 1.0127527216174184, + "grad_norm": 0.49594753980636597, + "learning_rate": 9.3e-06, + "log_odds_chosen": 9.20004653930664, + "log_odds_ratio": -0.16701725125312805, + "logits/chosen": 0.205534428358078, + "logits/rejected": 0.8317206501960754, + "logps/chosen": -1.0844453573226929, + "logps/rejected": -9.976225852966309, + "loss": 2.8505, + "nll_loss": 2.8337974548339844, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10844454169273376, + "rewards/margins": 0.8891780972480774, + "rewards/rejected": -0.9976226687431335, + "step": 1628 + }, + { + "epoch": 1.013374805598756, + "grad_norm": 0.5218719840049744, + "learning_rate": 9.275e-06, + "log_odds_chosen": 12.281805038452148, + "log_odds_ratio": -0.002139680553227663, + "logits/chosen": 0.30369701981544495, + "logits/rejected": 1.1859633922576904, + "logps/chosen": -0.9268889427185059, + "logps/rejected": -12.533408164978027, + "loss": 2.9504, + "nll_loss": 2.950138568878174, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0926888957619667, + "rewards/margins": 1.1606519222259521, + "rewards/rejected": -1.2533408403396606, + "step": 1629 + }, + { + "epoch": 1.0139968895800933, + "grad_norm": 0.4695424437522888, + "learning_rate": 9.25e-06, + "log_odds_chosen": 11.708459854125977, + "log_odds_ratio": -0.17082315683364868, + "logits/chosen": 0.063386932015419, + "logits/rejected": 0.5708033442497253, + "logps/chosen": -1.0007423162460327, + "logps/rejected": -12.331064224243164, + "loss": 3.1128, + "nll_loss": 3.0956997871398926, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10007423907518387, + "rewards/margins": 1.133032202720642, + "rewards/rejected": -1.2331064939498901, + "step": 1630 + }, + { + "epoch": 1.0146189735614308, + "grad_norm": 0.704513669013977, + "learning_rate": 9.225e-06, + "log_odds_chosen": 3.0890965461730957, + "log_odds_ratio": -0.2591760456562042, + "logits/chosen": 0.28655803203582764, + "logits/rejected": 0.2705000638961792, + "logps/chosen": -0.9935261011123657, + "logps/rejected": -3.74479079246521, + "loss": 3.3577, + "nll_loss": 3.331785202026367, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09935261309146881, + "rewards/margins": 0.27512648701667786, + "rewards/rejected": -0.37447911500930786, + "step": 1631 + }, + { + "epoch": 1.0152410575427682, + "grad_norm": 0.49560216069221497, + "learning_rate": 9.2e-06, + "log_odds_chosen": 11.836051940917969, + "log_odds_ratio": -0.0024351109750568867, + "logits/chosen": 0.23947438597679138, + "logits/rejected": 1.1317088603973389, + "logps/chosen": -0.7412389516830444, + "logps/rejected": -11.846299171447754, + "loss": 2.8303, + "nll_loss": 2.8300487995147705, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07412389665842056, + "rewards/margins": 1.1105060577392578, + "rewards/rejected": -1.1846299171447754, + "step": 1632 + }, + { + "epoch": 1.0158631415241057, + "grad_norm": 0.6829708218574524, + "learning_rate": 9.175000000000001e-06, + "log_odds_chosen": 13.079815864562988, + "log_odds_ratio": -0.011433121748268604, + "logits/chosen": 0.2853188216686249, + "logits/rejected": 1.2370648384094238, + "logps/chosen": -1.119140386581421, + "logps/rejected": -13.742539405822754, + "loss": 2.8621, + "nll_loss": 2.860992908477783, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11191404610872269, + "rewards/margins": 1.2623398303985596, + "rewards/rejected": -1.3742539882659912, + "step": 1633 + }, + { + "epoch": 1.0164852255054433, + "grad_norm": 0.47538915276527405, + "learning_rate": 9.15e-06, + "log_odds_chosen": 13.191617965698242, + "log_odds_ratio": -0.1977131962776184, + "logits/chosen": 0.2846056818962097, + "logits/rejected": 0.9628204107284546, + "logps/chosen": -1.1906006336212158, + "logps/rejected": -14.044559478759766, + "loss": 3.3493, + "nll_loss": 3.3294787406921387, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11906006187200546, + "rewards/margins": 1.285395860671997, + "rewards/rejected": -1.4044559001922607, + "step": 1634 + }, + { + "epoch": 1.0171073094867806, + "grad_norm": 0.6789840459823608, + "learning_rate": 9.125e-06, + "log_odds_chosen": 12.73701000213623, + "log_odds_ratio": -0.07127546519041061, + "logits/chosen": 0.19782181084156036, + "logits/rejected": 1.3850300312042236, + "logps/chosen": -1.0893549919128418, + "logps/rejected": -13.426777839660645, + "loss": 2.5798, + "nll_loss": 2.572679281234741, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10893549770116806, + "rewards/margins": 1.233742356300354, + "rewards/rejected": -1.3426778316497803, + "step": 1635 + }, + { + "epoch": 1.0177293934681182, + "grad_norm": 0.46701884269714355, + "learning_rate": 9.100000000000001e-06, + "log_odds_chosen": 12.932587623596191, + "log_odds_ratio": -0.0002816337801050395, + "logits/chosen": 0.3475741147994995, + "logits/rejected": 1.3140875101089478, + "logps/chosen": -1.0825169086456299, + "logps/rejected": -13.484697341918945, + "loss": 3.4196, + "nll_loss": 3.419621467590332, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10825169086456299, + "rewards/margins": 1.240217924118042, + "rewards/rejected": -1.3484697341918945, + "step": 1636 + }, + { + "epoch": 1.0183514774494558, + "grad_norm": 0.5956992506980896, + "learning_rate": 9.075e-06, + "log_odds_chosen": 9.46751594543457, + "log_odds_ratio": -0.12005427479743958, + "logits/chosen": 0.16276614367961884, + "logits/rejected": 0.7512280344963074, + "logps/chosen": -0.8385735154151917, + "logps/rejected": -9.679712295532227, + "loss": 2.728, + "nll_loss": 2.7160072326660156, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08385735750198364, + "rewards/margins": 0.8841139078140259, + "rewards/rejected": -0.9679712653160095, + "step": 1637 + }, + { + "epoch": 1.018973561430793, + "grad_norm": 0.44732701778411865, + "learning_rate": 9.05e-06, + "log_odds_chosen": 10.955498695373535, + "log_odds_ratio": -0.1099550873041153, + "logits/chosen": 0.13595137000083923, + "logits/rejected": 1.1490558385849, + "logps/chosen": -1.2426047325134277, + "logps/rejected": -11.894643783569336, + "loss": 2.7433, + "nll_loss": 2.7323482036590576, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12426047027111053, + "rewards/margins": 1.0652040243148804, + "rewards/rejected": -1.1894644498825073, + "step": 1638 + }, + { + "epoch": 1.0195956454121307, + "grad_norm": 0.8864432573318481, + "learning_rate": 9.025e-06, + "log_odds_chosen": 10.31212329864502, + "log_odds_ratio": -0.17925220727920532, + "logits/chosen": 0.18999649584293365, + "logits/rejected": 0.9962109327316284, + "logps/chosen": -1.1252002716064453, + "logps/rejected": -10.985133171081543, + "loss": 3.2673, + "nll_loss": 3.2493417263031006, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11252002418041229, + "rewards/margins": 0.9859933257102966, + "rewards/rejected": -1.0985133647918701, + "step": 1639 + }, + { + "epoch": 1.0202177293934682, + "grad_norm": 0.5043367743492126, + "learning_rate": 9e-06, + "log_odds_chosen": 14.583747863769531, + "log_odds_ratio": -0.005502515938133001, + "logits/chosen": 0.2616561949253082, + "logits/rejected": 1.3393410444259644, + "logps/chosen": -1.0964058637619019, + "logps/rejected": -15.243389129638672, + "loss": 3.0131, + "nll_loss": 3.0125889778137207, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10964059084653854, + "rewards/margins": 1.4146982431411743, + "rewards/rejected": -1.5243388414382935, + "step": 1640 + }, + { + "epoch": 1.0208398133748056, + "grad_norm": 0.4772394299507141, + "learning_rate": 8.975e-06, + "log_odds_chosen": 13.822057723999023, + "log_odds_ratio": -0.003005790524184704, + "logits/chosen": 0.2907000184059143, + "logits/rejected": 1.3935949802398682, + "logps/chosen": -1.2302254438400269, + "logps/rejected": -14.66832447052002, + "loss": 2.9518, + "nll_loss": 2.9514856338500977, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12302252650260925, + "rewards/margins": 1.3438100814819336, + "rewards/rejected": -1.4668323993682861, + "step": 1641 + }, + { + "epoch": 1.0214618973561431, + "grad_norm": 0.5077674984931946, + "learning_rate": 8.95e-06, + "log_odds_chosen": 10.747535705566406, + "log_odds_ratio": -0.11549379676580429, + "logits/chosen": 0.23091387748718262, + "logits/rejected": 1.3234198093414307, + "logps/chosen": -1.0908149480819702, + "logps/rejected": -11.471622467041016, + "loss": 2.7741, + "nll_loss": 2.7625174522399902, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10908149182796478, + "rewards/margins": 1.0380808115005493, + "rewards/rejected": -1.1471623182296753, + "step": 1642 + }, + { + "epoch": 1.0220839813374805, + "grad_norm": 0.4699145555496216, + "learning_rate": 8.925e-06, + "log_odds_chosen": 11.556468963623047, + "log_odds_ratio": -0.0014705258654430509, + "logits/chosen": 0.30578139424324036, + "logits/rejected": 1.0413280725479126, + "logps/chosen": -0.8892867565155029, + "logps/rejected": -11.754606246948242, + "loss": 3.3391, + "nll_loss": 3.338972806930542, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08892867714166641, + "rewards/margins": 1.0865321159362793, + "rewards/rejected": -1.175460696220398, + "step": 1643 + }, + { + "epoch": 1.022706065318818, + "grad_norm": 0.5027690529823303, + "learning_rate": 8.9e-06, + "log_odds_chosen": 10.694028854370117, + "log_odds_ratio": -0.10222534090280533, + "logits/chosen": 0.2960459589958191, + "logits/rejected": 1.1127769947052002, + "logps/chosen": -1.3843754529953003, + "logps/rejected": -11.760503768920898, + "loss": 3.3106, + "nll_loss": 3.3003430366516113, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13843755424022675, + "rewards/margins": 1.037612795829773, + "rewards/rejected": -1.1760503053665161, + "step": 1644 + }, + { + "epoch": 1.0233281493001556, + "grad_norm": 0.4155937135219574, + "learning_rate": 8.875e-06, + "log_odds_chosen": 11.716170310974121, + "log_odds_ratio": -0.0036328420974314213, + "logits/chosen": 0.22692963480949402, + "logits/rejected": 1.210436463356018, + "logps/chosen": -1.059539794921875, + "logps/rejected": -12.310782432556152, + "loss": 2.9858, + "nll_loss": 2.985433340072632, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10595397651195526, + "rewards/margins": 1.125124216079712, + "rewards/rejected": -1.2310782670974731, + "step": 1645 + }, + { + "epoch": 1.023950233281493, + "grad_norm": 0.5141968131065369, + "learning_rate": 8.85e-06, + "log_odds_chosen": 14.782506942749023, + "log_odds_ratio": -3.0398521175811766e-06, + "logits/chosen": 0.2925991714000702, + "logits/rejected": 1.2928005456924438, + "logps/chosen": -1.1867799758911133, + "logps/rejected": -15.458690643310547, + "loss": 3.3069, + "nll_loss": 3.3069450855255127, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11867799609899521, + "rewards/margins": 1.427191138267517, + "rewards/rejected": -1.5458691120147705, + "step": 1646 + }, + { + "epoch": 1.0245723172628305, + "grad_norm": 0.46805500984191895, + "learning_rate": 8.825e-06, + "log_odds_chosen": 11.715710639953613, + "log_odds_ratio": -0.05991049483418465, + "logits/chosen": 0.1678595393896103, + "logits/rejected": 0.6166034936904907, + "logps/chosen": -1.2212872505187988, + "logps/rejected": -12.560037612915039, + "loss": 2.9821, + "nll_loss": 2.976140022277832, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12212873995304108, + "rewards/margins": 1.1338750123977661, + "rewards/rejected": -1.256003737449646, + "step": 1647 + }, + { + "epoch": 1.025194401244168, + "grad_norm": 0.5412681102752686, + "learning_rate": 8.8e-06, + "log_odds_chosen": 6.542932510375977, + "log_odds_ratio": -0.17082630097866058, + "logits/chosen": 0.08377973735332489, + "logits/rejected": 0.2964077889919281, + "logps/chosen": -1.022648572921753, + "logps/rejected": -7.137678623199463, + "loss": 2.8866, + "nll_loss": 2.869472026824951, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10226485878229141, + "rewards/margins": 0.611503005027771, + "rewards/rejected": -0.7137678861618042, + "step": 1648 + }, + { + "epoch": 1.0258164852255054, + "grad_norm": 0.568443238735199, + "learning_rate": 8.775e-06, + "log_odds_chosen": 13.49286937713623, + "log_odds_ratio": -0.11019520461559296, + "logits/chosen": 0.21617048978805542, + "logits/rejected": 1.1071817874908447, + "logps/chosen": -1.297656774520874, + "logps/rejected": -14.47043228149414, + "loss": 2.8539, + "nll_loss": 2.8428421020507812, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12976567447185516, + "rewards/margins": 1.3172776699066162, + "rewards/rejected": -1.4470432996749878, + "step": 1649 + }, + { + "epoch": 1.026438569206843, + "grad_norm": 0.8365088105201721, + "learning_rate": 8.75e-06, + "log_odds_chosen": 8.931707382202148, + "log_odds_ratio": -0.054758794605731964, + "logits/chosen": 0.1608840823173523, + "logits/rejected": 0.92148357629776, + "logps/chosen": -1.4928791522979736, + "logps/rejected": -10.121048927307129, + "loss": 2.7444, + "nll_loss": 2.738970994949341, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14928790926933289, + "rewards/margins": 0.8628170490264893, + "rewards/rejected": -1.0121049880981445, + "step": 1650 + }, + { + "epoch": 1.0270606531881805, + "grad_norm": 0.593108594417572, + "learning_rate": 8.725e-06, + "log_odds_chosen": 8.8706636428833, + "log_odds_ratio": -0.1627025157213211, + "logits/chosen": 0.2817043960094452, + "logits/rejected": 0.9622519612312317, + "logps/chosen": -1.1062893867492676, + "logps/rejected": -9.600112915039062, + "loss": 2.8041, + "nll_loss": 2.7878713607788086, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11062893271446228, + "rewards/margins": 0.8493824005126953, + "rewards/rejected": -0.9600112438201904, + "step": 1651 + }, + { + "epoch": 1.0276827371695179, + "grad_norm": 0.527895987033844, + "learning_rate": 8.7e-06, + "log_odds_chosen": 10.27220630645752, + "log_odds_ratio": -0.0012983853230252862, + "logits/chosen": 0.32650476694107056, + "logits/rejected": 0.947507381439209, + "logps/chosen": -1.0816434621810913, + "logps/rejected": -10.885249137878418, + "loss": 3.1771, + "nll_loss": 3.1769237518310547, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10816435515880585, + "rewards/margins": 0.9803605079650879, + "rewards/rejected": -1.0885248184204102, + "step": 1652 + }, + { + "epoch": 1.0283048211508554, + "grad_norm": 0.5855411887168884, + "learning_rate": 8.674999999999999e-06, + "log_odds_chosen": 10.549446105957031, + "log_odds_ratio": -0.1294151395559311, + "logits/chosen": 0.24553564190864563, + "logits/rejected": 0.9942394495010376, + "logps/chosen": -1.028377890586853, + "logps/rejected": -11.156509399414062, + "loss": 3.0605, + "nll_loss": 3.0475263595581055, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10283779352903366, + "rewards/margins": 1.0128130912780762, + "rewards/rejected": -1.11565101146698, + "step": 1653 + }, + { + "epoch": 1.0289269051321928, + "grad_norm": 0.48511913418769836, + "learning_rate": 8.65e-06, + "log_odds_chosen": 11.069501876831055, + "log_odds_ratio": -0.2245204597711563, + "logits/chosen": 0.2477826178073883, + "logits/rejected": 1.0707584619522095, + "logps/chosen": -1.2009871006011963, + "logps/rejected": -12.028597831726074, + "loss": 3.0148, + "nll_loss": 2.9923830032348633, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12009871006011963, + "rewards/margins": 1.0827610492706299, + "rewards/rejected": -1.2028597593307495, + "step": 1654 + }, + { + "epoch": 1.0295489891135303, + "grad_norm": 0.4419611096382141, + "learning_rate": 8.625e-06, + "log_odds_chosen": 6.270127773284912, + "log_odds_ratio": -0.284975528717041, + "logits/chosen": 0.2656201422214508, + "logits/rejected": 0.5505940914154053, + "logps/chosen": -0.9445799589157104, + "logps/rejected": -6.642779350280762, + "loss": 3.4913, + "nll_loss": 3.4627652168273926, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09445799887180328, + "rewards/margins": 0.5698199272155762, + "rewards/rejected": -0.664277970790863, + "step": 1655 + }, + { + "epoch": 1.0301710730948679, + "grad_norm": 0.651220977306366, + "learning_rate": 8.599999999999999e-06, + "log_odds_chosen": 10.812170028686523, + "log_odds_ratio": -0.11016173660755157, + "logits/chosen": 0.4329462945461273, + "logits/rejected": 0.958261251449585, + "logps/chosen": -0.9240531921386719, + "logps/rejected": -11.261800765991211, + "loss": 3.3339, + "nll_loss": 3.3228507041931152, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09240531921386719, + "rewards/margins": 1.033774733543396, + "rewards/rejected": -1.1261800527572632, + "step": 1656 + }, + { + "epoch": 1.0307931570762052, + "grad_norm": 0.4697381556034088, + "learning_rate": 8.575000000000002e-06, + "log_odds_chosen": 13.0179443359375, + "log_odds_ratio": -0.04233914241194725, + "logits/chosen": 0.40351903438568115, + "logits/rejected": 1.2205008268356323, + "logps/chosen": -0.8437970280647278, + "logps/rejected": -13.180481910705566, + "loss": 3.3443, + "nll_loss": 3.340068817138672, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08437971025705338, + "rewards/margins": 1.2336684465408325, + "rewards/rejected": -1.3180482387542725, + "step": 1657 + }, + { + "epoch": 1.0314152410575428, + "grad_norm": 0.4366108775138855, + "learning_rate": 8.550000000000001e-06, + "log_odds_chosen": 14.342367172241211, + "log_odds_ratio": -0.0002054653305094689, + "logits/chosen": 0.38981306552886963, + "logits/rejected": 1.3780097961425781, + "logps/chosen": -1.0400646924972534, + "logps/rejected": -14.886777877807617, + "loss": 3.5205, + "nll_loss": 3.520503520965576, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10400646924972534, + "rewards/margins": 1.3846713304519653, + "rewards/rejected": -1.4886778593063354, + "step": 1658 + }, + { + "epoch": 1.0320373250388803, + "grad_norm": 0.6422800421714783, + "learning_rate": 8.525e-06, + "log_odds_chosen": 18.632835388183594, + "log_odds_ratio": -3.8743075947422767e-07, + "logits/chosen": 0.290744811296463, + "logits/rejected": 1.8333404064178467, + "logps/chosen": -1.349289894104004, + "logps/rejected": -19.636199951171875, + "loss": 2.8319, + "nll_loss": 2.8319075107574463, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13492897152900696, + "rewards/margins": 1.8286911249160767, + "rewards/rejected": -1.9636200666427612, + "step": 1659 + }, + { + "epoch": 1.0326594090202177, + "grad_norm": 0.5107365846633911, + "learning_rate": 8.500000000000002e-06, + "log_odds_chosen": 9.844313621520996, + "log_odds_ratio": -0.11742329597473145, + "logits/chosen": 0.2040708363056183, + "logits/rejected": 0.6333246231079102, + "logps/chosen": -1.2048256397247314, + "logps/rejected": -10.693470001220703, + "loss": 2.9243, + "nll_loss": 2.912597179412842, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12048256397247314, + "rewards/margins": 0.9488645195960999, + "rewards/rejected": -1.0693471431732178, + "step": 1660 + }, + { + "epoch": 1.0332814930015553, + "grad_norm": 0.4743788540363312, + "learning_rate": 8.475000000000001e-06, + "log_odds_chosen": 14.308911323547363, + "log_odds_ratio": -0.08154227584600449, + "logits/chosen": 0.16491791605949402, + "logits/rejected": 1.0353258848190308, + "logps/chosen": -1.0096592903137207, + "logps/rejected": -14.768461227416992, + "loss": 2.7397, + "nll_loss": 2.731551170349121, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1009659394621849, + "rewards/margins": 1.375880241394043, + "rewards/rejected": -1.4768463373184204, + "step": 1661 + }, + { + "epoch": 1.0339035769828926, + "grad_norm": 0.5415398478507996, + "learning_rate": 8.45e-06, + "log_odds_chosen": 12.673973083496094, + "log_odds_ratio": -0.0749824270606041, + "logits/chosen": 0.27741706371307373, + "logits/rejected": 1.1964267492294312, + "logps/chosen": -0.8574082851409912, + "logps/rejected": -13.011079788208008, + "loss": 2.7563, + "nll_loss": 2.7487845420837402, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0857408344745636, + "rewards/margins": 1.2153671979904175, + "rewards/rejected": -1.3011078834533691, + "step": 1662 + }, + { + "epoch": 1.0345256609642302, + "grad_norm": 2.5805556774139404, + "learning_rate": 8.425000000000001e-06, + "log_odds_chosen": 12.792621612548828, + "log_odds_ratio": -0.09287473559379578, + "logits/chosen": 0.3072134852409363, + "logits/rejected": 1.4438683986663818, + "logps/chosen": -1.1244754791259766, + "logps/rejected": -13.274697303771973, + "loss": 2.7362, + "nll_loss": 2.7268877029418945, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11244755238294601, + "rewards/margins": 1.2150222063064575, + "rewards/rejected": -1.327469825744629, + "step": 1663 + }, + { + "epoch": 1.0351477449455677, + "grad_norm": 0.5692621469497681, + "learning_rate": 8.400000000000001e-06, + "log_odds_chosen": 12.14323902130127, + "log_odds_ratio": -0.08631030470132828, + "logits/chosen": 0.2854219377040863, + "logits/rejected": 1.3667171001434326, + "logps/chosen": -1.0392296314239502, + "logps/rejected": -12.711332321166992, + "loss": 2.9046, + "nll_loss": 2.895948648452759, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10392297059297562, + "rewards/margins": 1.1672102212905884, + "rewards/rejected": -1.2711331844329834, + "step": 1664 + }, + { + "epoch": 1.035769828926905, + "grad_norm": 0.6746039986610413, + "learning_rate": 8.375e-06, + "log_odds_chosen": 10.063125610351562, + "log_odds_ratio": -0.1466355323791504, + "logits/chosen": 0.371590256690979, + "logits/rejected": 0.8608337044715881, + "logps/chosen": -0.8012755513191223, + "logps/rejected": -10.27314281463623, + "loss": 3.3805, + "nll_loss": 3.3658719062805176, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08012756705284119, + "rewards/margins": 0.9471867084503174, + "rewards/rejected": -1.027314305305481, + "step": 1665 + }, + { + "epoch": 1.0363919129082426, + "grad_norm": 1.6298115253448486, + "learning_rate": 8.350000000000001e-06, + "log_odds_chosen": 12.227920532226562, + "log_odds_ratio": -0.013340647332370281, + "logits/chosen": 0.1919909566640854, + "logits/rejected": 0.9904528856277466, + "logps/chosen": -1.073433756828308, + "logps/rejected": -12.796059608459473, + "loss": 2.9285, + "nll_loss": 2.927194595336914, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10734338313341141, + "rewards/margins": 1.1722626686096191, + "rewards/rejected": -1.2796061038970947, + "step": 1666 + }, + { + "epoch": 1.0370139968895802, + "grad_norm": 0.6101058125495911, + "learning_rate": 8.325e-06, + "log_odds_chosen": 10.183680534362793, + "log_odds_ratio": -0.03945440798997879, + "logits/chosen": 0.19264741241931915, + "logits/rejected": 1.0039281845092773, + "logps/chosen": -0.9766156077384949, + "logps/rejected": -10.625382423400879, + "loss": 2.5488, + "nll_loss": 2.544872999191284, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0976615697145462, + "rewards/margins": 0.9648767113685608, + "rewards/rejected": -1.0625382661819458, + "step": 1667 + }, + { + "epoch": 1.0376360808709175, + "grad_norm": 0.5250122547149658, + "learning_rate": 8.3e-06, + "log_odds_chosen": 9.799418449401855, + "log_odds_ratio": -0.010854106396436691, + "logits/chosen": 0.27762043476104736, + "logits/rejected": 0.7463703155517578, + "logps/chosen": -1.1591010093688965, + "logps/rejected": -10.477595329284668, + "loss": 3.3777, + "nll_loss": 3.3766472339630127, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.115910105407238, + "rewards/margins": 0.931849479675293, + "rewards/rejected": -1.0477595329284668, + "step": 1668 + }, + { + "epoch": 1.038258164852255, + "grad_norm": 0.6709027886390686, + "learning_rate": 8.275000000000001e-06, + "log_odds_chosen": 11.033429145812988, + "log_odds_ratio": -0.17414605617523193, + "logits/chosen": 0.4432452619075775, + "logits/rejected": 1.211717128753662, + "logps/chosen": -0.9460048675537109, + "logps/rejected": -11.531136512756348, + "loss": 3.2111, + "nll_loss": 3.1937031745910645, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09460049867630005, + "rewards/margins": 1.0585131645202637, + "rewards/rejected": -1.1531137228012085, + "step": 1669 + }, + { + "epoch": 1.0388802488335926, + "grad_norm": 2.698802947998047, + "learning_rate": 8.25e-06, + "log_odds_chosen": 13.314163208007812, + "log_odds_ratio": -0.07637029886245728, + "logits/chosen": 0.20780786871910095, + "logits/rejected": 0.8780178427696228, + "logps/chosen": -1.4755923748016357, + "logps/rejected": -14.065767288208008, + "loss": 2.9374, + "nll_loss": 2.929812431335449, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1475592404603958, + "rewards/margins": 1.2590174674987793, + "rewards/rejected": -1.4065766334533691, + "step": 1670 + }, + { + "epoch": 1.03950233281493, + "grad_norm": 0.4637618660926819, + "learning_rate": 8.225e-06, + "log_odds_chosen": 14.614171981811523, + "log_odds_ratio": -0.061297375708818436, + "logits/chosen": 0.36924469470977783, + "logits/rejected": 1.271532654762268, + "logps/chosen": -0.9310328364372253, + "logps/rejected": -15.078804969787598, + "loss": 3.3165, + "nll_loss": 3.310403347015381, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09310328215360641, + "rewards/margins": 1.4147772789001465, + "rewards/rejected": -1.507880449295044, + "step": 1671 + }, + { + "epoch": 1.0401244167962675, + "grad_norm": 0.5099697709083557, + "learning_rate": 8.200000000000001e-06, + "log_odds_chosen": 10.016826629638672, + "log_odds_ratio": -0.10711626708507538, + "logits/chosen": 0.01908724009990692, + "logits/rejected": 0.46566832065582275, + "logps/chosen": -0.7953150272369385, + "logps/rejected": -10.18687915802002, + "loss": 2.4798, + "nll_loss": 2.469115972518921, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.07953150570392609, + "rewards/margins": 0.9391564130783081, + "rewards/rejected": -1.0186879634857178, + "step": 1672 + }, + { + "epoch": 1.0407465007776049, + "grad_norm": 0.49379605054855347, + "learning_rate": 8.175e-06, + "log_odds_chosen": 12.284273147583008, + "log_odds_ratio": -0.05759778618812561, + "logits/chosen": 0.3830791413784027, + "logits/rejected": 1.3384016752243042, + "logps/chosen": -0.8844354748725891, + "logps/rejected": -12.67086410522461, + "loss": 2.7113, + "nll_loss": 2.7055153846740723, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08844354748725891, + "rewards/margins": 1.178642988204956, + "rewards/rejected": -1.2670865058898926, + "step": 1673 + }, + { + "epoch": 1.0413685847589425, + "grad_norm": 1.1188862323760986, + "learning_rate": 8.15e-06, + "log_odds_chosen": 9.94202709197998, + "log_odds_ratio": -0.09657282382249832, + "logits/chosen": 0.293910950422287, + "logits/rejected": 0.9388896226882935, + "logps/chosen": -1.2859009504318237, + "logps/rejected": -10.89704704284668, + "loss": 2.8781, + "nll_loss": 2.86840558052063, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12859009206295013, + "rewards/margins": 0.9611146450042725, + "rewards/rejected": -1.0897047519683838, + "step": 1674 + }, + { + "epoch": 1.04199066874028, + "grad_norm": 0.5678275227546692, + "learning_rate": 8.125000000000001e-06, + "log_odds_chosen": 13.660589218139648, + "log_odds_ratio": -0.0012313383631408215, + "logits/chosen": 0.20735794305801392, + "logits/rejected": 1.7398574352264404, + "logps/chosen": -1.1810181140899658, + "logps/rejected": -14.4437255859375, + "loss": 2.6848, + "nll_loss": 2.684652328491211, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1181018203496933, + "rewards/margins": 1.326270580291748, + "rewards/rejected": -1.4443724155426025, + "step": 1675 + }, + { + "epoch": 1.0426127527216174, + "grad_norm": 0.5290229916572571, + "learning_rate": 8.1e-06, + "log_odds_chosen": 4.564973831176758, + "log_odds_ratio": -0.2650543451309204, + "logits/chosen": 0.25153490900993347, + "logits/rejected": 0.5440673828125, + "logps/chosen": -1.191606044769287, + "logps/rejected": -5.467530250549316, + "loss": 3.2536, + "nll_loss": 3.2270967960357666, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11916061490774155, + "rewards/margins": 0.427592396736145, + "rewards/rejected": -0.5467530488967896, + "step": 1676 + }, + { + "epoch": 1.043234836702955, + "grad_norm": 0.39739978313446045, + "learning_rate": 8.075000000000001e-06, + "log_odds_chosen": 10.400979042053223, + "log_odds_ratio": -0.2506236732006073, + "logits/chosen": 0.16898976266384125, + "logits/rejected": 0.5718826055526733, + "logps/chosen": -1.1356854438781738, + "logps/rejected": -11.066657066345215, + "loss": 3.3225, + "nll_loss": 3.297466993331909, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11356855928897858, + "rewards/margins": 0.993097186088562, + "rewards/rejected": -1.106665849685669, + "step": 1677 + }, + { + "epoch": 1.0438569206842925, + "grad_norm": 0.5288723111152649, + "learning_rate": 8.050000000000001e-06, + "log_odds_chosen": 11.958285331726074, + "log_odds_ratio": -0.07352694123983383, + "logits/chosen": 0.2259128987789154, + "logits/rejected": 1.3740233182907104, + "logps/chosen": -1.3853117227554321, + "logps/rejected": -13.046581268310547, + "loss": 2.9374, + "nll_loss": 2.930062770843506, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1385311782360077, + "rewards/margins": 1.1661269664764404, + "rewards/rejected": -1.3046581745147705, + "step": 1678 + }, + { + "epoch": 1.0444790046656298, + "grad_norm": 2.1460111141204834, + "learning_rate": 8.025e-06, + "log_odds_chosen": 10.04307746887207, + "log_odds_ratio": -0.23023003339767456, + "logits/chosen": 0.18102434277534485, + "logits/rejected": 0.9223688840866089, + "logps/chosen": -1.1232291460037231, + "logps/rejected": -10.89987564086914, + "loss": 2.447, + "nll_loss": 2.423985481262207, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11232291162014008, + "rewards/margins": 0.9776646494865417, + "rewards/rejected": -1.0899875164031982, + "step": 1679 + }, + { + "epoch": 1.0451010886469674, + "grad_norm": 0.5670279264450073, + "learning_rate": 8.000000000000001e-06, + "log_odds_chosen": 7.932676315307617, + "log_odds_ratio": -0.1673051118850708, + "logits/chosen": 0.3382868766784668, + "logits/rejected": 0.7066163420677185, + "logps/chosen": -0.8997178077697754, + "logps/rejected": -8.353787422180176, + "loss": 3.5031, + "nll_loss": 3.486398935317993, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08997179567813873, + "rewards/margins": 0.745406985282898, + "rewards/rejected": -0.8353787660598755, + "step": 1680 + }, + { + "epoch": 1.0457231726283047, + "grad_norm": 0.5133948922157288, + "learning_rate": 7.975e-06, + "log_odds_chosen": 6.900932788848877, + "log_odds_ratio": -0.2566588222980499, + "logits/chosen": 0.30161863565444946, + "logits/rejected": 0.512200117111206, + "logps/chosen": -1.108983039855957, + "logps/rejected": -7.704831123352051, + "loss": 3.0651, + "nll_loss": 3.0394201278686523, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11089830845594406, + "rewards/margins": 0.6595847606658936, + "rewards/rejected": -0.770483136177063, + "step": 1681 + }, + { + "epoch": 1.0463452566096423, + "grad_norm": 1.7218198776245117, + "learning_rate": 7.95e-06, + "log_odds_chosen": 10.679391860961914, + "log_odds_ratio": -0.2528221607208252, + "logits/chosen": 0.16948404908180237, + "logits/rejected": 0.7292956709861755, + "logps/chosen": -1.3855730295181274, + "logps/rejected": -11.849591255187988, + "loss": 3.0227, + "nll_loss": 2.997460126876831, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1385573148727417, + "rewards/margins": 1.046401858329773, + "rewards/rejected": -1.184959053993225, + "step": 1682 + }, + { + "epoch": 1.0469673405909798, + "grad_norm": 0.543219804763794, + "learning_rate": 7.925000000000001e-06, + "log_odds_chosen": 11.195889472961426, + "log_odds_ratio": -0.05250005051493645, + "logits/chosen": 0.43830806016921997, + "logits/rejected": 1.0013113021850586, + "logps/chosen": -0.8631675243377686, + "logps/rejected": -11.426183700561523, + "loss": 3.1089, + "nll_loss": 3.103605270385742, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08631675690412521, + "rewards/margins": 1.0563015937805176, + "rewards/rejected": -1.1426184177398682, + "step": 1683 + }, + { + "epoch": 1.0475894245723172, + "grad_norm": 0.6750419735908508, + "learning_rate": 7.9e-06, + "log_odds_chosen": 11.189361572265625, + "log_odds_ratio": -0.07531260699033737, + "logits/chosen": 0.3269883394241333, + "logits/rejected": 1.2064783573150635, + "logps/chosen": -0.7809778451919556, + "logps/rejected": -11.376087188720703, + "loss": 2.3984, + "nll_loss": 2.3908650875091553, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07809779047966003, + "rewards/margins": 1.0595110654830933, + "rewards/rejected": -1.1376087665557861, + "step": 1684 + }, + { + "epoch": 1.0482115085536547, + "grad_norm": 0.4348224997520447, + "learning_rate": 7.875e-06, + "log_odds_chosen": 13.801148414611816, + "log_odds_ratio": -0.08034741878509521, + "logits/chosen": 0.3412080705165863, + "logits/rejected": 1.4113205671310425, + "logps/chosen": -1.0024620294570923, + "logps/rejected": -14.352299690246582, + "loss": 2.7022, + "nll_loss": 2.6942036151885986, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10024620592594147, + "rewards/margins": 1.3349838256835938, + "rewards/rejected": -1.435230016708374, + "step": 1685 + }, + { + "epoch": 1.0488335925349923, + "grad_norm": 0.6142764687538147, + "learning_rate": 7.850000000000001e-06, + "log_odds_chosen": 10.19394302368164, + "log_odds_ratio": -0.004159958567470312, + "logits/chosen": 0.2460165172815323, + "logits/rejected": 1.1073980331420898, + "logps/chosen": -0.8783658742904663, + "logps/rejected": -10.400548934936523, + "loss": 2.5735, + "nll_loss": 2.573049545288086, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08783659338951111, + "rewards/margins": 0.952218234539032, + "rewards/rejected": -1.0400547981262207, + "step": 1686 + }, + { + "epoch": 1.0494556765163296, + "grad_norm": 0.6551894545555115, + "learning_rate": 7.825e-06, + "log_odds_chosen": 10.567734718322754, + "log_odds_ratio": -0.010052100755274296, + "logits/chosen": 0.32441431283950806, + "logits/rejected": 0.8118258714675903, + "logps/chosen": -1.1712175607681274, + "logps/rejected": -10.970331192016602, + "loss": 3.2066, + "nll_loss": 3.2055797576904297, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11712174862623215, + "rewards/margins": 0.9799113273620605, + "rewards/rejected": -1.097033143043518, + "step": 1687 + }, + { + "epoch": 1.0500777604976672, + "grad_norm": 1.7960695028305054, + "learning_rate": 7.8e-06, + "log_odds_chosen": 8.924139022827148, + "log_odds_ratio": -0.19967907667160034, + "logits/chosen": 0.18073192238807678, + "logits/rejected": 0.5033700466156006, + "logps/chosen": -0.9381715059280396, + "logps/rejected": -9.414775848388672, + "loss": 2.5335, + "nll_loss": 2.513486862182617, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09381714463233948, + "rewards/margins": 0.8476604223251343, + "rewards/rejected": -0.9414775371551514, + "step": 1688 + }, + { + "epoch": 1.0506998444790048, + "grad_norm": 0.657584011554718, + "learning_rate": 7.775000000000001e-06, + "log_odds_chosen": 8.136945724487305, + "log_odds_ratio": -0.18999497592449188, + "logits/chosen": 0.2717944085597992, + "logits/rejected": 0.5513604283332825, + "logps/chosen": -1.0118772983551025, + "logps/rejected": -8.757270812988281, + "loss": 2.5468, + "nll_loss": 2.5278446674346924, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10118772834539413, + "rewards/margins": 0.7745393514633179, + "rewards/rejected": -0.8757270574569702, + "step": 1689 + }, + { + "epoch": 1.0513219284603421, + "grad_norm": 0.7142519950866699, + "learning_rate": 7.75e-06, + "log_odds_chosen": 13.280343055725098, + "log_odds_ratio": -0.0804281011223793, + "logits/chosen": 0.40052324533462524, + "logits/rejected": 1.358069896697998, + "logps/chosen": -0.9685522317886353, + "logps/rejected": -13.64314079284668, + "loss": 2.8942, + "nll_loss": 2.8861801624298096, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09685522317886353, + "rewards/margins": 1.2674589157104492, + "rewards/rejected": -1.364314079284668, + "step": 1690 + }, + { + "epoch": 1.0519440124416797, + "grad_norm": 0.5332468748092651, + "learning_rate": 7.725e-06, + "log_odds_chosen": 8.306745529174805, + "log_odds_ratio": -0.25401604175567627, + "logits/chosen": 0.29576265811920166, + "logits/rejected": 0.6935917735099792, + "logps/chosen": -1.3476217985153198, + "logps/rejected": -9.482418060302734, + "loss": 3.489, + "nll_loss": 3.463611125946045, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13476218283176422, + "rewards/margins": 0.8134796619415283, + "rewards/rejected": -0.9482417702674866, + "step": 1691 + }, + { + "epoch": 1.052566096423017, + "grad_norm": 0.5067847967147827, + "learning_rate": 7.7e-06, + "log_odds_chosen": 11.327463150024414, + "log_odds_ratio": -0.09150741249322891, + "logits/chosen": 0.3421590328216553, + "logits/rejected": 0.8589677810668945, + "logps/chosen": -1.1448135375976562, + "logps/rejected": -12.03975772857666, + "loss": 3.1256, + "nll_loss": 3.116487979888916, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1144813597202301, + "rewards/margins": 1.0894944667816162, + "rewards/rejected": -1.2039759159088135, + "step": 1692 + }, + { + "epoch": 1.0531881804043546, + "grad_norm": 0.45336204767227173, + "learning_rate": 7.675e-06, + "log_odds_chosen": 7.9443793296813965, + "log_odds_ratio": -0.08496344089508057, + "logits/chosen": 0.3444047272205353, + "logits/rejected": 0.8683087229728699, + "logps/chosen": -1.3129093647003174, + "logps/rejected": -8.973153114318848, + "loss": 3.3978, + "nll_loss": 3.3893444538116455, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13129094243049622, + "rewards/margins": 0.7660244107246399, + "rewards/rejected": -0.8973153233528137, + "step": 1693 + }, + { + "epoch": 1.0538102643856921, + "grad_norm": 10.097654342651367, + "learning_rate": 7.65e-06, + "log_odds_chosen": 9.851930618286133, + "log_odds_ratio": -0.30158087611198425, + "logits/chosen": 0.27799010276794434, + "logits/rejected": 0.9372337460517883, + "logps/chosen": -2.7150497436523438, + "logps/rejected": -12.091753005981445, + "loss": 3.1269, + "nll_loss": 3.0967228412628174, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.2715049684047699, + "rewards/margins": 0.9376704096794128, + "rewards/rejected": -1.2091753482818604, + "step": 1694 + }, + { + "epoch": 1.0544323483670295, + "grad_norm": 0.6404469609260559, + "learning_rate": 7.625e-06, + "log_odds_chosen": 11.932069778442383, + "log_odds_ratio": -0.016780562698841095, + "logits/chosen": 0.2930798828601837, + "logits/rejected": 0.8748061656951904, + "logps/chosen": -0.9199110269546509, + "logps/rejected": -12.094552993774414, + "loss": 2.7043, + "nll_loss": 2.7026607990264893, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09199110418558121, + "rewards/margins": 1.117464303970337, + "rewards/rejected": -1.2094552516937256, + "step": 1695 + }, + { + "epoch": 1.055054432348367, + "grad_norm": 0.672562301158905, + "learning_rate": 7.6e-06, + "log_odds_chosen": 12.345985412597656, + "log_odds_ratio": -0.01769334264099598, + "logits/chosen": 0.16025392711162567, + "logits/rejected": 0.7841507792472839, + "logps/chosen": -0.9052579998970032, + "logps/rejected": -12.281292915344238, + "loss": 2.5574, + "nll_loss": 2.5555901527404785, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0905258059501648, + "rewards/margins": 1.137603521347046, + "rewards/rejected": -1.2281293869018555, + "step": 1696 + }, + { + "epoch": 1.0556765163297046, + "grad_norm": 0.7864720225334167, + "learning_rate": 7.575e-06, + "log_odds_chosen": 6.648435115814209, + "log_odds_ratio": -0.21117697656154633, + "logits/chosen": 0.25911492109298706, + "logits/rejected": 0.46124711632728577, + "logps/chosen": -1.0256266593933105, + "logps/rejected": -7.215675354003906, + "loss": 3.2426, + "nll_loss": 3.2214407920837402, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10256266593933105, + "rewards/margins": 0.6190049052238464, + "rewards/rejected": -0.7215675115585327, + "step": 1697 + }, + { + "epoch": 1.056298600311042, + "grad_norm": 0.5210472345352173, + "learning_rate": 7.55e-06, + "log_odds_chosen": 6.7975172996521, + "log_odds_ratio": -0.24025194346904755, + "logits/chosen": 0.10606539249420166, + "logits/rejected": 0.5300160050392151, + "logps/chosen": -1.0408447980880737, + "logps/rejected": -7.56657600402832, + "loss": 2.8331, + "nll_loss": 2.8090970516204834, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10408448427915573, + "rewards/margins": 0.6525731086730957, + "rewards/rejected": -0.7566576600074768, + "step": 1698 + }, + { + "epoch": 1.0569206842923795, + "grad_norm": 1.3864878416061401, + "learning_rate": 7.525e-06, + "log_odds_chosen": 5.305638313293457, + "log_odds_ratio": -0.3785892724990845, + "logits/chosen": 0.11386309564113617, + "logits/rejected": 0.3513365089893341, + "logps/chosen": -1.1088687181472778, + "logps/rejected": -6.139446258544922, + "loss": 2.6211, + "nll_loss": 2.5832529067993164, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11088687181472778, + "rewards/margins": 0.5030577182769775, + "rewards/rejected": -0.6139446496963501, + "step": 1699 + }, + { + "epoch": 1.0575427682737168, + "grad_norm": 0.5979633927345276, + "learning_rate": 7.5e-06, + "log_odds_chosen": 8.369409561157227, + "log_odds_ratio": -0.044509388506412506, + "logits/chosen": 0.23924407362937927, + "logits/rejected": 0.7259291410446167, + "logps/chosen": -0.8297702670097351, + "logps/rejected": -8.520917892456055, + "loss": 2.8939, + "nll_loss": 2.889413595199585, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08297702670097351, + "rewards/margins": 0.7691147923469543, + "rewards/rejected": -0.8520918488502502, + "step": 1700 + }, + { + "epoch": 1.0581648522550544, + "grad_norm": 0.4679238796234131, + "learning_rate": 7.4750000000000004e-06, + "log_odds_chosen": 10.700567245483398, + "log_odds_ratio": -0.007277395576238632, + "logits/chosen": 0.2523863613605499, + "logits/rejected": 0.8801952600479126, + "logps/chosen": -1.0980184078216553, + "logps/rejected": -11.33210563659668, + "loss": 3.0063, + "nll_loss": 3.005614757537842, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10980185121297836, + "rewards/margins": 1.0234086513519287, + "rewards/rejected": -1.1332106590270996, + "step": 1701 + }, + { + "epoch": 1.058786936236392, + "grad_norm": 0.5518056154251099, + "learning_rate": 7.45e-06, + "log_odds_chosen": 8.354429244995117, + "log_odds_ratio": -0.1485568732023239, + "logits/chosen": 0.1999821811914444, + "logits/rejected": 0.6293431520462036, + "logps/chosen": -0.9266054034233093, + "logps/rejected": -8.75076961517334, + "loss": 2.8133, + "nll_loss": 2.79842472076416, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09266054630279541, + "rewards/margins": 0.7824164628982544, + "rewards/rejected": -0.8750770092010498, + "step": 1702 + }, + { + "epoch": 1.0594090202177293, + "grad_norm": 0.48427486419677734, + "learning_rate": 7.425e-06, + "log_odds_chosen": 9.299489974975586, + "log_odds_ratio": -0.12962597608566284, + "logits/chosen": 0.3212866485118866, + "logits/rejected": 0.7380868196487427, + "logps/chosen": -0.8832244277000427, + "logps/rejected": -9.699041366577148, + "loss": 3.5157, + "nll_loss": 3.5027828216552734, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08832244575023651, + "rewards/margins": 0.8815817832946777, + "rewards/rejected": -0.9699042439460754, + "step": 1703 + }, + { + "epoch": 1.0600311041990669, + "grad_norm": 0.5466374754905701, + "learning_rate": 7.4e-06, + "log_odds_chosen": 12.918460845947266, + "log_odds_ratio": -0.004811026621609926, + "logits/chosen": 0.23254846036434174, + "logits/rejected": 1.191979169845581, + "logps/chosen": -0.8864061832427979, + "logps/rejected": -13.163785934448242, + "loss": 2.6641, + "nll_loss": 2.663635015487671, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08864062279462814, + "rewards/margins": 1.2277379035949707, + "rewards/rejected": -1.3163785934448242, + "step": 1704 + }, + { + "epoch": 1.0606531881804044, + "grad_norm": 0.6290574669837952, + "learning_rate": 7.375e-06, + "log_odds_chosen": 10.326766967773438, + "log_odds_ratio": -0.010386736132204533, + "logits/chosen": 0.3104044198989868, + "logits/rejected": 1.3550554513931274, + "logps/chosen": -1.036363124847412, + "logps/rejected": -10.91328239440918, + "loss": 2.2605, + "nll_loss": 2.25947642326355, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10363630950450897, + "rewards/margins": 0.9876919984817505, + "rewards/rejected": -1.0913283824920654, + "step": 1705 + }, + { + "epoch": 1.0612752721617418, + "grad_norm": 0.6645586490631104, + "learning_rate": 7.35e-06, + "log_odds_chosen": 4.954842567443848, + "log_odds_ratio": -0.29886144399642944, + "logits/chosen": 0.36008620262145996, + "logits/rejected": 0.6080492734909058, + "logps/chosen": -0.9095826745033264, + "logps/rejected": -5.422713279724121, + "loss": 3.1555, + "nll_loss": 3.1255886554718018, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09095826745033264, + "rewards/margins": 0.45131298899650574, + "rewards/rejected": -0.5422712564468384, + "step": 1706 + }, + { + "epoch": 1.0618973561430793, + "grad_norm": 0.5843470096588135, + "learning_rate": 7.325e-06, + "log_odds_chosen": 8.344114303588867, + "log_odds_ratio": -0.15374304354190826, + "logits/chosen": 0.26468321681022644, + "logits/rejected": 0.9158927202224731, + "logps/chosen": -1.0647417306900024, + "logps/rejected": -9.070698738098145, + "loss": 2.8046, + "nll_loss": 2.789252996444702, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10647417604923248, + "rewards/margins": 0.800595760345459, + "rewards/rejected": -0.9070699214935303, + "step": 1707 + }, + { + "epoch": 1.062519440124417, + "grad_norm": 0.5199341773986816, + "learning_rate": 7.2999999999999996e-06, + "log_odds_chosen": 11.067071914672852, + "log_odds_ratio": -0.00585298053920269, + "logits/chosen": 0.23164747655391693, + "logits/rejected": 0.9034661650657654, + "logps/chosen": -1.055686116218567, + "logps/rejected": -11.569263458251953, + "loss": 2.746, + "nll_loss": 2.7453653812408447, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10556861758232117, + "rewards/margins": 1.0513577461242676, + "rewards/rejected": -1.1569262742996216, + "step": 1708 + }, + { + "epoch": 1.0631415241057542, + "grad_norm": 0.5655837655067444, + "learning_rate": 7.275e-06, + "log_odds_chosen": 11.287644386291504, + "log_odds_ratio": -0.0015687721315771341, + "logits/chosen": 0.289143830537796, + "logits/rejected": 0.7014958262443542, + "logps/chosen": -1.0149272680282593, + "logps/rejected": -11.732308387756348, + "loss": 3.1581, + "nll_loss": 3.157970905303955, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.101492740213871, + "rewards/margins": 1.0717381238937378, + "rewards/rejected": -1.1732308864593506, + "step": 1709 + }, + { + "epoch": 1.0637636080870918, + "grad_norm": 0.5841426849365234, + "learning_rate": 7.25e-06, + "log_odds_chosen": 13.971756935119629, + "log_odds_ratio": -0.0001850881817517802, + "logits/chosen": 0.5528026819229126, + "logits/rejected": 1.5337713956832886, + "logps/chosen": -1.308842420578003, + "logps/rejected": -14.919255256652832, + "loss": 3.5475, + "nll_loss": 3.547466993331909, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13088424503803253, + "rewards/margins": 1.3610413074493408, + "rewards/rejected": -1.491925597190857, + "step": 1710 + }, + { + "epoch": 1.0643856920684291, + "grad_norm": 0.602485179901123, + "learning_rate": 7.2249999999999994e-06, + "log_odds_chosen": 10.762537956237793, + "log_odds_ratio": -0.018083693459630013, + "logits/chosen": 0.3396477699279785, + "logits/rejected": 1.0870765447616577, + "logps/chosen": -0.8417822122573853, + "logps/rejected": -10.953207015991211, + "loss": 3.0177, + "nll_loss": 3.015902042388916, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08417822420597076, + "rewards/margins": 1.0111424922943115, + "rewards/rejected": -1.0953208208084106, + "step": 1711 + }, + { + "epoch": 1.0650077760497667, + "grad_norm": 1.0022923946380615, + "learning_rate": 7.2e-06, + "log_odds_chosen": 9.395156860351562, + "log_odds_ratio": -0.17347288131713867, + "logits/chosen": 0.23961828649044037, + "logits/rejected": 1.10887610912323, + "logps/chosen": -1.5142661333084106, + "logps/rejected": -10.611567497253418, + "loss": 2.423, + "nll_loss": 2.4056031703948975, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15142662823200226, + "rewards/margins": 0.909730076789856, + "rewards/rejected": -1.0611567497253418, + "step": 1712 + }, + { + "epoch": 1.0656298600311043, + "grad_norm": 0.7605722546577454, + "learning_rate": 7.175e-06, + "log_odds_chosen": 9.183586120605469, + "log_odds_ratio": -0.20111922919750214, + "logits/chosen": 0.30907413363456726, + "logits/rejected": 0.8606806993484497, + "logps/chosen": -1.424899697303772, + "logps/rejected": -10.347264289855957, + "loss": 3.4657, + "nll_loss": 3.4456028938293457, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1424899697303772, + "rewards/margins": 0.8922364711761475, + "rewards/rejected": -1.0347263813018799, + "step": 1713 + }, + { + "epoch": 1.0662519440124416, + "grad_norm": 0.3997776508331299, + "learning_rate": 7.15e-06, + "log_odds_chosen": 11.275423049926758, + "log_odds_ratio": -0.009868920780718327, + "logits/chosen": 0.34685301780700684, + "logits/rejected": 1.0028785467147827, + "logps/chosen": -1.0552277565002441, + "logps/rejected": -11.810014724731445, + "loss": 3.5462, + "nll_loss": 3.5452332496643066, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10552278161048889, + "rewards/margins": 1.0754787921905518, + "rewards/rejected": -1.1810015439987183, + "step": 1714 + }, + { + "epoch": 1.0668740279937792, + "grad_norm": 0.4696863889694214, + "learning_rate": 7.1249999999999995e-06, + "log_odds_chosen": 10.42939567565918, + "log_odds_ratio": -0.0563388392329216, + "logits/chosen": 0.36971479654312134, + "logits/rejected": 1.1650364398956299, + "logps/chosen": -1.241980791091919, + "logps/rejected": -11.253006935119629, + "loss": 3.3682, + "nll_loss": 3.3625478744506836, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1241980716586113, + "rewards/margins": 1.0011026859283447, + "rewards/rejected": -1.1253007650375366, + "step": 1715 + }, + { + "epoch": 1.0674961119751167, + "grad_norm": 0.8792657256126404, + "learning_rate": 7.1e-06, + "log_odds_chosen": 9.294808387756348, + "log_odds_ratio": -0.03534835949540138, + "logits/chosen": 0.25297093391418457, + "logits/rejected": 0.5003412365913391, + "logps/chosen": -0.8526089191436768, + "logps/rejected": -9.356329917907715, + "loss": 2.3576, + "nll_loss": 2.354109287261963, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08526089787483215, + "rewards/margins": 0.8503720760345459, + "rewards/rejected": -0.9356330037117004, + "step": 1716 + }, + { + "epoch": 1.068118195956454, + "grad_norm": 0.521109402179718, + "learning_rate": 7.075e-06, + "log_odds_chosen": 14.6859712600708, + "log_odds_ratio": -0.00011647976498352364, + "logits/chosen": 0.1722133308649063, + "logits/rejected": 0.9801969528198242, + "logps/chosen": -1.2429990768432617, + "logps/rejected": -15.523116111755371, + "loss": 3.2801, + "nll_loss": 3.2801103591918945, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12429991364479065, + "rewards/margins": 1.4280118942260742, + "rewards/rejected": -1.552311658859253, + "step": 1717 + }, + { + "epoch": 1.0687402799377916, + "grad_norm": 0.5586221218109131, + "learning_rate": 7.049999999999999e-06, + "log_odds_chosen": 7.838666915893555, + "log_odds_ratio": -0.08945189416408539, + "logits/chosen": 0.2482565939426422, + "logits/rejected": 0.694401741027832, + "logps/chosen": -1.0721337795257568, + "logps/rejected": -8.161520004272461, + "loss": 2.4886, + "nll_loss": 2.479682445526123, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10721337050199509, + "rewards/margins": 0.7089386582374573, + "rewards/rejected": -0.8161520957946777, + "step": 1718 + }, + { + "epoch": 1.069362363919129, + "grad_norm": 0.5216894745826721, + "learning_rate": 7.025000000000001e-06, + "log_odds_chosen": 11.799904823303223, + "log_odds_ratio": -0.002399260178208351, + "logits/chosen": 0.2988235354423523, + "logits/rejected": 1.5455994606018066, + "logps/chosen": -1.177297592163086, + "logps/rejected": -12.587946891784668, + "loss": 2.7591, + "nll_loss": 2.7588682174682617, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11772976815700531, + "rewards/margins": 1.141065001487732, + "rewards/rejected": -1.2587947845458984, + "step": 1719 + }, + { + "epoch": 1.0699844479004665, + "grad_norm": 0.6003878712654114, + "learning_rate": 7.000000000000001e-06, + "log_odds_chosen": 8.443526268005371, + "log_odds_ratio": -0.12604990601539612, + "logits/chosen": 0.3089401125907898, + "logits/rejected": 1.1080322265625, + "logps/chosen": -1.531690239906311, + "logps/rejected": -9.769110679626465, + "loss": 3.1147, + "nll_loss": 3.1020569801330566, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15316902101039886, + "rewards/margins": 0.8237420320510864, + "rewards/rejected": -0.9769110679626465, + "step": 1720 + }, + { + "epoch": 1.070606531881804, + "grad_norm": 0.5633937120437622, + "learning_rate": 6.975000000000001e-06, + "log_odds_chosen": 12.794734001159668, + "log_odds_ratio": -0.0027537632267922163, + "logits/chosen": 0.21119673550128937, + "logits/rejected": 1.5868358612060547, + "logps/chosen": -1.1883983612060547, + "logps/rejected": -13.517034530639648, + "loss": 2.517, + "nll_loss": 2.5166783332824707, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11883984506130219, + "rewards/margins": 1.2328636646270752, + "rewards/rejected": -1.351703405380249, + "step": 1721 + }, + { + "epoch": 1.0712286158631414, + "grad_norm": 0.45499175786972046, + "learning_rate": 6.950000000000001e-06, + "log_odds_chosen": 16.033639907836914, + "log_odds_ratio": -0.00026129043544642627, + "logits/chosen": 0.09925095736980438, + "logits/rejected": 0.7012900114059448, + "logps/chosen": -1.0431275367736816, + "logps/rejected": -16.531070709228516, + "loss": 2.851, + "nll_loss": 2.8509817123413086, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10431274771690369, + "rewards/margins": 1.5487942695617676, + "rewards/rejected": -1.653106927871704, + "step": 1722 + }, + { + "epoch": 1.071850699844479, + "grad_norm": 0.5466725826263428, + "learning_rate": 6.925000000000001e-06, + "log_odds_chosen": 11.074979782104492, + "log_odds_ratio": -0.04315614700317383, + "logits/chosen": 0.14270518720149994, + "logits/rejected": 0.8532730340957642, + "logps/chosen": -0.9979894161224365, + "logps/rejected": -11.582146644592285, + "loss": 2.9812, + "nll_loss": 2.976850986480713, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09979893267154694, + "rewards/margins": 1.0584158897399902, + "rewards/rejected": -1.158214807510376, + "step": 1723 + }, + { + "epoch": 1.0724727838258166, + "grad_norm": 0.4190523326396942, + "learning_rate": 6.900000000000001e-06, + "log_odds_chosen": 13.74249267578125, + "log_odds_ratio": -0.01285548135638237, + "logits/chosen": 0.2155817151069641, + "logits/rejected": 0.9092065691947937, + "logps/chosen": -0.9522172212600708, + "logps/rejected": -14.18885612487793, + "loss": 3.0457, + "nll_loss": 3.0444235801696777, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09522172808647156, + "rewards/margins": 1.3236639499664307, + "rewards/rejected": -1.4188857078552246, + "step": 1724 + }, + { + "epoch": 1.073094867807154, + "grad_norm": 1.1821922063827515, + "learning_rate": 6.875000000000001e-06, + "log_odds_chosen": 13.672684669494629, + "log_odds_ratio": -0.08438721299171448, + "logits/chosen": 0.347389817237854, + "logits/rejected": 1.5360487699508667, + "logps/chosen": -1.1511101722717285, + "logps/rejected": -14.482137680053711, + "loss": 2.6498, + "nll_loss": 2.6413512229919434, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11511103063821793, + "rewards/margins": 1.333102822303772, + "rewards/rejected": -1.448213815689087, + "step": 1725 + }, + { + "epoch": 1.0737169517884915, + "grad_norm": 0.5827188491821289, + "learning_rate": 6.8500000000000005e-06, + "log_odds_chosen": 13.990209579467773, + "log_odds_ratio": -0.07931892573833466, + "logits/chosen": 0.3765639662742615, + "logits/rejected": 1.4914324283599854, + "logps/chosen": -0.8575527667999268, + "logps/rejected": -14.12916374206543, + "loss": 3.0849, + "nll_loss": 3.076958179473877, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08575528115034103, + "rewards/margins": 1.3271610736846924, + "rewards/rejected": -1.4129164218902588, + "step": 1726 + }, + { + "epoch": 1.074339035769829, + "grad_norm": 0.6240293383598328, + "learning_rate": 6.825000000000001e-06, + "log_odds_chosen": 15.126672744750977, + "log_odds_ratio": -0.0863954946398735, + "logits/chosen": 0.1938783824443817, + "logits/rejected": 1.443101167678833, + "logps/chosen": -1.1526148319244385, + "logps/rejected": -15.915487289428711, + "loss": 2.5008, + "nll_loss": 2.4921345710754395, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11526148021221161, + "rewards/margins": 1.4762872457504272, + "rewards/rejected": -1.5915489196777344, + "step": 1727 + }, + { + "epoch": 1.0749611197511664, + "grad_norm": 9.35839557647705, + "learning_rate": 6.800000000000001e-06, + "log_odds_chosen": 12.475717544555664, + "log_odds_ratio": -0.039579086005687714, + "logits/chosen": 0.3538586497306824, + "logits/rejected": 1.454434871673584, + "logps/chosen": -1.897247552871704, + "logps/rejected": -14.156525611877441, + "loss": 3.1319, + "nll_loss": 3.1279637813568115, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18972477316856384, + "rewards/margins": 1.225927710533142, + "rewards/rejected": -1.4156526327133179, + "step": 1728 + }, + { + "epoch": 1.075583203732504, + "grad_norm": 1.0323598384857178, + "learning_rate": 6.775000000000001e-06, + "log_odds_chosen": 16.044645309448242, + "log_odds_ratio": -0.0008165925974026322, + "logits/chosen": 0.22339317202568054, + "logits/rejected": 1.895038366317749, + "logps/chosen": -1.209045648574829, + "logps/rejected": -16.88253402709961, + "loss": 2.3606, + "nll_loss": 2.3605496883392334, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12090456485748291, + "rewards/margins": 1.5673489570617676, + "rewards/rejected": -1.6882535219192505, + "step": 1729 + }, + { + "epoch": 1.0762052877138413, + "grad_norm": 0.4422605335712433, + "learning_rate": 6.750000000000001e-06, + "log_odds_chosen": 13.798027992248535, + "log_odds_ratio": -6.737739749951288e-05, + "logits/chosen": 0.34486210346221924, + "logits/rejected": 1.331770420074463, + "logps/chosen": -1.1659202575683594, + "logps/rejected": -14.552068710327148, + "loss": 3.3028, + "nll_loss": 3.302793502807617, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11659201979637146, + "rewards/margins": 1.3386149406433105, + "rewards/rejected": -1.4552068710327148, + "step": 1730 + }, + { + "epoch": 1.0768273716951788, + "grad_norm": 0.6502666473388672, + "learning_rate": 6.725000000000001e-06, + "log_odds_chosen": 12.585850715637207, + "log_odds_ratio": -0.11541411280632019, + "logits/chosen": 0.11757244169712067, + "logits/rejected": 0.8546798229217529, + "logps/chosen": -1.0822380781173706, + "logps/rejected": -13.226578712463379, + "loss": 2.4227, + "nll_loss": 2.411139965057373, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1082238182425499, + "rewards/margins": 1.2144341468811035, + "rewards/rejected": -1.3226580619812012, + "step": 1731 + }, + { + "epoch": 1.0774494556765164, + "grad_norm": 0.5105128288269043, + "learning_rate": 6.700000000000001e-06, + "log_odds_chosen": 15.417603492736816, + "log_odds_ratio": -0.000615728902630508, + "logits/chosen": 0.3644036054611206, + "logits/rejected": 1.0982917547225952, + "logps/chosen": -1.0642772912979126, + "logps/rejected": -15.99770450592041, + "loss": 3.3102, + "nll_loss": 3.3101632595062256, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10642772912979126, + "rewards/margins": 1.493342638015747, + "rewards/rejected": -1.5997705459594727, + "step": 1732 + }, + { + "epoch": 1.0780715396578537, + "grad_norm": 0.5010104179382324, + "learning_rate": 6.6750000000000005e-06, + "log_odds_chosen": 10.825895309448242, + "log_odds_ratio": -0.10436728596687317, + "logits/chosen": 0.10802976787090302, + "logits/rejected": 0.7807521820068359, + "logps/chosen": -1.1920297145843506, + "logps/rejected": -11.657849311828613, + "loss": 3.0259, + "nll_loss": 3.0154356956481934, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11920297890901566, + "rewards/margins": 1.0465819835662842, + "rewards/rejected": -1.1657850742340088, + "step": 1733 + }, + { + "epoch": 1.0786936236391913, + "grad_norm": 0.5057395100593567, + "learning_rate": 6.650000000000001e-06, + "log_odds_chosen": 11.468727111816406, + "log_odds_ratio": -0.12807990610599518, + "logits/chosen": 0.13424773514270782, + "logits/rejected": 0.758084237575531, + "logps/chosen": -0.8967731595039368, + "logps/rejected": -11.590532302856445, + "loss": 2.8589, + "nll_loss": 2.846139430999756, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08967731893062592, + "rewards/margins": 1.0693758726119995, + "rewards/rejected": -1.159053087234497, + "step": 1734 + }, + { + "epoch": 1.0793157076205289, + "grad_norm": 0.6509506702423096, + "learning_rate": 6.625000000000001e-06, + "log_odds_chosen": 10.882062911987305, + "log_odds_ratio": -0.012730767950415611, + "logits/chosen": 0.44164884090423584, + "logits/rejected": 1.0146621465682983, + "logps/chosen": -1.0620293617248535, + "logps/rejected": -11.419351577758789, + "loss": 3.15, + "nll_loss": 3.14876651763916, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10620293766260147, + "rewards/margins": 1.0357321500778198, + "rewards/rejected": -1.141935110092163, + "step": 1735 + }, + { + "epoch": 1.0799377916018662, + "grad_norm": 10.22143840789795, + "learning_rate": 6.6e-06, + "log_odds_chosen": 12.94996452331543, + "log_odds_ratio": -0.32687464356422424, + "logits/chosen": 0.186806783080101, + "logits/rejected": 0.8277633190155029, + "logps/chosen": -1.3755271434783936, + "logps/rejected": -14.012967109680176, + "loss": 2.9523, + "nll_loss": 2.9196386337280273, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13755272328853607, + "rewards/margins": 1.2637439966201782, + "rewards/rejected": -1.4012967348098755, + "step": 1736 + }, + { + "epoch": 1.0805598755832038, + "grad_norm": 0.6058777570724487, + "learning_rate": 6.5750000000000006e-06, + "log_odds_chosen": 10.573338508605957, + "log_odds_ratio": -0.22641825675964355, + "logits/chosen": 0.17677393555641174, + "logits/rejected": 0.6988618969917297, + "logps/chosen": -1.1388049125671387, + "logps/rejected": -11.44548225402832, + "loss": 2.9047, + "nll_loss": 2.882017135620117, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11388048529624939, + "rewards/margins": 1.030667781829834, + "rewards/rejected": -1.1445482969284058, + "step": 1737 + }, + { + "epoch": 1.081181959564541, + "grad_norm": 0.55973881483078, + "learning_rate": 6.550000000000001e-06, + "log_odds_chosen": 11.361783981323242, + "log_odds_ratio": -0.025486905127763748, + "logits/chosen": 0.2761267125606537, + "logits/rejected": 1.423222303390503, + "logps/chosen": -1.079453468322754, + "logps/rejected": -12.022642135620117, + "loss": 2.6204, + "nll_loss": 2.6178572177886963, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10794536024332047, + "rewards/margins": 1.0943188667297363, + "rewards/rejected": -1.2022643089294434, + "step": 1738 + }, + { + "epoch": 1.0818040435458787, + "grad_norm": 0.41827815771102905, + "learning_rate": 6.525e-06, + "log_odds_chosen": 11.45641040802002, + "log_odds_ratio": -0.04113250598311424, + "logits/chosen": 0.28797030448913574, + "logits/rejected": 0.9413681030273438, + "logps/chosen": -1.1444040536880493, + "logps/rejected": -12.222957611083984, + "loss": 3.2833, + "nll_loss": 3.279160261154175, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11444041132926941, + "rewards/margins": 1.1078553199768066, + "rewards/rejected": -1.2222957611083984, + "step": 1739 + }, + { + "epoch": 1.0824261275272162, + "grad_norm": 0.5239333510398865, + "learning_rate": 6.5000000000000004e-06, + "log_odds_chosen": 10.370500564575195, + "log_odds_ratio": -0.010260947048664093, + "logits/chosen": 0.1700163036584854, + "logits/rejected": 0.7396705150604248, + "logps/chosen": -0.9648128151893616, + "logps/rejected": -10.788455963134766, + "loss": 2.8972, + "nll_loss": 2.8962013721466064, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09648127853870392, + "rewards/margins": 0.9823644161224365, + "rewards/rejected": -1.078845739364624, + "step": 1740 + }, + { + "epoch": 1.0830482115085536, + "grad_norm": 0.5762307643890381, + "learning_rate": 6.475000000000001e-06, + "log_odds_chosen": 6.935182571411133, + "log_odds_ratio": -0.08945510536432266, + "logits/chosen": 0.16429941356182098, + "logits/rejected": 0.6121139526367188, + "logps/chosen": -0.9953298568725586, + "logps/rejected": -7.245857238769531, + "loss": 2.8649, + "nll_loss": 2.8559913635253906, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09953299164772034, + "rewards/margins": 0.625052809715271, + "rewards/rejected": -0.7245857119560242, + "step": 1741 + }, + { + "epoch": 1.0836702954898911, + "grad_norm": 0.7302781343460083, + "learning_rate": 6.45e-06, + "log_odds_chosen": 13.287824630737305, + "log_odds_ratio": -0.07792376726865768, + "logits/chosen": 0.07993796467781067, + "logits/rejected": 1.3684005737304688, + "logps/chosen": -1.4768130779266357, + "logps/rejected": -14.53031063079834, + "loss": 2.8254, + "nll_loss": 2.8176429271698, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1476813107728958, + "rewards/margins": 1.3053498268127441, + "rewards/rejected": -1.453031063079834, + "step": 1742 + }, + { + "epoch": 1.0842923794712287, + "grad_norm": 0.5578170418739319, + "learning_rate": 6.425e-06, + "log_odds_chosen": 13.632564544677734, + "log_odds_ratio": -0.06082217022776604, + "logits/chosen": 0.33293527364730835, + "logits/rejected": 1.096616506576538, + "logps/chosen": -0.7652254700660706, + "logps/rejected": -13.640064239501953, + "loss": 3.1935, + "nll_loss": 3.1874160766601562, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07652254402637482, + "rewards/margins": 1.2874839305877686, + "rewards/rejected": -1.364006519317627, + "step": 1743 + }, + { + "epoch": 1.084914463452566, + "grad_norm": 0.7817727327346802, + "learning_rate": 6.4000000000000006e-06, + "log_odds_chosen": 13.825263977050781, + "log_odds_ratio": -0.10829272121191025, + "logits/chosen": 0.18306320905685425, + "logits/rejected": 1.2139906883239746, + "logps/chosen": -1.0914957523345947, + "logps/rejected": -14.526753425598145, + "loss": 2.5395, + "nll_loss": 2.5286498069763184, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10914956778287888, + "rewards/margins": 1.343525767326355, + "rewards/rejected": -1.452675223350525, + "step": 1744 + }, + { + "epoch": 1.0855365474339036, + "grad_norm": 0.6816715598106384, + "learning_rate": 6.375000000000001e-06, + "log_odds_chosen": 8.512453079223633, + "log_odds_ratio": -0.23665356636047363, + "logits/chosen": 0.20159471035003662, + "logits/rejected": 0.7449344396591187, + "logps/chosen": -1.1736276149749756, + "logps/rejected": -9.346839904785156, + "loss": 2.8251, + "nll_loss": 2.8014824390411377, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11736276000738144, + "rewards/margins": 0.817321240901947, + "rewards/rejected": -0.9346840381622314, + "step": 1745 + }, + { + "epoch": 1.0861586314152412, + "grad_norm": 0.6403155326843262, + "learning_rate": 6.35e-06, + "log_odds_chosen": 14.663322448730469, + "log_odds_ratio": -0.0037884623743593693, + "logits/chosen": 0.279615193605423, + "logits/rejected": 1.2483148574829102, + "logps/chosen": -0.8240023851394653, + "logps/rejected": -14.657256126403809, + "loss": 2.7768, + "nll_loss": 2.7764101028442383, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08240024000406265, + "rewards/margins": 1.3833253383636475, + "rewards/rejected": -1.4657256603240967, + "step": 1746 + }, + { + "epoch": 1.0867807153965785, + "grad_norm": 0.5294352769851685, + "learning_rate": 6.3250000000000004e-06, + "log_odds_chosen": 7.447414398193359, + "log_odds_ratio": -0.08214883506298065, + "logits/chosen": 0.04716228321194649, + "logits/rejected": 0.14632737636566162, + "logps/chosen": -0.844534158706665, + "logps/rejected": -7.672712326049805, + "loss": 2.7177, + "nll_loss": 2.709484338760376, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08445341885089874, + "rewards/margins": 0.682817816734314, + "rewards/rejected": -0.7672712802886963, + "step": 1747 + }, + { + "epoch": 1.087402799377916, + "grad_norm": 0.5357289910316467, + "learning_rate": 6.300000000000001e-06, + "log_odds_chosen": 9.260400772094727, + "log_odds_ratio": -0.21293191611766815, + "logits/chosen": 0.44084519147872925, + "logits/rejected": 1.00531005859375, + "logps/chosen": -1.097980260848999, + "logps/rejected": -9.951017379760742, + "loss": 3.4195, + "nll_loss": 3.3982417583465576, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10979802906513214, + "rewards/margins": 0.8853037357330322, + "rewards/rejected": -0.9951016902923584, + "step": 1748 + }, + { + "epoch": 1.0880248833592534, + "grad_norm": 0.5099205374717712, + "learning_rate": 6.275e-06, + "log_odds_chosen": 10.247150421142578, + "log_odds_ratio": -0.09444563835859299, + "logits/chosen": 0.3867758512496948, + "logits/rejected": 1.0003869533538818, + "logps/chosen": -1.117705225944519, + "logps/rejected": -10.993670463562012, + "loss": 3.3419, + "nll_loss": 3.332432985305786, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11177052557468414, + "rewards/margins": 0.9875965714454651, + "rewards/rejected": -1.0993670225143433, + "step": 1749 + }, + { + "epoch": 1.088646967340591, + "grad_norm": 0.7787434458732605, + "learning_rate": 6.25e-06, + "log_odds_chosen": 9.081072807312012, + "log_odds_ratio": -0.27854081988334656, + "logits/chosen": 0.20520712435245514, + "logits/rejected": 0.7103989124298096, + "logps/chosen": -1.0034761428833008, + "logps/rejected": -9.524426460266113, + "loss": 2.9065, + "nll_loss": 2.8786349296569824, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1003476157784462, + "rewards/margins": 0.8520950675010681, + "rewards/rejected": -0.9524427056312561, + "step": 1750 + }, + { + "epoch": 1.0892690513219285, + "grad_norm": 0.5483114719390869, + "learning_rate": 6.2250000000000005e-06, + "log_odds_chosen": 14.3975191116333, + "log_odds_ratio": -0.0014387951232492924, + "logits/chosen": 0.19464844465255737, + "logits/rejected": 0.9625054597854614, + "logps/chosen": -0.960172176361084, + "logps/rejected": -14.856963157653809, + "loss": 2.7555, + "nll_loss": 2.7553367614746094, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09601722657680511, + "rewards/margins": 1.389679193496704, + "rewards/rejected": -1.4856964349746704, + "step": 1751 + }, + { + "epoch": 1.0898911353032659, + "grad_norm": 0.7367234230041504, + "learning_rate": 6.2e-06, + "log_odds_chosen": 11.96536922454834, + "log_odds_ratio": -0.005536233074963093, + "logits/chosen": 0.3735648989677429, + "logits/rejected": 0.7411990761756897, + "logps/chosen": -1.0733413696289062, + "logps/rejected": -12.566356658935547, + "loss": 2.6859, + "nll_loss": 2.6853647232055664, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10733415186405182, + "rewards/margins": 1.1493016481399536, + "rewards/rejected": -1.2566356658935547, + "step": 1752 + }, + { + "epoch": 1.0905132192846034, + "grad_norm": 0.5547901391983032, + "learning_rate": 6.175e-06, + "log_odds_chosen": 14.394362449645996, + "log_odds_ratio": -0.08104939758777618, + "logits/chosen": 0.31781625747680664, + "logits/rejected": 1.255645751953125, + "logps/chosen": -1.1181135177612305, + "logps/rejected": -15.128317832946777, + "loss": 2.5999, + "nll_loss": 2.5917768478393555, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11181135475635529, + "rewards/margins": 1.4010204076766968, + "rewards/rejected": -1.512831687927246, + "step": 1753 + }, + { + "epoch": 1.091135303265941, + "grad_norm": 0.7209290862083435, + "learning_rate": 6.15e-06, + "log_odds_chosen": 13.295998573303223, + "log_odds_ratio": -0.05274088308215141, + "logits/chosen": 0.2680150866508484, + "logits/rejected": 0.8705407381057739, + "logps/chosen": -1.395832896232605, + "logps/rejected": -14.312849998474121, + "loss": 3.0833, + "nll_loss": 3.078002452850342, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1395832896232605, + "rewards/margins": 1.2917016744613647, + "rewards/rejected": -1.43128502368927, + "step": 1754 + }, + { + "epoch": 1.0917573872472783, + "grad_norm": 0.6560238003730774, + "learning_rate": 6.125e-06, + "log_odds_chosen": 12.961246490478516, + "log_odds_ratio": -0.12485496699810028, + "logits/chosen": 0.1595935821533203, + "logits/rejected": 1.1436913013458252, + "logps/chosen": -1.2793207168579102, + "logps/rejected": -13.835441589355469, + "loss": 2.8327, + "nll_loss": 2.8201873302459717, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12793207168579102, + "rewards/margins": 1.2556120157241821, + "rewards/rejected": -1.3835442066192627, + "step": 1755 + }, + { + "epoch": 1.092379471228616, + "grad_norm": 0.5627850294113159, + "learning_rate": 6.1e-06, + "log_odds_chosen": 10.657611846923828, + "log_odds_ratio": -0.07279589027166367, + "logits/chosen": 0.5128650665283203, + "logits/rejected": 1.0891677141189575, + "logps/chosen": -1.0589655637741089, + "logps/rejected": -11.189449310302734, + "loss": 3.5616, + "nll_loss": 3.554288625717163, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10589656233787537, + "rewards/margins": 1.0130484104156494, + "rewards/rejected": -1.1189448833465576, + "step": 1756 + }, + { + "epoch": 1.0930015552099532, + "grad_norm": 0.5608973503112793, + "learning_rate": 6.075e-06, + "log_odds_chosen": 14.850683212280273, + "log_odds_ratio": -0.14289139211177826, + "logits/chosen": 0.2707276940345764, + "logits/rejected": 1.1643056869506836, + "logps/chosen": -1.0261825323104858, + "logps/rejected": -15.539467811584473, + "loss": 2.7926, + "nll_loss": 2.778284788131714, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1026182621717453, + "rewards/margins": 1.4513285160064697, + "rewards/rejected": -1.553946852684021, + "step": 1757 + }, + { + "epoch": 1.0936236391912908, + "grad_norm": 0.5250260829925537, + "learning_rate": 6.0500000000000005e-06, + "log_odds_chosen": 16.427053451538086, + "log_odds_ratio": -0.006266839802265167, + "logits/chosen": 0.35286745429039, + "logits/rejected": 1.6788630485534668, + "logps/chosen": -1.3709660768508911, + "logps/rejected": -17.149799346923828, + "loss": 3.1616, + "nll_loss": 3.1609363555908203, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1370965987443924, + "rewards/margins": 1.5778833627700806, + "rewards/rejected": -1.7149800062179565, + "step": 1758 + }, + { + "epoch": 1.0942457231726284, + "grad_norm": 0.5247893929481506, + "learning_rate": 6.025e-06, + "log_odds_chosen": 12.925125122070312, + "log_odds_ratio": -0.13547226786613464, + "logits/chosen": 0.15424951910972595, + "logits/rejected": 0.8768571019172668, + "logps/chosen": -1.0417335033416748, + "logps/rejected": -13.585920333862305, + "loss": 2.6077, + "nll_loss": 2.594149589538574, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10417336225509644, + "rewards/margins": 1.2544187307357788, + "rewards/rejected": -1.3585920333862305, + "step": 1759 + }, + { + "epoch": 1.0948678071539657, + "grad_norm": 0.7948043942451477, + "learning_rate": 6e-06, + "log_odds_chosen": 10.840213775634766, + "log_odds_ratio": -0.24047577381134033, + "logits/chosen": 0.19382421672344208, + "logits/rejected": 0.7827101349830627, + "logps/chosen": -1.3276411294937134, + "logps/rejected": -11.8134183883667, + "loss": 3.0138, + "nll_loss": 2.9897780418395996, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13276411592960358, + "rewards/margins": 1.0485777854919434, + "rewards/rejected": -1.1813418865203857, + "step": 1760 + }, + { + "epoch": 1.0954898911353033, + "grad_norm": 0.4992409646511078, + "learning_rate": 5.975e-06, + "log_odds_chosen": 11.336246490478516, + "log_odds_ratio": -0.10798382014036179, + "logits/chosen": 0.30136653780937195, + "logits/rejected": 0.8445361256599426, + "logps/chosen": -1.0902445316314697, + "logps/rejected": -12.009919166564941, + "loss": 3.1134, + "nll_loss": 3.1026368141174316, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10902444273233414, + "rewards/margins": 1.0919674634933472, + "rewards/rejected": -1.2009918689727783, + "step": 1761 + }, + { + "epoch": 1.0961119751166408, + "grad_norm": 0.5600183010101318, + "learning_rate": 5.95e-06, + "log_odds_chosen": 13.113090515136719, + "log_odds_ratio": -0.07676493376493454, + "logits/chosen": 0.345098614692688, + "logits/rejected": 1.2766406536102295, + "logps/chosen": -0.9305613040924072, + "logps/rejected": -13.452409744262695, + "loss": 3.1596, + "nll_loss": 3.1518924236297607, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09305613487958908, + "rewards/margins": 1.2521848678588867, + "rewards/rejected": -1.3452410697937012, + "step": 1762 + }, + { + "epoch": 1.0967340590979782, + "grad_norm": 0.5616739988327026, + "learning_rate": 5.925e-06, + "log_odds_chosen": 11.648482322692871, + "log_odds_ratio": -0.004391771741211414, + "logits/chosen": 0.34886276721954346, + "logits/rejected": 1.2682275772094727, + "logps/chosen": -0.8755768537521362, + "logps/rejected": -11.964359283447266, + "loss": 2.9164, + "nll_loss": 2.9159722328186035, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08755768835544586, + "rewards/margins": 1.1088783740997314, + "rewards/rejected": -1.1964359283447266, + "step": 1763 + }, + { + "epoch": 1.0973561430793157, + "grad_norm": 0.3752902150154114, + "learning_rate": 5.9e-06, + "log_odds_chosen": 15.04450798034668, + "log_odds_ratio": -0.0849987119436264, + "logits/chosen": 0.32096612453460693, + "logits/rejected": 1.0471903085708618, + "logps/chosen": -0.9109809398651123, + "logps/rejected": -15.317863464355469, + "loss": 3.5641, + "nll_loss": 3.555647134780884, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09109809994697571, + "rewards/margins": 1.4406882524490356, + "rewards/rejected": -1.531786322593689, + "step": 1764 + }, + { + "epoch": 1.0979782270606533, + "grad_norm": 0.5268061757087708, + "learning_rate": 5.875e-06, + "log_odds_chosen": 13.961103439331055, + "log_odds_ratio": -0.0894414633512497, + "logits/chosen": 0.34207406640052795, + "logits/rejected": 1.2890832424163818, + "logps/chosen": -1.0298205614089966, + "logps/rejected": -14.520515441894531, + "loss": 3.1615, + "nll_loss": 3.1525745391845703, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1029820516705513, + "rewards/margins": 1.349069595336914, + "rewards/rejected": -1.4520515203475952, + "step": 1765 + }, + { + "epoch": 1.0986003110419906, + "grad_norm": 2.851046323776245, + "learning_rate": 5.850000000000001e-06, + "log_odds_chosen": 12.226277351379395, + "log_odds_ratio": -0.09023725241422653, + "logits/chosen": 0.23562204837799072, + "logits/rejected": 0.862393856048584, + "logps/chosen": -1.3986942768096924, + "logps/rejected": -13.135051727294922, + "loss": 3.2844, + "nll_loss": 3.275351047515869, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13986943662166595, + "rewards/margins": 1.173635721206665, + "rewards/rejected": -1.3135051727294922, + "step": 1766 + }, + { + "epoch": 1.0992223950233282, + "grad_norm": 0.31515470147132874, + "learning_rate": 5.825000000000001e-06, + "log_odds_chosen": 12.286757469177246, + "log_odds_ratio": -0.14368560910224915, + "logits/chosen": 0.30606141686439514, + "logits/rejected": 0.9302334785461426, + "logps/chosen": -1.1010560989379883, + "logps/rejected": -13.028209686279297, + "loss": 3.4221, + "nll_loss": 3.4077136516571045, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11010561138391495, + "rewards/margins": 1.1927152872085571, + "rewards/rejected": -1.3028209209442139, + "step": 1767 + }, + { + "epoch": 1.0998444790046655, + "grad_norm": 0.6244063973426819, + "learning_rate": 5.8e-06, + "log_odds_chosen": 16.554611206054688, + "log_odds_ratio": -0.005652461666613817, + "logits/chosen": 0.27573028206825256, + "logits/rejected": 1.5867812633514404, + "logps/chosen": -0.9618663191795349, + "logps/rejected": -16.94882583618164, + "loss": 2.3865, + "nll_loss": 2.3859264850616455, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09618663787841797, + "rewards/margins": 1.598695993423462, + "rewards/rejected": -1.6948826313018799, + "step": 1768 + }, + { + "epoch": 1.100466562986003, + "grad_norm": 0.5130845904350281, + "learning_rate": 5.775000000000001e-06, + "log_odds_chosen": 8.970913887023926, + "log_odds_ratio": -0.13339346647262573, + "logits/chosen": 0.18855807185173035, + "logits/rejected": 0.5617761611938477, + "logps/chosen": -1.3988968133926392, + "logps/rejected": -10.13296127319336, + "loss": 2.7901, + "nll_loss": 2.7767951488494873, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1398896872997284, + "rewards/margins": 0.8734064102172852, + "rewards/rejected": -1.0132960081100464, + "step": 1769 + }, + { + "epoch": 1.1010886469673407, + "grad_norm": 1.5145896673202515, + "learning_rate": 5.750000000000001e-06, + "log_odds_chosen": 9.153180122375488, + "log_odds_ratio": -0.08820240199565887, + "logits/chosen": 0.39347460865974426, + "logits/rejected": 0.7451803088188171, + "logps/chosen": -1.138330340385437, + "logps/rejected": -9.729453086853027, + "loss": 3.2612, + "nll_loss": 3.2524142265319824, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11383302509784698, + "rewards/margins": 0.8591122627258301, + "rewards/rejected": -0.9729453325271606, + "step": 1770 + }, + { + "epoch": 1.101710730948678, + "grad_norm": 0.5078622698783875, + "learning_rate": 5.725e-06, + "log_odds_chosen": 13.681772232055664, + "log_odds_ratio": -0.006565184332430363, + "logits/chosen": 0.354367196559906, + "logits/rejected": 1.6420621871948242, + "logps/chosen": -1.1012815237045288, + "logps/rejected": -14.348349571228027, + "loss": 2.9864, + "nll_loss": 2.9857606887817383, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11012815684080124, + "rewards/margins": 1.3247069120407104, + "rewards/rejected": -1.4348350763320923, + "step": 1771 + }, + { + "epoch": 1.1023328149300156, + "grad_norm": 0.5735906362533569, + "learning_rate": 5.7000000000000005e-06, + "log_odds_chosen": 12.572833061218262, + "log_odds_ratio": -0.002868115436285734, + "logits/chosen": 0.202457457780838, + "logits/rejected": 1.019594669342041, + "logps/chosen": -0.9506032466888428, + "logps/rejected": -12.859609603881836, + "loss": 2.9578, + "nll_loss": 2.9574880599975586, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0950603187084198, + "rewards/margins": 1.1909008026123047, + "rewards/rejected": -1.2859611511230469, + "step": 1772 + }, + { + "epoch": 1.1029548989113531, + "grad_norm": 0.38094738125801086, + "learning_rate": 5.675000000000001e-06, + "log_odds_chosen": 11.872749328613281, + "log_odds_ratio": -0.00020267191575840116, + "logits/chosen": 0.23029054701328278, + "logits/rejected": 1.1186127662658691, + "logps/chosen": -1.0671327114105225, + "logps/rejected": -12.310469627380371, + "loss": 3.1701, + "nll_loss": 3.170095443725586, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10671327263116837, + "rewards/margins": 1.1243336200714111, + "rewards/rejected": -1.2310469150543213, + "step": 1773 + }, + { + "epoch": 1.1035769828926905, + "grad_norm": 0.5917351841926575, + "learning_rate": 5.65e-06, + "log_odds_chosen": 12.933120727539062, + "log_odds_ratio": -0.18520431220531464, + "logits/chosen": 0.4661065638065338, + "logits/rejected": 1.3536592721939087, + "logps/chosen": -0.9066671133041382, + "logps/rejected": -13.326446533203125, + "loss": 2.971, + "nll_loss": 2.9524595737457275, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09066671133041382, + "rewards/margins": 1.2419780492782593, + "rewards/rejected": -1.3326445817947388, + "step": 1774 + }, + { + "epoch": 1.104199066874028, + "grad_norm": 0.5249805450439453, + "learning_rate": 5.625e-06, + "log_odds_chosen": 14.55476188659668, + "log_odds_ratio": -0.10264108330011368, + "logits/chosen": 0.23866206407546997, + "logits/rejected": 0.718443751335144, + "logps/chosen": -0.9315866231918335, + "logps/rejected": -14.850624084472656, + "loss": 3.249, + "nll_loss": 3.238722801208496, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09315866231918335, + "rewards/margins": 1.3919038772583008, + "rewards/rejected": -1.4850624799728394, + "step": 1775 + }, + { + "epoch": 1.1048211508553654, + "grad_norm": 0.48771780729293823, + "learning_rate": 5.600000000000001e-06, + "log_odds_chosen": 10.107994079589844, + "log_odds_ratio": -0.13262391090393066, + "logits/chosen": 0.1523360162973404, + "logits/rejected": 0.8286849856376648, + "logps/chosen": -1.151200771331787, + "logps/rejected": -10.873334884643555, + "loss": 2.7703, + "nll_loss": 2.757059335708618, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11512007564306259, + "rewards/margins": 0.9722133874893188, + "rewards/rejected": -1.0873334407806396, + "step": 1776 + }, + { + "epoch": 1.105443234836703, + "grad_norm": 0.5238807797431946, + "learning_rate": 5.575e-06, + "log_odds_chosen": 12.62802505493164, + "log_odds_ratio": -0.09053575247526169, + "logits/chosen": 0.08372609317302704, + "logits/rejected": 0.6036351919174194, + "logps/chosen": -0.9051854014396667, + "logps/rejected": -12.867120742797852, + "loss": 2.6631, + "nll_loss": 2.6540231704711914, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0905185416340828, + "rewards/margins": 1.1961935758590698, + "rewards/rejected": -1.2867119312286377, + "step": 1777 + }, + { + "epoch": 1.1060653188180405, + "grad_norm": 1.3776423931121826, + "learning_rate": 5.55e-06, + "log_odds_chosen": 12.941963195800781, + "log_odds_ratio": -0.105911985039711, + "logits/chosen": 0.24506743252277374, + "logits/rejected": 1.3793317079544067, + "logps/chosen": -1.4116719961166382, + "logps/rejected": -14.101229667663574, + "loss": 2.3892, + "nll_loss": 2.378603458404541, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.14116720855236053, + "rewards/margins": 1.2689557075500488, + "rewards/rejected": -1.4101228713989258, + "step": 1778 + }, + { + "epoch": 1.1066874027993778, + "grad_norm": 0.5238211750984192, + "learning_rate": 5.5250000000000005e-06, + "log_odds_chosen": 13.194835662841797, + "log_odds_ratio": -0.13603851199150085, + "logits/chosen": 0.12899631261825562, + "logits/rejected": 0.5138747692108154, + "logps/chosen": -1.0636062622070312, + "logps/rejected": -13.72635269165039, + "loss": 2.9434, + "nll_loss": 2.929769277572632, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10636062920093536, + "rewards/margins": 1.2662746906280518, + "rewards/rejected": -1.3726352453231812, + "step": 1779 + }, + { + "epoch": 1.1073094867807154, + "grad_norm": 0.6795265078544617, + "learning_rate": 5.500000000000001e-06, + "log_odds_chosen": 9.647849082946777, + "log_odds_ratio": -0.09886172413825989, + "logits/chosen": 0.11817916482686996, + "logits/rejected": 0.804192066192627, + "logps/chosen": -1.1804863214492798, + "logps/rejected": -10.412225723266602, + "loss": 2.6481, + "nll_loss": 2.6382088661193848, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11804862320423126, + "rewards/margins": 0.9231740236282349, + "rewards/rejected": -1.0412226915359497, + "step": 1780 + }, + { + "epoch": 1.107931570762053, + "grad_norm": 0.4870269298553467, + "learning_rate": 5.475e-06, + "log_odds_chosen": 13.686765670776367, + "log_odds_ratio": -0.09282388538122177, + "logits/chosen": 0.08411633968353271, + "logits/rejected": 0.6649541854858398, + "logps/chosen": -0.9135332703590393, + "logps/rejected": -13.984970092773438, + "loss": 2.7795, + "nll_loss": 2.7702245712280273, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09135332703590393, + "rewards/margins": 1.3071436882019043, + "rewards/rejected": -1.3984969854354858, + "step": 1781 + }, + { + "epoch": 1.1085536547433903, + "grad_norm": 0.47613298892974854, + "learning_rate": 5.45e-06, + "log_odds_chosen": 11.947006225585938, + "log_odds_ratio": -0.004251046106219292, + "logits/chosen": 0.21725809574127197, + "logits/rejected": 0.9395244121551514, + "logps/chosen": -1.0874109268188477, + "logps/rejected": -12.611610412597656, + "loss": 3.0012, + "nll_loss": 3.0007901191711426, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10874108970165253, + "rewards/margins": 1.1524200439453125, + "rewards/rejected": -1.2611610889434814, + "step": 1782 + }, + { + "epoch": 1.1091757387247279, + "grad_norm": 0.5383672118186951, + "learning_rate": 5.4250000000000006e-06, + "log_odds_chosen": 10.831711769104004, + "log_odds_ratio": -0.007172089070081711, + "logits/chosen": 0.28789806365966797, + "logits/rejected": 0.9247058629989624, + "logps/chosen": -0.7739531397819519, + "logps/rejected": -10.731185913085938, + "loss": 3.1642, + "nll_loss": 3.163468360900879, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07739531993865967, + "rewards/margins": 0.9957231879234314, + "rewards/rejected": -1.0731184482574463, + "step": 1783 + }, + { + "epoch": 1.1097978227060654, + "grad_norm": 0.58636474609375, + "learning_rate": 5.4e-06, + "log_odds_chosen": 2.5373220443725586, + "log_odds_ratio": -0.48750007152557373, + "logits/chosen": 0.18775655329227448, + "logits/rejected": 0.3388059139251709, + "logps/chosen": -0.9539185166358948, + "logps/rejected": -3.1987171173095703, + "loss": 2.3384, + "nll_loss": 2.2896625995635986, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09539185464382172, + "rewards/margins": 0.22447985410690308, + "rewards/rejected": -0.319871723651886, + "step": 1784 + }, + { + "epoch": 1.1104199066874028, + "grad_norm": 3.15838885307312, + "learning_rate": 5.375e-06, + "log_odds_chosen": 9.975196838378906, + "log_odds_ratio": -0.0638727992773056, + "logits/chosen": 0.09930723160505295, + "logits/rejected": 1.253692865371704, + "logps/chosen": -1.0593295097351074, + "logps/rejected": -10.476996421813965, + "loss": 2.2838, + "nll_loss": 2.2774205207824707, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10593295097351074, + "rewards/margins": 0.9417667984962463, + "rewards/rejected": -1.0476996898651123, + "step": 1785 + }, + { + "epoch": 1.1110419906687403, + "grad_norm": 0.49239975214004517, + "learning_rate": 5.3500000000000004e-06, + "log_odds_chosen": 13.380136489868164, + "log_odds_ratio": -0.00853438675403595, + "logits/chosen": 0.004909025505185127, + "logits/rejected": 0.968700647354126, + "logps/chosen": -0.8748571872711182, + "logps/rejected": -13.532394409179688, + "loss": 2.3398, + "nll_loss": 2.338928699493408, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08748571574687958, + "rewards/margins": 1.2657537460327148, + "rewards/rejected": -1.3532395362854004, + "step": 1786 + }, + { + "epoch": 1.1116640746500777, + "grad_norm": 0.499944269657135, + "learning_rate": 5.325e-06, + "log_odds_chosen": 12.283882141113281, + "log_odds_ratio": -0.026974359527230263, + "logits/chosen": 0.1541229486465454, + "logits/rejected": 1.2575428485870361, + "logps/chosen": -1.1521315574645996, + "logps/rejected": -13.037506103515625, + "loss": 2.6301, + "nll_loss": 2.62740421295166, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11521315574645996, + "rewards/margins": 1.188537359237671, + "rewards/rejected": -1.3037505149841309, + "step": 1787 + }, + { + "epoch": 1.1122861586314152, + "grad_norm": 0.5761716961860657, + "learning_rate": 5.3e-06, + "log_odds_chosen": 9.24363899230957, + "log_odds_ratio": -0.13806326687335968, + "logits/chosen": 0.26516130566596985, + "logits/rejected": 0.7415081262588501, + "logps/chosen": -1.1681675910949707, + "logps/rejected": -10.058372497558594, + "loss": 3.0679, + "nll_loss": 3.0540828704833984, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11681675910949707, + "rewards/margins": 0.8890204429626465, + "rewards/rejected": -1.0058372020721436, + "step": 1788 + }, + { + "epoch": 1.1129082426127528, + "grad_norm": 0.40672749280929565, + "learning_rate": 5.275e-06, + "log_odds_chosen": 9.430431365966797, + "log_odds_ratio": -0.06735905259847641, + "logits/chosen": 0.17814627289772034, + "logits/rejected": 0.5426368713378906, + "logps/chosen": -1.0851701498031616, + "logps/rejected": -10.129546165466309, + "loss": 3.3547, + "nll_loss": 3.3479928970336914, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10851702094078064, + "rewards/margins": 0.9044376611709595, + "rewards/rejected": -1.0129547119140625, + "step": 1789 + }, + { + "epoch": 1.1135303265940901, + "grad_norm": 1.1642420291900635, + "learning_rate": 5.25e-06, + "log_odds_chosen": 9.519857406616211, + "log_odds_ratio": -0.08131834864616394, + "logits/chosen": 0.15677277743816376, + "logits/rejected": 0.6440751552581787, + "logps/chosen": -1.1484295129776, + "logps/rejected": -10.29475212097168, + "loss": 3.0109, + "nll_loss": 3.0027811527252197, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11484295129776001, + "rewards/margins": 0.9146323204040527, + "rewards/rejected": -1.029475212097168, + "step": 1790 + }, + { + "epoch": 1.1141524105754277, + "grad_norm": 0.6786203384399414, + "learning_rate": 5.225e-06, + "log_odds_chosen": 10.164011001586914, + "log_odds_ratio": -0.024791117757558823, + "logits/chosen": 0.16322621703147888, + "logits/rejected": 0.7032598257064819, + "logps/chosen": -1.053176760673523, + "logps/rejected": -10.773063659667969, + "loss": 2.464, + "nll_loss": 2.4615001678466797, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10531767457723618, + "rewards/margins": 0.9719887375831604, + "rewards/rejected": -1.0773065090179443, + "step": 1791 + }, + { + "epoch": 1.1147744945567652, + "grad_norm": 0.9429541230201721, + "learning_rate": 5.2e-06, + "log_odds_chosen": 9.98597240447998, + "log_odds_ratio": -0.12461309880018234, + "logits/chosen": 0.26712629199028015, + "logits/rejected": 1.0227980613708496, + "logps/chosen": -1.159783959388733, + "logps/rejected": -10.689939498901367, + "loss": 3.3261, + "nll_loss": 3.313620090484619, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11597839742898941, + "rewards/margins": 0.9530156254768372, + "rewards/rejected": -1.0689939260482788, + "step": 1792 + }, + { + "epoch": 1.1153965785381026, + "grad_norm": 0.556646466255188, + "learning_rate": 5.175e-06, + "log_odds_chosen": 11.009801864624023, + "log_odds_ratio": -0.1342574954032898, + "logits/chosen": 0.4200906753540039, + "logits/rejected": 1.036113977432251, + "logps/chosen": -1.2086161375045776, + "logps/rejected": -11.90412712097168, + "loss": 2.9352, + "nll_loss": 2.921736717224121, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12086161226034164, + "rewards/margins": 1.0695511102676392, + "rewards/rejected": -1.1904127597808838, + "step": 1793 + }, + { + "epoch": 1.1160186625194402, + "grad_norm": 0.5953820943832397, + "learning_rate": 5.15e-06, + "log_odds_chosen": 9.301898956298828, + "log_odds_ratio": -0.12582379579544067, + "logits/chosen": 0.18042080104351044, + "logits/rejected": 0.9847798347473145, + "logps/chosen": -1.0908702611923218, + "logps/rejected": -10.03856372833252, + "loss": 2.3381, + "nll_loss": 2.3255486488342285, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1090870276093483, + "rewards/margins": 0.8947693109512329, + "rewards/rejected": -1.0038564205169678, + "step": 1794 + }, + { + "epoch": 1.1166407465007775, + "grad_norm": 0.5287635326385498, + "learning_rate": 5.125e-06, + "log_odds_chosen": 13.280316352844238, + "log_odds_ratio": -0.0002730822598095983, + "logits/chosen": 0.3489305078983307, + "logits/rejected": 0.8643382787704468, + "logps/chosen": -1.0686596632003784, + "logps/rejected": -13.85269832611084, + "loss": 3.1597, + "nll_loss": 3.159668207168579, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10686597228050232, + "rewards/margins": 1.2784039974212646, + "rewards/rejected": -1.3852697610855103, + "step": 1795 + }, + { + "epoch": 1.117262830482115, + "grad_norm": 0.40285539627075195, + "learning_rate": 5.1e-06, + "log_odds_chosen": 13.911418914794922, + "log_odds_ratio": -0.00035264677717350423, + "logits/chosen": 0.3598060607910156, + "logits/rejected": 1.39473295211792, + "logps/chosen": -1.0195471048355103, + "logps/rejected": -14.366622924804688, + "loss": 3.6114, + "nll_loss": 3.611367702484131, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10195470601320267, + "rewards/margins": 1.334707498550415, + "rewards/rejected": -1.4366623163223267, + "step": 1796 + }, + { + "epoch": 1.1178849144634526, + "grad_norm": 0.4381580054759979, + "learning_rate": 5.0750000000000005e-06, + "log_odds_chosen": 8.426492691040039, + "log_odds_ratio": -0.1798814982175827, + "logits/chosen": 0.21182674169540405, + "logits/rejected": 0.8382251858711243, + "logps/chosen": -0.8595851063728333, + "logps/rejected": -8.753094673156738, + "loss": 3.2126, + "nll_loss": 3.1946401596069336, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08595851063728333, + "rewards/margins": 0.7893509268760681, + "rewards/rejected": -0.8753094673156738, + "step": 1797 + }, + { + "epoch": 1.11850699844479, + "grad_norm": 0.5452844500541687, + "learning_rate": 5.050000000000001e-06, + "log_odds_chosen": 13.306865692138672, + "log_odds_ratio": -0.0013203290291130543, + "logits/chosen": 0.32285499572753906, + "logits/rejected": 0.9466825127601624, + "logps/chosen": -0.9000481963157654, + "logps/rejected": -13.57664680480957, + "loss": 3.4016, + "nll_loss": 3.4015135765075684, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0900048241019249, + "rewards/margins": 1.2676599025726318, + "rewards/rejected": -1.357664704322815, + "step": 1798 + }, + { + "epoch": 1.1191290824261275, + "grad_norm": 0.4899674654006958, + "learning_rate": 5.025e-06, + "log_odds_chosen": 7.679771423339844, + "log_odds_ratio": -0.2658516764640808, + "logits/chosen": 0.24746796488761902, + "logits/rejected": 0.7958407402038574, + "logps/chosen": -1.0952465534210205, + "logps/rejected": -8.41911792755127, + "loss": 3.2483, + "nll_loss": 3.221743106842041, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10952465236186981, + "rewards/margins": 0.7323871850967407, + "rewards/rejected": -0.8419118523597717, + "step": 1799 + }, + { + "epoch": 1.119751166407465, + "grad_norm": 0.5063171982765198, + "learning_rate": 5e-06, + "log_odds_chosen": 12.445841789245605, + "log_odds_ratio": -0.007602700963616371, + "logits/chosen": 0.18936318159103394, + "logits/rejected": 1.0643928050994873, + "logps/chosen": -0.8866111636161804, + "logps/rejected": -12.559014320373535, + "loss": 3.2235, + "nll_loss": 3.222745180130005, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08866112679243088, + "rewards/margins": 1.1672402620315552, + "rewards/rejected": -1.2559013366699219, + "step": 1800 + }, + { + "epoch": 1.1203732503888024, + "grad_norm": 11.608192443847656, + "learning_rate": 4.975000000000001e-06, + "log_odds_chosen": 6.945075511932373, + "log_odds_ratio": -0.18060587346553802, + "logits/chosen": 0.2911841869354248, + "logits/rejected": 0.6231019496917725, + "logps/chosen": -1.6782355308532715, + "logps/rejected": -8.222484588623047, + "loss": 3.2746, + "nll_loss": 3.256551742553711, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.16782355308532715, + "rewards/margins": 0.6544249057769775, + "rewards/rejected": -0.8222484588623047, + "step": 1801 + }, + { + "epoch": 1.12099533437014, + "grad_norm": 0.6419535875320435, + "learning_rate": 4.950000000000001e-06, + "log_odds_chosen": 9.327446937561035, + "log_odds_ratio": -0.36593329906463623, + "logits/chosen": 0.21668633818626404, + "logits/rejected": 0.6384451389312744, + "logps/chosen": -1.1445276737213135, + "logps/rejected": -10.077396392822266, + "loss": 3.156, + "nll_loss": 3.1194300651550293, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1144527718424797, + "rewards/margins": 0.893286943435669, + "rewards/rejected": -1.007739782333374, + "step": 1802 + }, + { + "epoch": 1.1216174183514775, + "grad_norm": 0.5713719725608826, + "learning_rate": 4.925e-06, + "log_odds_chosen": 12.993343353271484, + "log_odds_ratio": -0.18610899150371552, + "logits/chosen": 0.13531741499900818, + "logits/rejected": 1.0787577629089355, + "logps/chosen": -0.9270740747451782, + "logps/rejected": -13.453575134277344, + "loss": 2.559, + "nll_loss": 2.540419816970825, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09270741045475006, + "rewards/margins": 1.252650260925293, + "rewards/rejected": -1.3453575372695923, + "step": 1803 + }, + { + "epoch": 1.1222395023328149, + "grad_norm": 0.5962578058242798, + "learning_rate": 4.9000000000000005e-06, + "log_odds_chosen": 9.101219177246094, + "log_odds_ratio": -0.1596328318119049, + "logits/chosen": 0.1289364993572235, + "logits/rejected": 0.7330648899078369, + "logps/chosen": -1.3524481058120728, + "logps/rejected": -10.174253463745117, + "loss": 2.687, + "nll_loss": 2.6709885597229004, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13524481654167175, + "rewards/margins": 0.8821805715560913, + "rewards/rejected": -1.017425298690796, + "step": 1804 + }, + { + "epoch": 1.1228615863141524, + "grad_norm": 3.5306475162506104, + "learning_rate": 4.875000000000001e-06, + "log_odds_chosen": 10.527320861816406, + "log_odds_ratio": -0.13010427355766296, + "logits/chosen": 0.34094375371932983, + "logits/rejected": 1.1596550941467285, + "logps/chosen": -1.4645371437072754, + "logps/rejected": -11.655326843261719, + "loss": 3.0124, + "nll_loss": 2.9993646144866943, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14645370841026306, + "rewards/margins": 1.0190789699554443, + "rewards/rejected": -1.1655325889587402, + "step": 1805 + }, + { + "epoch": 1.1234836702954898, + "grad_norm": 0.7061519026756287, + "learning_rate": 4.85e-06, + "log_odds_chosen": 9.48027515411377, + "log_odds_ratio": -0.12131432443857193, + "logits/chosen": 0.0875740572810173, + "logits/rejected": 0.6032061576843262, + "logps/chosen": -0.8424355983734131, + "logps/rejected": -9.36489200592041, + "loss": 2.832, + "nll_loss": 2.819911479949951, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08424356579780579, + "rewards/margins": 0.8522456884384155, + "rewards/rejected": -0.9364892840385437, + "step": 1806 + }, + { + "epoch": 1.1241057542768274, + "grad_norm": 0.5254723429679871, + "learning_rate": 4.825e-06, + "log_odds_chosen": 11.07059097290039, + "log_odds_ratio": -0.009043725207448006, + "logits/chosen": 0.22842171788215637, + "logits/rejected": 0.9652547836303711, + "logps/chosen": -1.137882947921753, + "logps/rejected": -11.800371170043945, + "loss": 3.1437, + "nll_loss": 3.142822265625, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11378829926252365, + "rewards/margins": 1.066248893737793, + "rewards/rejected": -1.180037021636963, + "step": 1807 + }, + { + "epoch": 1.124727838258165, + "grad_norm": 5.473237037658691, + "learning_rate": 4.800000000000001e-06, + "log_odds_chosen": 2.729802370071411, + "log_odds_ratio": -0.5633368492126465, + "logits/chosen": 0.1724277287721634, + "logits/rejected": 0.2686040699481964, + "logps/chosen": -1.4853651523590088, + "logps/rejected": -4.039682388305664, + "loss": 3.0733, + "nll_loss": 3.0169594287872314, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14853651821613312, + "rewards/margins": 0.2554317116737366, + "rewards/rejected": -0.4039682447910309, + "step": 1808 + }, + { + "epoch": 1.1253499222395023, + "grad_norm": 0.5656214952468872, + "learning_rate": 4.775e-06, + "log_odds_chosen": 8.675018310546875, + "log_odds_ratio": -0.022862181067466736, + "logits/chosen": 0.10422070324420929, + "logits/rejected": 0.7279854416847229, + "logps/chosen": -1.305293083190918, + "logps/rejected": -9.647176742553711, + "loss": 2.836, + "nll_loss": 2.8336758613586426, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13052931427955627, + "rewards/margins": 0.8341884016990662, + "rewards/rejected": -0.9647176265716553, + "step": 1809 + }, + { + "epoch": 1.1259720062208398, + "grad_norm": 0.6115718483924866, + "learning_rate": 4.75e-06, + "log_odds_chosen": 10.84763240814209, + "log_odds_ratio": -0.08447157591581345, + "logits/chosen": 0.288266122341156, + "logits/rejected": 1.0908019542694092, + "logps/chosen": -1.0053138732910156, + "logps/rejected": -11.463065147399902, + "loss": 2.7034, + "nll_loss": 2.6949832439422607, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10053138434886932, + "rewards/margins": 1.0457751750946045, + "rewards/rejected": -1.1463065147399902, + "step": 1810 + }, + { + "epoch": 1.1265940902021774, + "grad_norm": 0.45324620604515076, + "learning_rate": 4.7250000000000005e-06, + "log_odds_chosen": 6.809017181396484, + "log_odds_ratio": -0.136130690574646, + "logits/chosen": 0.09738215804100037, + "logits/rejected": 0.2309304028749466, + "logps/chosen": -0.8674330711364746, + "logps/rejected": -7.067953586578369, + "loss": 3.1252, + "nll_loss": 3.111586570739746, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0867433100938797, + "rewards/margins": 0.6200520992279053, + "rewards/rejected": -0.706795334815979, + "step": 1811 + }, + { + "epoch": 1.1272161741835147, + "grad_norm": 0.41881006956100464, + "learning_rate": 4.7e-06, + "log_odds_chosen": 9.960844993591309, + "log_odds_ratio": -0.2785137891769409, + "logits/chosen": 0.21459250152111053, + "logits/rejected": 0.8106105327606201, + "logps/chosen": -1.2352036237716675, + "logps/rejected": -10.74962329864502, + "loss": 3.3827, + "nll_loss": 3.3548502922058105, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12352035939693451, + "rewards/margins": 0.9514420628547668, + "rewards/rejected": -1.0749623775482178, + "step": 1812 + }, + { + "epoch": 1.1278382581648523, + "grad_norm": 0.34027522802352905, + "learning_rate": 4.675e-06, + "log_odds_chosen": 7.443631649017334, + "log_odds_ratio": -0.09579671174287796, + "logits/chosen": 0.35332971811294556, + "logits/rejected": 0.8371080160140991, + "logps/chosen": -1.1036956310272217, + "logps/rejected": -8.065750122070312, + "loss": 3.8843, + "nll_loss": 3.874685287475586, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11036956310272217, + "rewards/margins": 0.6962054371833801, + "rewards/rejected": -0.8065750002861023, + "step": 1813 + }, + { + "epoch": 1.1284603421461896, + "grad_norm": 6.815546035766602, + "learning_rate": 4.65e-06, + "log_odds_chosen": 8.074342727661133, + "log_odds_ratio": -0.07201213389635086, + "logits/chosen": 0.18229568004608154, + "logits/rejected": 0.6928460001945496, + "logps/chosen": -0.9151440858840942, + "logps/rejected": -8.362491607666016, + "loss": 2.7987, + "nll_loss": 2.791532516479492, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09151440858840942, + "rewards/margins": 0.7447347044944763, + "rewards/rejected": -0.8362491130828857, + "step": 1814 + }, + { + "epoch": 1.1290824261275272, + "grad_norm": 0.6330059766769409, + "learning_rate": 4.625e-06, + "log_odds_chosen": 8.535118103027344, + "log_odds_ratio": -0.13910330832004547, + "logits/chosen": 0.2871561646461487, + "logits/rejected": 1.1333131790161133, + "logps/chosen": -1.05423903465271, + "logps/rejected": -9.111215591430664, + "loss": 2.8813, + "nll_loss": 2.867426633834839, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10542389750480652, + "rewards/margins": 0.8056977391242981, + "rewards/rejected": -0.911121666431427, + "step": 1815 + }, + { + "epoch": 1.1297045101088647, + "grad_norm": 0.5443134903907776, + "learning_rate": 4.6e-06, + "log_odds_chosen": 8.793806076049805, + "log_odds_ratio": -0.11311511695384979, + "logits/chosen": 0.03522084280848503, + "logits/rejected": 0.4860665202140808, + "logps/chosen": -1.1860641241073608, + "logps/rejected": -9.650390625, + "loss": 2.5238, + "nll_loss": 2.5124518871307373, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11860641837120056, + "rewards/margins": 0.8464326858520508, + "rewards/rejected": -0.965039074420929, + "step": 1816 + }, + { + "epoch": 1.130326594090202, + "grad_norm": 2.309431314468384, + "learning_rate": 4.575e-06, + "log_odds_chosen": 9.45278549194336, + "log_odds_ratio": -0.10079053044319153, + "logits/chosen": 0.2651061713695526, + "logits/rejected": 1.1936180591583252, + "logps/chosen": -1.343247652053833, + "logps/rejected": -10.476812362670898, + "loss": 3.1809, + "nll_loss": 3.1708319187164307, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13432475924491882, + "rewards/margins": 0.9133565425872803, + "rewards/rejected": -1.0476813316345215, + "step": 1817 + }, + { + "epoch": 1.1309486780715396, + "grad_norm": 0.5004618763923645, + "learning_rate": 4.5500000000000005e-06, + "log_odds_chosen": 9.56799030303955, + "log_odds_ratio": -0.003998002503067255, + "logits/chosen": 0.12169669568538666, + "logits/rejected": 0.5667641162872314, + "logps/chosen": -1.166806697845459, + "logps/rejected": -10.264081954956055, + "loss": 2.6758, + "nll_loss": 2.675359010696411, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11668066680431366, + "rewards/margins": 0.9097274541854858, + "rewards/rejected": -1.0264081954956055, + "step": 1818 + }, + { + "epoch": 1.1315707620528772, + "grad_norm": 0.616479754447937, + "learning_rate": 4.525e-06, + "log_odds_chosen": 11.951116561889648, + "log_odds_ratio": -0.013583849184215069, + "logits/chosen": 0.08394649624824524, + "logits/rejected": 0.9181208610534668, + "logps/chosen": -1.046438217163086, + "logps/rejected": -12.497675895690918, + "loss": 2.5327, + "nll_loss": 2.5313260555267334, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1046438217163086, + "rewards/margins": 1.1451237201690674, + "rewards/rejected": -1.249767541885376, + "step": 1819 + }, + { + "epoch": 1.1321928460342146, + "grad_norm": 0.6492127776145935, + "learning_rate": 4.5e-06, + "log_odds_chosen": 14.50720500946045, + "log_odds_ratio": -0.0004224668664392084, + "logits/chosen": 0.2038557529449463, + "logits/rejected": 1.0356361865997314, + "logps/chosen": -0.7524871826171875, + "logps/rejected": -14.405542373657227, + "loss": 2.7266, + "nll_loss": 2.726580858230591, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07524871826171875, + "rewards/margins": 1.3653055429458618, + "rewards/rejected": -1.4405542612075806, + "step": 1820 + }, + { + "epoch": 1.1328149300155521, + "grad_norm": 0.3873848021030426, + "learning_rate": 4.475e-06, + "log_odds_chosen": 9.258432388305664, + "log_odds_ratio": -0.1324738711118698, + "logits/chosen": 0.24150168895721436, + "logits/rejected": 0.7265496253967285, + "logps/chosen": -0.9946005344390869, + "logps/rejected": -9.797269821166992, + "loss": 3.4301, + "nll_loss": 3.4168717861175537, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09946005046367645, + "rewards/margins": 0.8802669048309326, + "rewards/rejected": -0.9797269105911255, + "step": 1821 + }, + { + "epoch": 1.1334370139968897, + "grad_norm": 0.651413083076477, + "learning_rate": 4.45e-06, + "log_odds_chosen": 6.027961730957031, + "log_odds_ratio": -0.16579578816890717, + "logits/chosen": 0.2767557203769684, + "logits/rejected": 0.3758124113082886, + "logps/chosen": -1.2263211011886597, + "logps/rejected": -6.595951080322266, + "loss": 3.507, + "nll_loss": 3.4903945922851562, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12263211607933044, + "rewards/margins": 0.5369629859924316, + "rewards/rejected": -0.6595950722694397, + "step": 1822 + }, + { + "epoch": 1.134059097978227, + "grad_norm": 0.4178617596626282, + "learning_rate": 4.425e-06, + "log_odds_chosen": 9.204516410827637, + "log_odds_ratio": -0.08950802683830261, + "logits/chosen": 0.25004613399505615, + "logits/rejected": 0.8217270374298096, + "logps/chosen": -0.8077847361564636, + "logps/rejected": -9.325156211853027, + "loss": 3.2034, + "nll_loss": 3.1944754123687744, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08077847212553024, + "rewards/margins": 0.8517371416091919, + "rewards/rejected": -0.9325156807899475, + "step": 1823 + }, + { + "epoch": 1.1346811819595646, + "grad_norm": 0.8599444031715393, + "learning_rate": 4.4e-06, + "log_odds_chosen": 7.78436279296875, + "log_odds_ratio": -0.3169845640659332, + "logits/chosen": 0.21967101097106934, + "logits/rejected": 0.54742431640625, + "logps/chosen": -1.0425914525985718, + "logps/rejected": -8.581308364868164, + "loss": 2.8782, + "nll_loss": 2.8464527130126953, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10425914824008942, + "rewards/margins": 0.7538717985153198, + "rewards/rejected": -0.858130931854248, + "step": 1824 + }, + { + "epoch": 1.1353032659409021, + "grad_norm": 0.42188894748687744, + "learning_rate": 4.375e-06, + "log_odds_chosen": 6.164718151092529, + "log_odds_ratio": -0.1897582858800888, + "logits/chosen": 0.31425851583480835, + "logits/rejected": 0.5133528709411621, + "logps/chosen": -1.1853477954864502, + "logps/rejected": -7.080450057983398, + "loss": 3.604, + "nll_loss": 3.5849769115448, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11853478848934174, + "rewards/margins": 0.5895102024078369, + "rewards/rejected": -0.7080450057983398, + "step": 1825 + }, + { + "epoch": 1.1359253499222395, + "grad_norm": 0.6564217805862427, + "learning_rate": 4.35e-06, + "log_odds_chosen": 8.594808578491211, + "log_odds_ratio": -0.06983879208564758, + "logits/chosen": 0.27198323607444763, + "logits/rejected": 1.182363748550415, + "logps/chosen": -1.3396228551864624, + "logps/rejected": -9.5281400680542, + "loss": 3.3834, + "nll_loss": 3.3763747215270996, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13396227359771729, + "rewards/margins": 0.8188517093658447, + "rewards/rejected": -0.9528140425682068, + "step": 1826 + }, + { + "epoch": 1.136547433903577, + "grad_norm": 0.899903416633606, + "learning_rate": 4.325e-06, + "log_odds_chosen": 9.354620933532715, + "log_odds_ratio": -0.019159851595759392, + "logits/chosen": 0.1395963877439499, + "logits/rejected": 0.6038464307785034, + "logps/chosen": -1.1089383363723755, + "logps/rejected": -9.972417831420898, + "loss": 2.7222, + "nll_loss": 2.7203056812286377, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11089383065700531, + "rewards/margins": 0.8863480091094971, + "rewards/rejected": -0.9972418546676636, + "step": 1827 + }, + { + "epoch": 1.1371695178849144, + "grad_norm": 0.5476040244102478, + "learning_rate": 4.2999999999999995e-06, + "log_odds_chosen": 12.213828086853027, + "log_odds_ratio": -0.005907875020056963, + "logits/chosen": 0.12920621037483215, + "logits/rejected": 1.0453824996948242, + "logps/chosen": -0.9840941429138184, + "logps/rejected": -12.59911060333252, + "loss": 2.2828, + "nll_loss": 2.2822105884552, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09840941429138184, + "rewards/margins": 1.1615017652511597, + "rewards/rejected": -1.2599111795425415, + "step": 1828 + }, + { + "epoch": 1.137791601866252, + "grad_norm": 0.5353556275367737, + "learning_rate": 4.2750000000000006e-06, + "log_odds_chosen": 8.780527114868164, + "log_odds_ratio": -0.19837351143360138, + "logits/chosen": 0.1902414858341217, + "logits/rejected": 1.0131205320358276, + "logps/chosen": -1.11326265335083, + "logps/rejected": -9.524097442626953, + "loss": 2.8569, + "nll_loss": 2.837033271789551, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11132626235485077, + "rewards/margins": 0.8410835266113281, + "rewards/rejected": -0.9524097442626953, + "step": 1829 + }, + { + "epoch": 1.1384136858475895, + "grad_norm": 0.4632808268070221, + "learning_rate": 4.250000000000001e-06, + "log_odds_chosen": 12.49989128112793, + "log_odds_ratio": -0.01967625506222248, + "logits/chosen": 0.11193087697029114, + "logits/rejected": 0.8796758651733398, + "logps/chosen": -0.9191639423370361, + "logps/rejected": -12.864387512207031, + "loss": 2.8738, + "nll_loss": 2.871879816055298, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09191639721393585, + "rewards/margins": 1.1945223808288574, + "rewards/rejected": -1.2864387035369873, + "step": 1830 + }, + { + "epoch": 1.1390357698289268, + "grad_norm": 0.562156617641449, + "learning_rate": 4.225e-06, + "log_odds_chosen": 9.081823348999023, + "log_odds_ratio": -0.08209239691495895, + "logits/chosen": -0.011233558878302574, + "logits/rejected": 0.5702467560768127, + "logps/chosen": -0.9023143649101257, + "logps/rejected": -9.305441856384277, + "loss": 2.3841, + "nll_loss": 2.3758649826049805, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09023144096136093, + "rewards/margins": 0.8403127789497375, + "rewards/rejected": -0.9305441975593567, + "step": 1831 + }, + { + "epoch": 1.1396578538102644, + "grad_norm": 0.5008666515350342, + "learning_rate": 4.2000000000000004e-06, + "log_odds_chosen": 8.269814491271973, + "log_odds_ratio": -0.020565472543239594, + "logits/chosen": 0.047942303121089935, + "logits/rejected": 0.6168398857116699, + "logps/chosen": -0.8375715017318726, + "logps/rejected": -8.36706829071045, + "loss": 2.8247, + "nll_loss": 2.8226683139801025, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08375715464353561, + "rewards/margins": 0.7529496550559998, + "rewards/rejected": -0.836706817150116, + "step": 1832 + }, + { + "epoch": 1.1402799377916017, + "grad_norm": 0.6325581669807434, + "learning_rate": 4.175000000000001e-06, + "log_odds_chosen": 5.459460735321045, + "log_odds_ratio": -0.12678371369838715, + "logits/chosen": 0.324423611164093, + "logits/rejected": 0.6908161044120789, + "logps/chosen": -1.1747678518295288, + "logps/rejected": -6.291014671325684, + "loss": 3.437, + "nll_loss": 3.424285411834717, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11747679114341736, + "rewards/margins": 0.5116246938705444, + "rewards/rejected": -0.6291014552116394, + "step": 1833 + }, + { + "epoch": 1.1409020217729393, + "grad_norm": 1.8699506521224976, + "learning_rate": 4.15e-06, + "log_odds_chosen": 5.80875825881958, + "log_odds_ratio": -0.19153320789337158, + "logits/chosen": 0.1311289221048355, + "logits/rejected": 0.5716495513916016, + "logps/chosen": -1.1367210149765015, + "logps/rejected": -6.573773384094238, + "loss": 2.6326, + "nll_loss": 2.6134707927703857, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11367210745811462, + "rewards/margins": 0.5437052249908447, + "rewards/rejected": -0.6573773622512817, + "step": 1834 + }, + { + "epoch": 1.1415241057542769, + "grad_norm": 0.5624792575836182, + "learning_rate": 4.125e-06, + "log_odds_chosen": 10.041004180908203, + "log_odds_ratio": -0.10364656895399094, + "logits/chosen": 0.3881472647190094, + "logits/rejected": 1.2136845588684082, + "logps/chosen": -1.314047932624817, + "logps/rejected": -11.067365646362305, + "loss": 3.0385, + "nll_loss": 3.0281782150268555, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1314047873020172, + "rewards/margins": 0.9753317832946777, + "rewards/rejected": -1.106736660003662, + "step": 1835 + }, + { + "epoch": 1.1421461897356142, + "grad_norm": 0.6248362064361572, + "learning_rate": 4.1000000000000006e-06, + "log_odds_chosen": 7.472942352294922, + "log_odds_ratio": -0.2258700728416443, + "logits/chosen": 0.26532885432243347, + "logits/rejected": 1.0628668069839478, + "logps/chosen": -1.1556396484375, + "logps/rejected": -8.379240989685059, + "loss": 2.5733, + "nll_loss": 2.5506765842437744, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11556396633386612, + "rewards/margins": 0.7223601341247559, + "rewards/rejected": -0.8379241228103638, + "step": 1836 + }, + { + "epoch": 1.1427682737169518, + "grad_norm": 0.640946626663208, + "learning_rate": 4.075e-06, + "log_odds_chosen": 11.358403205871582, + "log_odds_ratio": -0.011467165313661098, + "logits/chosen": 0.1835147589445114, + "logits/rejected": 0.8875374794006348, + "logps/chosen": -1.098917007446289, + "logps/rejected": -12.045377731323242, + "loss": 2.4127, + "nll_loss": 2.4115188121795654, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10989170521497726, + "rewards/margins": 1.0946459770202637, + "rewards/rejected": -1.2045376300811768, + "step": 1837 + }, + { + "epoch": 1.1433903576982893, + "grad_norm": 0.6438122987747192, + "learning_rate": 4.05e-06, + "log_odds_chosen": 9.392248153686523, + "log_odds_ratio": -0.04183017462491989, + "logits/chosen": 0.30040043592453003, + "logits/rejected": 1.151014804840088, + "logps/chosen": -1.1815377473831177, + "logps/rejected": -10.117670059204102, + "loss": 3.2376, + "nll_loss": 3.2334647178649902, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11815378069877625, + "rewards/margins": 0.8936132192611694, + "rewards/rejected": -1.011767029762268, + "step": 1838 + }, + { + "epoch": 1.1440124416796267, + "grad_norm": 0.5689122676849365, + "learning_rate": 4.0250000000000004e-06, + "log_odds_chosen": 11.757156372070312, + "log_odds_ratio": -0.15141499042510986, + "logits/chosen": 0.38766422867774963, + "logits/rejected": 1.2692524194717407, + "logps/chosen": -1.2827192544937134, + "logps/rejected": -12.719758033752441, + "loss": 3.2818, + "nll_loss": 3.266641139984131, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1282719373703003, + "rewards/margins": 1.143703818321228, + "rewards/rejected": -1.2719756364822388, + "step": 1839 + }, + { + "epoch": 1.1446345256609642, + "grad_norm": 0.5184850692749023, + "learning_rate": 4.000000000000001e-06, + "log_odds_chosen": 9.902469635009766, + "log_odds_ratio": -0.006094024516642094, + "logits/chosen": 0.17294877767562866, + "logits/rejected": 0.9480814933776855, + "logps/chosen": -1.2275193929672241, + "logps/rejected": -10.757221221923828, + "loss": 3.0007, + "nll_loss": 3.000067949295044, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12275194376707077, + "rewards/margins": 0.9529702663421631, + "rewards/rejected": -1.0757222175598145, + "step": 1840 + }, + { + "epoch": 1.1452566096423018, + "grad_norm": 0.7144693732261658, + "learning_rate": 3.975e-06, + "log_odds_chosen": 3.845844030380249, + "log_odds_ratio": -0.27779051661491394, + "logits/chosen": 0.24042358994483948, + "logits/rejected": 0.4206497073173523, + "logps/chosen": -1.3415154218673706, + "logps/rejected": -4.978623390197754, + "loss": 3.2546, + "nll_loss": 3.2268238067626953, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13415154814720154, + "rewards/margins": 0.36371082067489624, + "rewards/rejected": -0.4978623688220978, + "step": 1841 + }, + { + "epoch": 1.1458786936236391, + "grad_norm": 0.6551204919815063, + "learning_rate": 3.95e-06, + "log_odds_chosen": 8.937030792236328, + "log_odds_ratio": -0.2580510675907135, + "logits/chosen": 0.36829322576522827, + "logits/rejected": 0.6801607012748718, + "logps/chosen": -0.9728015065193176, + "logps/rejected": -9.46196174621582, + "loss": 3.3067, + "nll_loss": 3.2808713912963867, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09728015959262848, + "rewards/margins": 0.8489160537719727, + "rewards/rejected": -0.9461961984634399, + "step": 1842 + }, + { + "epoch": 1.1465007776049767, + "grad_norm": 0.5541713237762451, + "learning_rate": 3.9250000000000005e-06, + "log_odds_chosen": 11.716171264648438, + "log_odds_ratio": -0.11533959209918976, + "logits/chosen": 0.31325313448905945, + "logits/rejected": 0.6892248392105103, + "logps/chosen": -0.795279324054718, + "logps/rejected": -11.923837661743164, + "loss": 3.444, + "nll_loss": 3.4324710369110107, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07952792942523956, + "rewards/margins": 1.1128557920455933, + "rewards/rejected": -1.1923837661743164, + "step": 1843 + }, + { + "epoch": 1.1471228615863143, + "grad_norm": 0.7045103907585144, + "learning_rate": 3.9e-06, + "log_odds_chosen": 9.548257827758789, + "log_odds_ratio": -0.12770326435565948, + "logits/chosen": 0.19907158613204956, + "logits/rejected": 0.7976346611976624, + "logps/chosen": -1.2352797985076904, + "logps/rejected": -10.314152717590332, + "loss": 2.605, + "nll_loss": 2.5922439098358154, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12352797389030457, + "rewards/margins": 0.90788733959198, + "rewards/rejected": -1.031415343284607, + "step": 1844 + }, + { + "epoch": 1.1477449455676516, + "grad_norm": 0.5564942955970764, + "learning_rate": 3.875e-06, + "log_odds_chosen": 13.43726634979248, + "log_odds_ratio": -0.015584073029458523, + "logits/chosen": 0.24572142958641052, + "logits/rejected": 1.1272077560424805, + "logps/chosen": -1.1543810367584229, + "logps/rejected": -14.049823760986328, + "loss": 2.7776, + "nll_loss": 2.7759971618652344, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11543810367584229, + "rewards/margins": 1.2895443439483643, + "rewards/rejected": -1.4049824476242065, + "step": 1845 + }, + { + "epoch": 1.1483670295489892, + "grad_norm": 0.539889931678772, + "learning_rate": 3.85e-06, + "log_odds_chosen": 7.889827728271484, + "log_odds_ratio": -0.23806874454021454, + "logits/chosen": 0.3377843201160431, + "logits/rejected": 0.6474952697753906, + "logps/chosen": -1.1127874851226807, + "logps/rejected": -8.637657165527344, + "loss": 3.512, + "nll_loss": 3.4881978034973145, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11127874255180359, + "rewards/margins": 0.7524869441986084, + "rewards/rejected": -0.8637657165527344, + "step": 1846 + }, + { + "epoch": 1.1489891135303265, + "grad_norm": 0.5164604187011719, + "learning_rate": 3.825e-06, + "log_odds_chosen": 7.681192398071289, + "log_odds_ratio": -0.15600374341011047, + "logits/chosen": 0.2687627375125885, + "logits/rejected": 0.6659653186798096, + "logps/chosen": -1.1743711233139038, + "logps/rejected": -8.557535171508789, + "loss": 3.3275, + "nll_loss": 3.3118975162506104, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11743712425231934, + "rewards/margins": 0.7383164167404175, + "rewards/rejected": -0.8557536005973816, + "step": 1847 + }, + { + "epoch": 1.149611197511664, + "grad_norm": 0.5505557656288147, + "learning_rate": 3.8e-06, + "log_odds_chosen": 8.284540176391602, + "log_odds_ratio": -0.04684235155582428, + "logits/chosen": 0.1542847603559494, + "logits/rejected": 0.35668423771858215, + "logps/chosen": -0.7916722893714905, + "logps/rejected": -8.380890846252441, + "loss": 3.157, + "nll_loss": 3.152329921722412, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07916723191738129, + "rewards/margins": 0.7589218616485596, + "rewards/rejected": -0.8380890488624573, + "step": 1848 + }, + { + "epoch": 1.1502332814930016, + "grad_norm": 0.6552994847297668, + "learning_rate": 3.775e-06, + "log_odds_chosen": 15.202106475830078, + "log_odds_ratio": -0.01975049264729023, + "logits/chosen": 0.44040918350219727, + "logits/rejected": 1.5805052518844604, + "logps/chosen": -1.228332757949829, + "logps/rejected": -16.05695152282715, + "loss": 2.9889, + "nll_loss": 2.9869132041931152, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12283328920602798, + "rewards/margins": 1.4828619956970215, + "rewards/rejected": -1.6056952476501465, + "step": 1849 + }, + { + "epoch": 1.150855365474339, + "grad_norm": 7.42947244644165, + "learning_rate": 3.75e-06, + "log_odds_chosen": 5.40254545211792, + "log_odds_ratio": -0.21582013368606567, + "logits/chosen": 0.16621777415275574, + "logits/rejected": 0.7174571752548218, + "logps/chosen": -1.1043572425842285, + "logps/rejected": -6.091453552246094, + "loss": 2.4599, + "nll_loss": 2.4383544921875, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11043573170900345, + "rewards/margins": 0.49870961904525757, + "rewards/rejected": -0.6091454029083252, + "step": 1850 + }, + { + "epoch": 1.1514774494556765, + "grad_norm": 0.5765233635902405, + "learning_rate": 3.725e-06, + "log_odds_chosen": 11.660882949829102, + "log_odds_ratio": -0.11469542235136032, + "logits/chosen": 0.15178298950195312, + "logits/rejected": 0.9491125345230103, + "logps/chosen": -1.121006965637207, + "logps/rejected": -12.402009963989258, + "loss": 2.7552, + "nll_loss": 2.7437798976898193, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11210069060325623, + "rewards/margins": 1.1281002759933472, + "rewards/rejected": -1.2402009963989258, + "step": 1851 + }, + { + "epoch": 1.1520995334370139, + "grad_norm": 0.792013943195343, + "learning_rate": 3.7e-06, + "log_odds_chosen": 4.792044639587402, + "log_odds_ratio": -0.3845703601837158, + "logits/chosen": 0.11802121996879578, + "logits/rejected": 0.3419199585914612, + "logps/chosen": -1.216029405593872, + "logps/rejected": -5.794693946838379, + "loss": 2.6536, + "nll_loss": 2.6151530742645264, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12160294502973557, + "rewards/margins": 0.4578664302825928, + "rewards/rejected": -0.5794694423675537, + "step": 1852 + }, + { + "epoch": 1.1527216174183514, + "grad_norm": 0.5236743688583374, + "learning_rate": 3.675e-06, + "log_odds_chosen": 9.352622985839844, + "log_odds_ratio": -0.05456789955496788, + "logits/chosen": 0.17611032724380493, + "logits/rejected": 0.6998663544654846, + "logps/chosen": -1.2798631191253662, + "logps/rejected": -9.97517204284668, + "loss": 3.0422, + "nll_loss": 3.0367519855499268, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12798631191253662, + "rewards/margins": 0.8695309162139893, + "rewards/rejected": -0.9975172281265259, + "step": 1853 + }, + { + "epoch": 1.153343701399689, + "grad_norm": 0.5596763491630554, + "learning_rate": 3.6499999999999998e-06, + "log_odds_chosen": 11.331252098083496, + "log_odds_ratio": -0.17287488281726837, + "logits/chosen": 0.16702744364738464, + "logits/rejected": 0.8460006713867188, + "logps/chosen": -0.9080240726470947, + "logps/rejected": -11.69823169708252, + "loss": 2.494, + "nll_loss": 2.4767417907714844, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09080240875482559, + "rewards/margins": 1.0790207386016846, + "rewards/rejected": -1.169823169708252, + "step": 1854 + }, + { + "epoch": 1.1539657853810263, + "grad_norm": 0.5587321519851685, + "learning_rate": 3.625e-06, + "log_odds_chosen": 12.046408653259277, + "log_odds_ratio": -0.09323586523532867, + "logits/chosen": 0.31840091943740845, + "logits/rejected": 1.0333279371261597, + "logps/chosen": -1.2253146171569824, + "logps/rejected": -12.962678909301758, + "loss": 3.1042, + "nll_loss": 3.094883441925049, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.122531458735466, + "rewards/margins": 1.1737364530563354, + "rewards/rejected": -1.2962678670883179, + "step": 1855 + }, + { + "epoch": 1.154587869362364, + "grad_norm": 5.766540050506592, + "learning_rate": 3.6e-06, + "log_odds_chosen": 7.31904935836792, + "log_odds_ratio": -0.12297351658344269, + "logits/chosen": 0.2189761996269226, + "logits/rejected": 0.9063780903816223, + "logps/chosen": -1.0311709642410278, + "logps/rejected": -7.974186897277832, + "loss": 2.1915, + "nll_loss": 2.1791834831237793, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10311709344387054, + "rewards/margins": 0.6943015456199646, + "rewards/rejected": -0.7974186539649963, + "step": 1856 + }, + { + "epoch": 1.1552099533437015, + "grad_norm": 0.5681399703025818, + "learning_rate": 3.575e-06, + "log_odds_chosen": 12.468045234680176, + "log_odds_ratio": -0.008419721387326717, + "logits/chosen": 0.30339351296424866, + "logits/rejected": 1.4633084535598755, + "logps/chosen": -1.0308117866516113, + "logps/rejected": -13.030942916870117, + "loss": 2.5128, + "nll_loss": 2.5119991302490234, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10308118164539337, + "rewards/margins": 1.2000131607055664, + "rewards/rejected": -1.3030942678451538, + "step": 1857 + }, + { + "epoch": 1.1558320373250388, + "grad_norm": 0.616908848285675, + "learning_rate": 3.55e-06, + "log_odds_chosen": 6.324096202850342, + "log_odds_ratio": -0.22164684534072876, + "logits/chosen": 0.2558293640613556, + "logits/rejected": 0.6237554550170898, + "logps/chosen": -0.7948259115219116, + "logps/rejected": -6.515898704528809, + "loss": 2.936, + "nll_loss": 2.9138057231903076, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07948260009288788, + "rewards/margins": 0.5721072554588318, + "rewards/rejected": -0.6515898704528809, + "step": 1858 + }, + { + "epoch": 1.1564541213063764, + "grad_norm": 0.6301389932632446, + "learning_rate": 3.5249999999999997e-06, + "log_odds_chosen": 7.7862467765808105, + "log_odds_ratio": -0.2558721601963043, + "logits/chosen": 0.2493916153907776, + "logits/rejected": 0.9461332559585571, + "logps/chosen": -1.1818265914916992, + "logps/rejected": -8.736333847045898, + "loss": 3.0082, + "nll_loss": 2.98262619972229, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11818264424800873, + "rewards/margins": 0.7554507255554199, + "rewards/rejected": -0.8736333847045898, + "step": 1859 + }, + { + "epoch": 1.157076205287714, + "grad_norm": 0.6364076137542725, + "learning_rate": 3.5000000000000004e-06, + "log_odds_chosen": 5.857247352600098, + "log_odds_ratio": -0.38049325346946716, + "logits/chosen": 0.0820159912109375, + "logits/rejected": 0.3089779317378998, + "logps/chosen": -1.0933566093444824, + "logps/rejected": -6.80075216293335, + "loss": 2.4313, + "nll_loss": 2.393275022506714, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10933566093444824, + "rewards/margins": 0.5707396268844604, + "rewards/rejected": -0.6800752282142639, + "step": 1860 + }, + { + "epoch": 1.1576982892690513, + "grad_norm": 0.5961266160011292, + "learning_rate": 3.4750000000000006e-06, + "log_odds_chosen": 7.8994646072387695, + "log_odds_ratio": -0.1790763884782791, + "logits/chosen": 0.2821861505508423, + "logits/rejected": 0.820894718170166, + "logps/chosen": -1.0732139348983765, + "logps/rejected": -8.610696792602539, + "loss": 3.1621, + "nll_loss": 3.1442208290100098, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10732139647006989, + "rewards/margins": 0.7537482976913452, + "rewards/rejected": -0.8610696196556091, + "step": 1861 + }, + { + "epoch": 1.1583203732503888, + "grad_norm": 0.5593898892402649, + "learning_rate": 3.4500000000000004e-06, + "log_odds_chosen": 15.28304386138916, + "log_odds_ratio": -0.00035143038257956505, + "logits/chosen": 0.2785932719707489, + "logits/rejected": 1.468186855316162, + "logps/chosen": -1.0075515508651733, + "logps/rejected": -15.71903133392334, + "loss": 3.0106, + "nll_loss": 3.0106000900268555, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10075514763593674, + "rewards/margins": 1.4711480140686035, + "rewards/rejected": -1.571903109550476, + "step": 1862 + }, + { + "epoch": 1.1589424572317264, + "grad_norm": 0.6021642684936523, + "learning_rate": 3.4250000000000002e-06, + "log_odds_chosen": 9.942659378051758, + "log_odds_ratio": -0.0021312516182661057, + "logits/chosen": 0.46570685505867004, + "logits/rejected": 1.2161035537719727, + "logps/chosen": -0.8891934156417847, + "logps/rejected": -10.150493621826172, + "loss": 3.5963, + "nll_loss": 3.596067428588867, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08891934156417847, + "rewards/margins": 0.9261300563812256, + "rewards/rejected": -1.0150494575500488, + "step": 1863 + }, + { + "epoch": 1.1595645412130637, + "grad_norm": 0.5865781903266907, + "learning_rate": 3.4000000000000005e-06, + "log_odds_chosen": 11.61959171295166, + "log_odds_ratio": -0.0861300453543663, + "logits/chosen": 0.24954769015312195, + "logits/rejected": 0.8293600082397461, + "logps/chosen": -0.9044547080993652, + "logps/rejected": -11.792139053344727, + "loss": 3.3048, + "nll_loss": 3.296236515045166, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09044547379016876, + "rewards/margins": 1.088768482208252, + "rewards/rejected": -1.1792140007019043, + "step": 1864 + }, + { + "epoch": 1.1601866251944013, + "grad_norm": 0.6469467282295227, + "learning_rate": 3.3750000000000003e-06, + "log_odds_chosen": 14.448548316955566, + "log_odds_ratio": -0.00036828042357228696, + "logits/chosen": 0.365757554769516, + "logits/rejected": 1.18428635597229, + "logps/chosen": -0.9528321027755737, + "logps/rejected": -14.689295768737793, + "loss": 3.5187, + "nll_loss": 3.518705129623413, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09528321772813797, + "rewards/margins": 1.3736464977264404, + "rewards/rejected": -1.4689295291900635, + "step": 1865 + }, + { + "epoch": 1.1608087091757386, + "grad_norm": 0.9285478591918945, + "learning_rate": 3.3500000000000005e-06, + "log_odds_chosen": 14.083860397338867, + "log_odds_ratio": -0.07330363243818283, + "logits/chosen": 0.35714104771614075, + "logits/rejected": 1.0330679416656494, + "logps/chosen": -1.1653668880462646, + "logps/rejected": -14.7406005859375, + "loss": 2.6367, + "nll_loss": 2.6293601989746094, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1165367066860199, + "rewards/margins": 1.3575233221054077, + "rewards/rejected": -1.4740599393844604, + "step": 1866 + }, + { + "epoch": 1.1614307931570762, + "grad_norm": 0.9891173243522644, + "learning_rate": 3.3250000000000004e-06, + "log_odds_chosen": 9.35483169555664, + "log_odds_ratio": -0.08359131217002869, + "logits/chosen": -0.014555513858795166, + "logits/rejected": 0.7392335534095764, + "logps/chosen": -1.41230046749115, + "logps/rejected": -10.510553359985352, + "loss": 2.5851, + "nll_loss": 2.576704978942871, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.141230046749115, + "rewards/margins": 0.909825325012207, + "rewards/rejected": -1.0510554313659668, + "step": 1867 + }, + { + "epoch": 1.1620528771384138, + "grad_norm": 0.5011563897132874, + "learning_rate": 3.3e-06, + "log_odds_chosen": 13.634634017944336, + "log_odds_ratio": -0.0271232258528471, + "logits/chosen": 0.1796010583639145, + "logits/rejected": 0.7554137110710144, + "logps/chosen": -1.1477197408676147, + "logps/rejected": -14.320469856262207, + "loss": 3.1094, + "nll_loss": 3.106682300567627, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11477198451757431, + "rewards/margins": 1.317275047302246, + "rewards/rejected": -1.4320470094680786, + "step": 1868 + }, + { + "epoch": 1.162674961119751, + "grad_norm": 0.9561259746551514, + "learning_rate": 3.2750000000000004e-06, + "log_odds_chosen": 6.736767768859863, + "log_odds_ratio": -0.26292210817337036, + "logits/chosen": 0.23208418488502502, + "logits/rejected": 0.7597588300704956, + "logps/chosen": -1.131181001663208, + "logps/rejected": -7.403827667236328, + "loss": 3.0907, + "nll_loss": 3.06437349319458, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11311808228492737, + "rewards/margins": 0.6272647380828857, + "rewards/rejected": -0.7403827905654907, + "step": 1869 + }, + { + "epoch": 1.1632970451010887, + "grad_norm": 0.5985185503959656, + "learning_rate": 3.2500000000000002e-06, + "log_odds_chosen": 8.262455940246582, + "log_odds_ratio": -0.18344323337078094, + "logits/chosen": 0.3155595064163208, + "logits/rejected": 1.0940701961517334, + "logps/chosen": -1.1838172674179077, + "logps/rejected": -9.031631469726562, + "loss": 2.8803, + "nll_loss": 2.8619325160980225, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11838172376155853, + "rewards/margins": 0.7847813963890076, + "rewards/rejected": -0.9031630754470825, + "step": 1870 + }, + { + "epoch": 1.163919129082426, + "grad_norm": 0.4131832420825958, + "learning_rate": 3.225e-06, + "log_odds_chosen": 12.02147102355957, + "log_odds_ratio": -2.4021293938858435e-05, + "logits/chosen": 0.40716037154197693, + "logits/rejected": 1.4584306478500366, + "logps/chosen": -1.1131404638290405, + "logps/rejected": -12.671215057373047, + "loss": 3.6546, + "nll_loss": 3.6546096801757812, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11131405085325241, + "rewards/margins": 1.1558074951171875, + "rewards/rejected": -1.2671215534210205, + "step": 1871 + }, + { + "epoch": 1.1645412130637636, + "grad_norm": 0.4233200252056122, + "learning_rate": 3.2000000000000003e-06, + "log_odds_chosen": 4.853551864624023, + "log_odds_ratio": -0.28688034415245056, + "logits/chosen": 0.36333611607551575, + "logits/rejected": 0.8579514026641846, + "logps/chosen": -0.8897204399108887, + "logps/rejected": -5.366914749145508, + "loss": 3.5207, + "nll_loss": 3.4920222759246826, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08897203207015991, + "rewards/margins": 0.4477194547653198, + "rewards/rejected": -0.5366915464401245, + "step": 1872 + }, + { + "epoch": 1.1651632970451011, + "grad_norm": 0.765011191368103, + "learning_rate": 3.175e-06, + "log_odds_chosen": 12.80850887298584, + "log_odds_ratio": -0.07324609905481339, + "logits/chosen": 0.21928678452968597, + "logits/rejected": 0.8805117607116699, + "logps/chosen": -1.0177444219589233, + "logps/rejected": -13.257856369018555, + "loss": 2.8547, + "nll_loss": 2.847330331802368, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1017744317650795, + "rewards/margins": 1.2240111827850342, + "rewards/rejected": -1.325785517692566, + "step": 1873 + }, + { + "epoch": 1.1657853810264385, + "grad_norm": 0.777523934841156, + "learning_rate": 3.1500000000000003e-06, + "log_odds_chosen": 10.86896800994873, + "log_odds_ratio": -0.01855648122727871, + "logits/chosen": 0.12363356351852417, + "logits/rejected": 0.8761640787124634, + "logps/chosen": -0.9145961999893188, + "logps/rejected": -11.168540954589844, + "loss": 2.5963, + "nll_loss": 2.594484806060791, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09145961701869965, + "rewards/margins": 1.0253944396972656, + "rewards/rejected": -1.116853952407837, + "step": 1874 + }, + { + "epoch": 1.166407465007776, + "grad_norm": 0.7729889154434204, + "learning_rate": 3.125e-06, + "log_odds_chosen": 11.862380981445312, + "log_odds_ratio": -0.10916010290384293, + "logits/chosen": 0.27635759115219116, + "logits/rejected": 1.2793161869049072, + "logps/chosen": -1.297967791557312, + "logps/rejected": -12.863824844360352, + "loss": 2.5581, + "nll_loss": 2.5471975803375244, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12979678809642792, + "rewards/margins": 1.156585693359375, + "rewards/rejected": -1.2863824367523193, + "step": 1875 + }, + { + "epoch": 1.1670295489891136, + "grad_norm": 0.5068461894989014, + "learning_rate": 3.1e-06, + "log_odds_chosen": 10.792291641235352, + "log_odds_ratio": -0.23234255611896515, + "logits/chosen": 0.22562365233898163, + "logits/rejected": 0.9575470685958862, + "logps/chosen": -1.1553939580917358, + "logps/rejected": -11.633508682250977, + "loss": 3.1962, + "nll_loss": 3.173001289367676, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11553940176963806, + "rewards/margins": 1.0478113889694214, + "rewards/rejected": -1.1633508205413818, + "step": 1876 + }, + { + "epoch": 1.167651632970451, + "grad_norm": 0.5159857273101807, + "learning_rate": 3.075e-06, + "log_odds_chosen": 11.215340614318848, + "log_odds_ratio": -0.0021687590051442385, + "logits/chosen": 0.31632381677627563, + "logits/rejected": 0.8091593384742737, + "logps/chosen": -1.214293122291565, + "logps/rejected": -11.843792915344238, + "loss": 3.2419, + "nll_loss": 3.2416889667510986, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12142931669950485, + "rewards/margins": 1.0629500150680542, + "rewards/rejected": -1.1843793392181396, + "step": 1877 + }, + { + "epoch": 1.1682737169517885, + "grad_norm": 0.7218311429023743, + "learning_rate": 3.05e-06, + "log_odds_chosen": 10.48986530303955, + "log_odds_ratio": -0.13859295845031738, + "logits/chosen": 0.20560768246650696, + "logits/rejected": 0.9777731895446777, + "logps/chosen": -1.1056513786315918, + "logps/rejected": -11.218057632446289, + "loss": 2.6993, + "nll_loss": 2.685471534729004, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11056514084339142, + "rewards/margins": 1.0112406015396118, + "rewards/rejected": -1.1218056678771973, + "step": 1878 + }, + { + "epoch": 1.168895800933126, + "grad_norm": 0.5191105604171753, + "learning_rate": 3.0250000000000003e-06, + "log_odds_chosen": 10.223533630371094, + "log_odds_ratio": -0.09211105108261108, + "logits/chosen": 0.2968738079071045, + "logits/rejected": 1.1603482961654663, + "logps/chosen": -1.0198242664337158, + "logps/rejected": -10.802665710449219, + "loss": 2.9144, + "nll_loss": 2.9052276611328125, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10198242217302322, + "rewards/margins": 0.9782842993736267, + "rewards/rejected": -1.0802667140960693, + "step": 1879 + }, + { + "epoch": 1.1695178849144634, + "grad_norm": 0.8860030770301819, + "learning_rate": 3e-06, + "log_odds_chosen": 12.134418487548828, + "log_odds_ratio": -0.02829170599579811, + "logits/chosen": 0.06693208962678909, + "logits/rejected": 1.0038888454437256, + "logps/chosen": -0.9355678558349609, + "logps/rejected": -12.557945251464844, + "loss": 2.4539, + "nll_loss": 2.45109486579895, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09355678409337997, + "rewards/margins": 1.1622377634048462, + "rewards/rejected": -1.2557945251464844, + "step": 1880 + }, + { + "epoch": 1.170139968895801, + "grad_norm": 0.536885678768158, + "learning_rate": 2.975e-06, + "log_odds_chosen": 11.629366874694824, + "log_odds_ratio": -0.14055614173412323, + "logits/chosen": 0.39352768659591675, + "logits/rejected": 0.9422281980514526, + "logps/chosen": -1.0389788150787354, + "logps/rejected": -12.211631774902344, + "loss": 3.2538, + "nll_loss": 3.2397899627685547, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10389788448810577, + "rewards/margins": 1.1172653436660767, + "rewards/rejected": -1.2211631536483765, + "step": 1881 + }, + { + "epoch": 1.1707620528771385, + "grad_norm": 0.5991978645324707, + "learning_rate": 2.95e-06, + "log_odds_chosen": 11.063156127929688, + "log_odds_ratio": -0.03398752585053444, + "logits/chosen": 0.30659687519073486, + "logits/rejected": 1.1551581621170044, + "logps/chosen": -1.0710886716842651, + "logps/rejected": -11.712461471557617, + "loss": 3.1061, + "nll_loss": 3.1026854515075684, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10710887610912323, + "rewards/margins": 1.06413733959198, + "rewards/rejected": -1.1712462902069092, + "step": 1882 + }, + { + "epoch": 1.1713841368584759, + "grad_norm": 0.41771650314331055, + "learning_rate": 2.9250000000000004e-06, + "log_odds_chosen": 11.414490699768066, + "log_odds_ratio": -0.0612252876162529, + "logits/chosen": 0.31374531984329224, + "logits/rejected": 1.0078083276748657, + "logps/chosen": -0.7967260479927063, + "logps/rejected": -11.597383499145508, + "loss": 3.441, + "nll_loss": 3.4348974227905273, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07967260479927063, + "rewards/margins": 1.0800657272338867, + "rewards/rejected": -1.159738302230835, + "step": 1883 + }, + { + "epoch": 1.1720062208398134, + "grad_norm": 0.5465978384017944, + "learning_rate": 2.9e-06, + "log_odds_chosen": 12.942608833312988, + "log_odds_ratio": -0.027714960277080536, + "logits/chosen": 0.2509208023548126, + "logits/rejected": 1.0975139141082764, + "logps/chosen": -0.9628081321716309, + "logps/rejected": -13.391410827636719, + "loss": 2.9462, + "nll_loss": 2.943429470062256, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09628081321716309, + "rewards/margins": 1.2428603172302246, + "rewards/rejected": -1.3391410112380981, + "step": 1884 + }, + { + "epoch": 1.1726283048211508, + "grad_norm": 0.5699530839920044, + "learning_rate": 2.8750000000000004e-06, + "log_odds_chosen": 9.245712280273438, + "log_odds_ratio": -0.09234052896499634, + "logits/chosen": 0.12563082575798035, + "logits/rejected": 0.7737298011779785, + "logps/chosen": -0.9688528776168823, + "logps/rejected": -9.617731094360352, + "loss": 2.6494, + "nll_loss": 2.640190362930298, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09688529372215271, + "rewards/margins": 0.8648878931999207, + "rewards/rejected": -0.961773157119751, + "step": 1885 + }, + { + "epoch": 1.1732503888024883, + "grad_norm": 0.5037267208099365, + "learning_rate": 2.8500000000000002e-06, + "log_odds_chosen": 11.182029724121094, + "log_odds_ratio": -0.0029421194922178984, + "logits/chosen": 0.27966320514678955, + "logits/rejected": 1.0112600326538086, + "logps/chosen": -0.8653491735458374, + "logps/rejected": -11.44352912902832, + "loss": 3.3134, + "nll_loss": 3.313063144683838, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08653491735458374, + "rewards/margins": 1.0578179359436035, + "rewards/rejected": -1.144352912902832, + "step": 1886 + }, + { + "epoch": 1.173872472783826, + "grad_norm": 0.6436689496040344, + "learning_rate": 2.825e-06, + "log_odds_chosen": 8.10422420501709, + "log_odds_ratio": -0.1899256408214569, + "logits/chosen": 0.2365320920944214, + "logits/rejected": 0.6297204494476318, + "logps/chosen": -0.9122384190559387, + "logps/rejected": -8.375236511230469, + "loss": 3.0991, + "nll_loss": 3.0801029205322266, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0912238359451294, + "rewards/margins": 0.7462998628616333, + "rewards/rejected": -0.8375236988067627, + "step": 1887 + }, + { + "epoch": 1.1744945567651632, + "grad_norm": 0.7984195947647095, + "learning_rate": 2.8000000000000003e-06, + "log_odds_chosen": 6.958744049072266, + "log_odds_ratio": -0.2660243809223175, + "logits/chosen": 0.23720431327819824, + "logits/rejected": 0.4664103388786316, + "logps/chosen": -1.3364598751068115, + "logps/rejected": -8.00857925415039, + "loss": 3.1654, + "nll_loss": 3.138822317123413, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13364599645137787, + "rewards/margins": 0.6672119498252869, + "rewards/rejected": -0.8008580207824707, + "step": 1888 + }, + { + "epoch": 1.1751166407465008, + "grad_norm": 0.3695003092288971, + "learning_rate": 2.775e-06, + "log_odds_chosen": 10.706849098205566, + "log_odds_ratio": -0.013503124937415123, + "logits/chosen": 0.39301347732543945, + "logits/rejected": 1.17519211769104, + "logps/chosen": -1.2024290561676025, + "logps/rejected": -11.415552139282227, + "loss": 3.8668, + "nll_loss": 3.865440607070923, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12024290859699249, + "rewards/margins": 1.0213122367858887, + "rewards/rejected": -1.1415551900863647, + "step": 1889 + }, + { + "epoch": 1.1757387247278381, + "grad_norm": 0.4984581768512726, + "learning_rate": 2.7500000000000004e-06, + "log_odds_chosen": 9.745952606201172, + "log_odds_ratio": -0.015363501384854317, + "logits/chosen": 0.22919657826423645, + "logits/rejected": 0.6598438024520874, + "logps/chosen": -0.9327662587165833, + "logps/rejected": -9.936866760253906, + "loss": 3.3242, + "nll_loss": 3.322669506072998, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09327662736177444, + "rewards/margins": 0.9004101157188416, + "rewards/rejected": -0.9936867356300354, + "step": 1890 + }, + { + "epoch": 1.1763608087091757, + "grad_norm": 0.47122901678085327, + "learning_rate": 2.725e-06, + "log_odds_chosen": 5.912145614624023, + "log_odds_ratio": -0.23905989527702332, + "logits/chosen": 0.11490034312009811, + "logits/rejected": 0.5542351007461548, + "logps/chosen": -1.2273865938186646, + "logps/rejected": -6.7846503257751465, + "loss": 3.0799, + "nll_loss": 3.0560312271118164, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12273865938186646, + "rewards/margins": 0.5557263493537903, + "rewards/rejected": -0.6784650087356567, + "step": 1891 + }, + { + "epoch": 1.1769828926905133, + "grad_norm": 0.5710389018058777, + "learning_rate": 2.7e-06, + "log_odds_chosen": 8.211050987243652, + "log_odds_ratio": -0.14245814085006714, + "logits/chosen": 0.22267159819602966, + "logits/rejected": 0.8585968613624573, + "logps/chosen": -0.9339166879653931, + "logps/rejected": -8.653953552246094, + "loss": 2.8897, + "nll_loss": 2.875443458557129, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09339166432619095, + "rewards/margins": 0.7720035910606384, + "rewards/rejected": -0.8653952479362488, + "step": 1892 + }, + { + "epoch": 1.1776049766718506, + "grad_norm": 0.8877511024475098, + "learning_rate": 2.6750000000000002e-06, + "log_odds_chosen": 11.314347267150879, + "log_odds_ratio": -0.1028999611735344, + "logits/chosen": 0.08349957317113876, + "logits/rejected": 0.7912660837173462, + "logps/chosen": -0.825186550617218, + "logps/rejected": -11.350780487060547, + "loss": 2.194, + "nll_loss": 2.183689832687378, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08251865208148956, + "rewards/margins": 1.0525593757629395, + "rewards/rejected": -1.135077953338623, + "step": 1893 + }, + { + "epoch": 1.1782270606531882, + "grad_norm": 0.6872034072875977, + "learning_rate": 2.65e-06, + "log_odds_chosen": 11.63272762298584, + "log_odds_ratio": -0.024180106818675995, + "logits/chosen": 0.2922358214855194, + "logits/rejected": 1.0772125720977783, + "logps/chosen": -1.1416473388671875, + "logps/rejected": -12.256592750549316, + "loss": 2.7443, + "nll_loss": 2.7419087886810303, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11416472494602203, + "rewards/margins": 1.111494541168213, + "rewards/rejected": -1.2256593704223633, + "step": 1894 + }, + { + "epoch": 1.1788491446345257, + "grad_norm": 0.5088832974433899, + "learning_rate": 2.625e-06, + "log_odds_chosen": 7.584236145019531, + "log_odds_ratio": -0.17331501841545105, + "logits/chosen": 0.10206744074821472, + "logits/rejected": 0.6197749376296997, + "logps/chosen": -1.1107993125915527, + "logps/rejected": -8.259223937988281, + "loss": 2.9389, + "nll_loss": 2.92161226272583, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11107993870973587, + "rewards/margins": 0.7148424983024597, + "rewards/rejected": -0.8259223699569702, + "step": 1895 + }, + { + "epoch": 1.179471228615863, + "grad_norm": 2.0029592514038086, + "learning_rate": 2.6e-06, + "log_odds_chosen": 8.409797668457031, + "log_odds_ratio": -0.11552348732948303, + "logits/chosen": 0.2863270342350006, + "logits/rejected": 0.6867630481719971, + "logps/chosen": -1.2449003458023071, + "logps/rejected": -9.326054573059082, + "loss": 3.5466, + "nll_loss": 3.53503680229187, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12449003756046295, + "rewards/margins": 0.8081154227256775, + "rewards/rejected": -0.932605504989624, + "step": 1896 + }, + { + "epoch": 1.1800933125972006, + "grad_norm": 3.2555623054504395, + "learning_rate": 2.575e-06, + "log_odds_chosen": 9.229421615600586, + "log_odds_ratio": -0.47473472356796265, + "logits/chosen": 0.11302335560321808, + "logits/rejected": 0.9297094345092773, + "logps/chosen": -1.431499719619751, + "logps/rejected": -9.87209701538086, + "loss": 2.771, + "nll_loss": 2.7234785556793213, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1431499719619751, + "rewards/margins": 0.8440598249435425, + "rewards/rejected": -0.9872097373008728, + "step": 1897 + }, + { + "epoch": 1.1807153965785382, + "grad_norm": 0.5698276162147522, + "learning_rate": 2.55e-06, + "log_odds_chosen": 7.369045257568359, + "log_odds_ratio": -0.22586429119110107, + "logits/chosen": 0.23034963011741638, + "logits/rejected": 0.6743384003639221, + "logps/chosen": -1.0172677040100098, + "logps/rejected": -7.886266708374023, + "loss": 3.0338, + "nll_loss": 3.0111746788024902, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10172677040100098, + "rewards/margins": 0.6868999004364014, + "rewards/rejected": -0.7886266708374023, + "step": 1898 + }, + { + "epoch": 1.1813374805598755, + "grad_norm": 0.622840404510498, + "learning_rate": 2.5250000000000004e-06, + "log_odds_chosen": 11.24142837524414, + "log_odds_ratio": -0.08297882229089737, + "logits/chosen": 0.16179004311561584, + "logits/rejected": 1.013938069343567, + "logps/chosen": -1.1062037944793701, + "logps/rejected": -11.925838470458984, + "loss": 2.5435, + "nll_loss": 2.5351955890655518, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11062037199735641, + "rewards/margins": 1.0819635391235352, + "rewards/rejected": -1.1925837993621826, + "step": 1899 + }, + { + "epoch": 1.181959564541213, + "grad_norm": 0.5149745345115662, + "learning_rate": 2.5e-06, + "log_odds_chosen": 7.838262557983398, + "log_odds_ratio": -0.16004499793052673, + "logits/chosen": 0.2962034046649933, + "logits/rejected": 0.741732120513916, + "logps/chosen": -0.8723784685134888, + "logps/rejected": -8.200945854187012, + "loss": 3.3196, + "nll_loss": 3.3036234378814697, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08723784238100052, + "rewards/margins": 0.7328566908836365, + "rewards/rejected": -0.8200945854187012, + "step": 1900 + }, + { + "epoch": 1.1825816485225507, + "grad_norm": 0.4984101951122284, + "learning_rate": 2.4750000000000004e-06, + "log_odds_chosen": 14.134041786193848, + "log_odds_ratio": -6.005212526360992e-06, + "logits/chosen": 0.24021825194358826, + "logits/rejected": 1.185408592224121, + "logps/chosen": -1.0472640991210938, + "logps/rejected": -14.570968627929688, + "loss": 3.3729, + "nll_loss": 3.372877359390259, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10472641885280609, + "rewards/margins": 1.3523705005645752, + "rewards/rejected": -1.457096815109253, + "step": 1901 + }, + { + "epoch": 1.183203732503888, + "grad_norm": 0.8151748180389404, + "learning_rate": 2.4500000000000003e-06, + "log_odds_chosen": 12.901820182800293, + "log_odds_ratio": -0.18067723512649536, + "logits/chosen": 0.2967347502708435, + "logits/rejected": 0.981220006942749, + "logps/chosen": -1.1056855916976929, + "logps/rejected": -13.67314624786377, + "loss": 2.6885, + "nll_loss": 2.67038631439209, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1105685606598854, + "rewards/margins": 1.2567460536956787, + "rewards/rejected": -1.3673145771026611, + "step": 1902 + }, + { + "epoch": 1.1838258164852256, + "grad_norm": 0.5125972628593445, + "learning_rate": 2.425e-06, + "log_odds_chosen": 8.416955947875977, + "log_odds_ratio": -0.2502775192260742, + "logits/chosen": 0.32387620210647583, + "logits/rejected": 0.6450578570365906, + "logps/chosen": -1.0750399827957153, + "logps/rejected": -9.137263298034668, + "loss": 3.3234, + "nll_loss": 3.2983758449554443, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1075039878487587, + "rewards/margins": 0.8062223792076111, + "rewards/rejected": -0.9137263298034668, + "step": 1903 + }, + { + "epoch": 1.184447900466563, + "grad_norm": 8.312177658081055, + "learning_rate": 2.4000000000000003e-06, + "log_odds_chosen": 5.083847522735596, + "log_odds_ratio": -0.8792902231216431, + "logits/chosen": 0.3725608289241791, + "logits/rejected": 0.6591469049453735, + "logps/chosen": -1.7544732093811035, + "logps/rejected": -6.63493013381958, + "loss": 2.8066, + "nll_loss": 2.718681812286377, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.17544731497764587, + "rewards/margins": 0.48804575204849243, + "rewards/rejected": -0.6634930372238159, + "step": 1904 + }, + { + "epoch": 1.1850699844479005, + "grad_norm": 0.5036944150924683, + "learning_rate": 2.375e-06, + "log_odds_chosen": 11.233821868896484, + "log_odds_ratio": -0.07140740752220154, + "logits/chosen": 0.28917139768600464, + "logits/rejected": 1.0645439624786377, + "logps/chosen": -0.9894380569458008, + "logps/rejected": -11.69849967956543, + "loss": 3.0867, + "nll_loss": 3.0795483589172363, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0989438146352768, + "rewards/margins": 1.0709060430526733, + "rewards/rejected": -1.1698498725891113, + "step": 1905 + }, + { + "epoch": 1.185692068429238, + "grad_norm": 0.6781418323516846, + "learning_rate": 2.35e-06, + "log_odds_chosen": 11.125399589538574, + "log_odds_ratio": -0.11678078025579453, + "logits/chosen": 0.1555495262145996, + "logits/rejected": 0.8147391080856323, + "logps/chosen": -0.9576234221458435, + "logps/rejected": -11.602968215942383, + "loss": 2.5192, + "nll_loss": 2.507535219192505, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09576234221458435, + "rewards/margins": 1.0645344257354736, + "rewards/rejected": -1.1602967977523804, + "step": 1906 + }, + { + "epoch": 1.1863141524105754, + "grad_norm": 0.44551971554756165, + "learning_rate": 2.325e-06, + "log_odds_chosen": 12.629438400268555, + "log_odds_ratio": -0.014774455688893795, + "logits/chosen": 0.3876957595348358, + "logits/rejected": 1.470685362815857, + "logps/chosen": -1.0328607559204102, + "logps/rejected": -13.184697151184082, + "loss": 3.1014, + "nll_loss": 3.099900960922241, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10328607261180878, + "rewards/margins": 1.2151837348937988, + "rewards/rejected": -1.318469762802124, + "step": 1907 + }, + { + "epoch": 1.186936236391913, + "grad_norm": 0.7646956443786621, + "learning_rate": 2.3e-06, + "log_odds_chosen": 12.166587829589844, + "log_odds_ratio": -0.05765797197818756, + "logits/chosen": 0.31056147813796997, + "logits/rejected": 1.0030930042266846, + "logps/chosen": -0.8595593571662903, + "logps/rejected": -12.483160018920898, + "loss": 2.7176, + "nll_loss": 2.7118818759918213, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08595594763755798, + "rewards/margins": 1.1623600721359253, + "rewards/rejected": -1.2483160495758057, + "step": 1908 + }, + { + "epoch": 1.1875583203732503, + "grad_norm": 0.5104154348373413, + "learning_rate": 2.2750000000000002e-06, + "log_odds_chosen": 10.230204582214355, + "log_odds_ratio": -0.021832279860973358, + "logits/chosen": 0.19930848479270935, + "logits/rejected": 0.8651219606399536, + "logps/chosen": -1.2513785362243652, + "logps/rejected": -11.117950439453125, + "loss": 3.0008, + "nll_loss": 2.9986038208007812, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12513785064220428, + "rewards/margins": 0.9866572618484497, + "rewards/rejected": -1.11179518699646, + "step": 1909 + }, + { + "epoch": 1.1881804043545878, + "grad_norm": 0.45462679862976074, + "learning_rate": 2.25e-06, + "log_odds_chosen": 10.267870903015137, + "log_odds_ratio": -0.04462050646543503, + "logits/chosen": 0.2697553336620331, + "logits/rejected": 1.0339744091033936, + "logps/chosen": -0.8061079382896423, + "logps/rejected": -10.42862319946289, + "loss": 2.9761, + "nll_loss": 2.9715969562530518, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08061079680919647, + "rewards/margins": 0.9622515439987183, + "rewards/rejected": -1.0428624153137207, + "step": 1910 + }, + { + "epoch": 1.1888024883359254, + "grad_norm": 0.6434316635131836, + "learning_rate": 2.225e-06, + "log_odds_chosen": 15.738075256347656, + "log_odds_ratio": -1.4901207805451122e-06, + "logits/chosen": 0.23433563113212585, + "logits/rejected": 1.4968116283416748, + "logps/chosen": -1.2706167697906494, + "logps/rejected": -16.622678756713867, + "loss": 2.814, + "nll_loss": 2.8140363693237305, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12706167995929718, + "rewards/margins": 1.5352060794830322, + "rewards/rejected": -1.662267804145813, + "step": 1911 + }, + { + "epoch": 1.1894245723172627, + "grad_norm": 0.5044808387756348, + "learning_rate": 2.2e-06, + "log_odds_chosen": 10.514906883239746, + "log_odds_ratio": -0.1468035727739334, + "logits/chosen": 0.1761753112077713, + "logits/rejected": 0.8280108571052551, + "logps/chosen": -1.1078742742538452, + "logps/rejected": -11.255012512207031, + "loss": 2.6505, + "nll_loss": 2.6358470916748047, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11078742891550064, + "rewards/margins": 1.0147137641906738, + "rewards/rejected": -1.125501275062561, + "step": 1912 + }, + { + "epoch": 1.1900466562986003, + "grad_norm": 0.5661005973815918, + "learning_rate": 2.175e-06, + "log_odds_chosen": 10.606189727783203, + "log_odds_ratio": -0.3332817554473877, + "logits/chosen": 0.35058706998825073, + "logits/rejected": 0.8177105188369751, + "logps/chosen": -0.9934715032577515, + "logps/rejected": -11.091846466064453, + "loss": 3.4516, + "nll_loss": 3.4183170795440674, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09934714436531067, + "rewards/margins": 1.0098373889923096, + "rewards/rejected": -1.1091846227645874, + "step": 1913 + }, + { + "epoch": 1.1906687402799379, + "grad_norm": 4.56205940246582, + "learning_rate": 2.1499999999999997e-06, + "log_odds_chosen": 10.877635955810547, + "log_odds_ratio": -0.10927483439445496, + "logits/chosen": 0.3463170528411865, + "logits/rejected": 0.9182144403457642, + "logps/chosen": -1.5428080558776855, + "logps/rejected": -12.063437461853027, + "loss": 3.6504, + "nll_loss": 3.639470100402832, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15428081154823303, + "rewards/margins": 1.05206298828125, + "rewards/rejected": -1.2063437700271606, + "step": 1914 + }, + { + "epoch": 1.1912908242612752, + "grad_norm": 0.582009494304657, + "learning_rate": 2.1250000000000004e-06, + "log_odds_chosen": 11.911093711853027, + "log_odds_ratio": -0.023971393704414368, + "logits/chosen": 0.4232202172279358, + "logits/rejected": 1.2884031534194946, + "logps/chosen": -1.0782442092895508, + "logps/rejected": -12.535236358642578, + "loss": 3.2242, + "nll_loss": 3.2217798233032227, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1078244298696518, + "rewards/margins": 1.1456992626190186, + "rewards/rejected": -1.2535237073898315, + "step": 1915 + }, + { + "epoch": 1.1919129082426128, + "grad_norm": 0.4537990987300873, + "learning_rate": 2.1000000000000002e-06, + "log_odds_chosen": 12.903481483459473, + "log_odds_ratio": -0.002569683827459812, + "logits/chosen": 0.23533426225185394, + "logits/rejected": 1.1300369501113892, + "logps/chosen": -1.0497938394546509, + "logps/rejected": -13.376167297363281, + "loss": 2.9572, + "nll_loss": 2.956960678100586, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10497938096523285, + "rewards/margins": 1.2326374053955078, + "rewards/rejected": -1.3376166820526123, + "step": 1916 + }, + { + "epoch": 1.1925349922239503, + "grad_norm": 0.5528674721717834, + "learning_rate": 2.075e-06, + "log_odds_chosen": 8.131418228149414, + "log_odds_ratio": -0.12049117684364319, + "logits/chosen": 0.35378021001815796, + "logits/rejected": 1.0061860084533691, + "logps/chosen": -0.9170488119125366, + "logps/rejected": -8.42977523803711, + "loss": 3.5667, + "nll_loss": 3.5546751022338867, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09170487523078918, + "rewards/margins": 0.7512726783752441, + "rewards/rejected": -0.8429775238037109, + "step": 1917 + }, + { + "epoch": 1.1931570762052877, + "grad_norm": 0.46475347876548767, + "learning_rate": 2.0500000000000003e-06, + "log_odds_chosen": 9.595006942749023, + "log_odds_ratio": -0.13916951417922974, + "logits/chosen": 0.2627370357513428, + "logits/rejected": 1.1726372241973877, + "logps/chosen": -1.1297882795333862, + "logps/rejected": -10.375534057617188, + "loss": 3.1715, + "nll_loss": 3.1576147079467773, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11297882348299026, + "rewards/margins": 0.9245747327804565, + "rewards/rejected": -1.0375535488128662, + "step": 1918 + }, + { + "epoch": 1.1937791601866252, + "grad_norm": 0.8667030930519104, + "learning_rate": 2.025e-06, + "log_odds_chosen": 9.676931381225586, + "log_odds_ratio": -0.1655566692352295, + "logits/chosen": 0.3479841351509094, + "logits/rejected": 1.114953637123108, + "logps/chosen": -0.9280308485031128, + "logps/rejected": -10.123562812805176, + "loss": 3.0449, + "nll_loss": 3.0283515453338623, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09280309081077576, + "rewards/margins": 0.9195532202720642, + "rewards/rejected": -1.0123562812805176, + "step": 1919 + }, + { + "epoch": 1.1944012441679628, + "grad_norm": 0.448517382144928, + "learning_rate": 2.0000000000000003e-06, + "log_odds_chosen": 9.664508819580078, + "log_odds_ratio": -0.030703043565154076, + "logits/chosen": 0.23289135098457336, + "logits/rejected": 0.8441465497016907, + "logps/chosen": -0.8699073791503906, + "logps/rejected": -9.8641996383667, + "loss": 3.4263, + "nll_loss": 3.423267364501953, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08699073642492294, + "rewards/margins": 0.8994292616844177, + "rewards/rejected": -0.9864200353622437, + "step": 1920 + }, + { + "epoch": 1.1950233281493001, + "grad_norm": 0.5516140460968018, + "learning_rate": 1.975e-06, + "log_odds_chosen": 9.347766876220703, + "log_odds_ratio": -0.031423501670360565, + "logits/chosen": 0.2751286029815674, + "logits/rejected": 0.904050350189209, + "logps/chosen": -1.0117491483688354, + "logps/rejected": -9.64893627166748, + "loss": 3.015, + "nll_loss": 3.0118672847747803, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10117491334676743, + "rewards/margins": 0.8637187480926514, + "rewards/rejected": -0.964893639087677, + "step": 1921 + }, + { + "epoch": 1.1956454121306377, + "grad_norm": 0.5590149760246277, + "learning_rate": 1.95e-06, + "log_odds_chosen": 9.179625511169434, + "log_odds_ratio": -0.14231599867343903, + "logits/chosen": 0.31218940019607544, + "logits/rejected": 0.9436146020889282, + "logps/chosen": -1.2564001083374023, + "logps/rejected": -10.086761474609375, + "loss": 3.4529, + "nll_loss": 3.4386985301971436, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12564000487327576, + "rewards/margins": 0.8830360770225525, + "rewards/rejected": -1.0086760520935059, + "step": 1922 + }, + { + "epoch": 1.196267496111975, + "grad_norm": 0.5352418422698975, + "learning_rate": 1.925e-06, + "log_odds_chosen": 9.404342651367188, + "log_odds_ratio": -0.005691325291991234, + "logits/chosen": 0.41761916875839233, + "logits/rejected": 1.0610673427581787, + "logps/chosen": -1.1509430408477783, + "logps/rejected": -10.142035484313965, + "loss": 3.3106, + "nll_loss": 3.3100385665893555, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11509430408477783, + "rewards/margins": 0.8991092443466187, + "rewards/rejected": -1.0142035484313965, + "step": 1923 + }, + { + "epoch": 1.1968895800933126, + "grad_norm": 0.6540836095809937, + "learning_rate": 1.9e-06, + "log_odds_chosen": 12.195796966552734, + "log_odds_ratio": -0.026557566598057747, + "logits/chosen": 0.18317702412605286, + "logits/rejected": 0.8327803611755371, + "logps/chosen": -1.0502777099609375, + "logps/rejected": -12.741544723510742, + "loss": 2.6182, + "nll_loss": 2.6155149936676025, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10502777993679047, + "rewards/margins": 1.1691266298294067, + "rewards/rejected": -1.2741544246673584, + "step": 1924 + }, + { + "epoch": 1.1975116640746502, + "grad_norm": 1.1352347135543823, + "learning_rate": 1.875e-06, + "log_odds_chosen": 11.687995910644531, + "log_odds_ratio": -0.10948586463928223, + "logits/chosen": 0.2166191041469574, + "logits/rejected": 1.157933235168457, + "logps/chosen": -0.9563741683959961, + "logps/rejected": -12.046046257019043, + "loss": 2.6403, + "nll_loss": 2.6293962001800537, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09563742578029633, + "rewards/margins": 1.1089673042297363, + "rewards/rejected": -1.2046046257019043, + "step": 1925 + }, + { + "epoch": 1.1981337480559875, + "grad_norm": 0.6544418334960938, + "learning_rate": 1.85e-06, + "log_odds_chosen": 7.253926753997803, + "log_odds_ratio": -0.11793742328882217, + "logits/chosen": 0.31673333048820496, + "logits/rejected": 0.8479666709899902, + "logps/chosen": -0.9824406504631042, + "logps/rejected": -7.802856922149658, + "loss": 2.611, + "nll_loss": 2.599242687225342, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0982440710067749, + "rewards/margins": 0.6820417046546936, + "rewards/rejected": -0.7802857756614685, + "step": 1926 + }, + { + "epoch": 1.198755832037325, + "grad_norm": 0.4343709945678711, + "learning_rate": 1.8249999999999999e-06, + "log_odds_chosen": 10.073572158813477, + "log_odds_ratio": -0.23366577923297882, + "logits/chosen": 0.3042418360710144, + "logits/rejected": 0.5469282865524292, + "logps/chosen": -0.8838576078414917, + "logps/rejected": -10.546527862548828, + "loss": 3.3211, + "nll_loss": 3.2977826595306396, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08838576078414917, + "rewards/margins": 0.9662670493125916, + "rewards/rejected": -1.0546526908874512, + "step": 1927 + }, + { + "epoch": 1.1993779160186626, + "grad_norm": 0.5009332299232483, + "learning_rate": 1.8e-06, + "log_odds_chosen": 12.118463516235352, + "log_odds_ratio": -0.14113126695156097, + "logits/chosen": 0.27596867084503174, + "logits/rejected": 1.2588975429534912, + "logps/chosen": -1.0183926820755005, + "logps/rejected": -12.717033386230469, + "loss": 3.1475, + "nll_loss": 3.1334147453308105, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10183926671743393, + "rewards/margins": 1.1698640584945679, + "rewards/rejected": -1.2717033624649048, + "step": 1928 + }, + { + "epoch": 1.2, + "grad_norm": 0.596346914768219, + "learning_rate": 1.775e-06, + "log_odds_chosen": 17.941753387451172, + "log_odds_ratio": -1.1920931797249068e-07, + "logits/chosen": 0.3379727900028229, + "logits/rejected": 1.684706449508667, + "logps/chosen": -1.0429511070251465, + "logps/rejected": -18.38841438293457, + "loss": 2.7556, + "nll_loss": 2.7555999755859375, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10429511219263077, + "rewards/margins": 1.7345463037490845, + "rewards/rejected": -1.838841438293457, + "step": 1929 + }, + { + "epoch": 1.2006220839813375, + "grad_norm": 0.6008011102676392, + "learning_rate": 1.7500000000000002e-06, + "log_odds_chosen": 11.824764251708984, + "log_odds_ratio": -0.3278602361679077, + "logits/chosen": 0.272664338350296, + "logits/rejected": 0.922380805015564, + "logps/chosen": -1.0240790843963623, + "logps/rejected": -12.52273178100586, + "loss": 2.7195, + "nll_loss": 2.6867575645446777, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10240790247917175, + "rewards/margins": 1.1498652696609497, + "rewards/rejected": -1.2522730827331543, + "step": 1930 + }, + { + "epoch": 1.2012441679626749, + "grad_norm": 0.6269004344940186, + "learning_rate": 1.7250000000000002e-06, + "log_odds_chosen": 11.44296646118164, + "log_odds_ratio": -0.16116118431091309, + "logits/chosen": 0.34534481167793274, + "logits/rejected": 0.9442611336708069, + "logps/chosen": -0.6290194988250732, + "logps/rejected": -11.348631858825684, + "loss": 3.2711, + "nll_loss": 3.2550272941589355, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.06290195137262344, + "rewards/margins": 1.0719611644744873, + "rewards/rejected": -1.134863257408142, + "step": 1931 + }, + { + "epoch": 1.2018662519440124, + "grad_norm": 0.5232813954353333, + "learning_rate": 1.7000000000000002e-06, + "log_odds_chosen": 5.6479949951171875, + "log_odds_ratio": -0.29262030124664307, + "logits/chosen": 0.2010812908411026, + "logits/rejected": 0.22653111815452576, + "logps/chosen": -0.8776240348815918, + "logps/rejected": -6.111688137054443, + "loss": 3.0866, + "nll_loss": 3.057365894317627, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08776240050792694, + "rewards/margins": 0.5234063863754272, + "rewards/rejected": -0.6111688017845154, + "step": 1932 + }, + { + "epoch": 1.20248833592535, + "grad_norm": 0.6146247386932373, + "learning_rate": 1.6750000000000003e-06, + "log_odds_chosen": 11.847469329833984, + "log_odds_ratio": -0.035451874136924744, + "logits/chosen": 0.26600876450538635, + "logits/rejected": 0.7060739398002625, + "logps/chosen": -1.013869047164917, + "logps/rejected": -12.30055046081543, + "loss": 3.0871, + "nll_loss": 3.0835165977478027, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1013869121670723, + "rewards/margins": 1.1286683082580566, + "rewards/rejected": -1.2300550937652588, + "step": 1933 + }, + { + "epoch": 1.2031104199066873, + "grad_norm": 0.5205911993980408, + "learning_rate": 1.65e-06, + "log_odds_chosen": 6.819182872772217, + "log_odds_ratio": -0.23442748188972473, + "logits/chosen": 0.2185009866952896, + "logits/rejected": 0.6514573693275452, + "logps/chosen": -0.9856964349746704, + "logps/rejected": -7.467264652252197, + "loss": 3.0668, + "nll_loss": 3.0433638095855713, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09856964647769928, + "rewards/margins": 0.6481568813323975, + "rewards/rejected": -0.7467264533042908, + "step": 1934 + }, + { + "epoch": 1.2037325038880249, + "grad_norm": 0.349277138710022, + "learning_rate": 1.6250000000000001e-06, + "log_odds_chosen": 11.960783004760742, + "log_odds_ratio": -0.05431406572461128, + "logits/chosen": 0.19932757318019867, + "logits/rejected": 0.8002393245697021, + "logps/chosen": -0.8757328391075134, + "logps/rejected": -12.314295768737793, + "loss": 3.3128, + "nll_loss": 3.3073294162750244, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08757328987121582, + "rewards/margins": 1.1438562870025635, + "rewards/rejected": -1.2314295768737793, + "step": 1935 + }, + { + "epoch": 1.2043545878693624, + "grad_norm": 0.5340843200683594, + "learning_rate": 1.6000000000000001e-06, + "log_odds_chosen": 13.420051574707031, + "log_odds_ratio": -0.00024055530957411975, + "logits/chosen": 0.36679941415786743, + "logits/rejected": 1.3175077438354492, + "logps/chosen": -1.0074667930603027, + "logps/rejected": -13.865694046020508, + "loss": 2.8256, + "nll_loss": 2.8255414962768555, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10074667632579803, + "rewards/margins": 1.2858227491378784, + "rewards/rejected": -1.3865693807601929, + "step": 1936 + }, + { + "epoch": 1.2049766718506998, + "grad_norm": 0.6181488037109375, + "learning_rate": 1.5750000000000002e-06, + "log_odds_chosen": 6.692278861999512, + "log_odds_ratio": -0.2322840541601181, + "logits/chosen": 0.4331367015838623, + "logits/rejected": 0.7943879961967468, + "logps/chosen": -0.8845175504684448, + "logps/rejected": -6.956637859344482, + "loss": 3.3336, + "nll_loss": 3.3103556632995605, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08845175057649612, + "rewards/margins": 0.6072120666503906, + "rewards/rejected": -0.6956638097763062, + "step": 1937 + }, + { + "epoch": 1.2055987558320373, + "grad_norm": 0.4572494924068451, + "learning_rate": 1.55e-06, + "log_odds_chosen": 8.461591720581055, + "log_odds_ratio": -0.10262424498796463, + "logits/chosen": 0.20304995775222778, + "logits/rejected": 0.4690617322921753, + "logps/chosen": -0.6836486458778381, + "logps/rejected": -8.393146514892578, + "loss": 3.1867, + "nll_loss": 3.176400661468506, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06836486607789993, + "rewards/margins": 0.7709497809410095, + "rewards/rejected": -0.8393145799636841, + "step": 1938 + }, + { + "epoch": 1.206220839813375, + "grad_norm": 0.641258180141449, + "learning_rate": 1.525e-06, + "log_odds_chosen": 13.561309814453125, + "log_odds_ratio": -0.053132813423871994, + "logits/chosen": 0.281563937664032, + "logits/rejected": 1.5124027729034424, + "logps/chosen": -0.9380850195884705, + "logps/rejected": -14.03787612915039, + "loss": 2.3357, + "nll_loss": 2.330416202545166, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09380850195884705, + "rewards/margins": 1.3099792003631592, + "rewards/rejected": -1.403787612915039, + "step": 1939 + }, + { + "epoch": 1.2068429237947123, + "grad_norm": 0.6043529510498047, + "learning_rate": 1.5e-06, + "log_odds_chosen": 11.235591888427734, + "log_odds_ratio": -0.08132461458444595, + "logits/chosen": 0.11666382849216461, + "logits/rejected": 1.035499095916748, + "logps/chosen": -1.076815128326416, + "logps/rejected": -11.899983406066895, + "loss": 2.4499, + "nll_loss": 2.441749334335327, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1076815202832222, + "rewards/margins": 1.0823167562484741, + "rewards/rejected": -1.1899982690811157, + "step": 1940 + }, + { + "epoch": 1.2074650077760498, + "grad_norm": 0.5861474871635437, + "learning_rate": 1.475e-06, + "log_odds_chosen": 12.166252136230469, + "log_odds_ratio": -0.058929406106472015, + "logits/chosen": 0.20770826935768127, + "logits/rejected": 0.8990826606750488, + "logps/chosen": -1.0073878765106201, + "logps/rejected": -12.550826072692871, + "loss": 2.6517, + "nll_loss": 2.645765781402588, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10073879361152649, + "rewards/margins": 1.154343843460083, + "rewards/rejected": -1.255082607269287, + "step": 1941 + }, + { + "epoch": 1.2080870917573872, + "grad_norm": 0.4842999577522278, + "learning_rate": 1.45e-06, + "log_odds_chosen": 12.450490951538086, + "log_odds_ratio": -0.043036721646785736, + "logits/chosen": 0.3495686650276184, + "logits/rejected": 1.1103694438934326, + "logps/chosen": -0.8654146194458008, + "logps/rejected": -12.739120483398438, + "loss": 3.5126, + "nll_loss": 3.5082592964172363, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08654146641492844, + "rewards/margins": 1.1873705387115479, + "rewards/rejected": -1.2739120721817017, + "step": 1942 + }, + { + "epoch": 1.2087091757387247, + "grad_norm": 0.5130143165588379, + "learning_rate": 1.4250000000000001e-06, + "log_odds_chosen": 11.402581214904785, + "log_odds_ratio": -0.1462099850177765, + "logits/chosen": 0.26440829038619995, + "logits/rejected": 1.2154403924942017, + "logps/chosen": -1.113938331604004, + "logps/rejected": -12.0927095413208, + "loss": 3.0039, + "nll_loss": 2.989281177520752, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11139383167028427, + "rewards/margins": 1.0978771448135376, + "rewards/rejected": -1.20927095413208, + "step": 1943 + }, + { + "epoch": 1.2093312597200623, + "grad_norm": 0.5196940302848816, + "learning_rate": 1.4000000000000001e-06, + "log_odds_chosen": 13.953886985778809, + "log_odds_ratio": -0.07984312623739243, + "logits/chosen": 0.2587972581386566, + "logits/rejected": 1.134250283241272, + "logps/chosen": -1.1607818603515625, + "logps/rejected": -14.70931625366211, + "loss": 2.9967, + "nll_loss": 2.988737106323242, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1160781979560852, + "rewards/margins": 1.3548533916473389, + "rewards/rejected": -1.4709315299987793, + "step": 1944 + }, + { + "epoch": 1.2099533437013996, + "grad_norm": 0.43494367599487305, + "learning_rate": 1.3750000000000002e-06, + "log_odds_chosen": 11.031208038330078, + "log_odds_ratio": -0.24586418271064758, + "logits/chosen": 0.36235183477401733, + "logits/rejected": 1.051892638206482, + "logps/chosen": -1.0590839385986328, + "logps/rejected": -11.676322937011719, + "loss": 3.7231, + "nll_loss": 3.6984784603118896, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10590839385986328, + "rewards/margins": 1.0617239475250244, + "rewards/rejected": -1.1676323413848877, + "step": 1945 + }, + { + "epoch": 1.2105754276827372, + "grad_norm": 0.6046903133392334, + "learning_rate": 1.35e-06, + "log_odds_chosen": 11.70355224609375, + "log_odds_ratio": -0.027556756511330605, + "logits/chosen": 0.15471243858337402, + "logits/rejected": 0.9451410174369812, + "logps/chosen": -1.006124496459961, + "logps/rejected": -12.17332649230957, + "loss": 2.3758, + "nll_loss": 2.3730697631835938, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10061244666576385, + "rewards/margins": 1.116720199584961, + "rewards/rejected": -1.2173326015472412, + "step": 1946 + }, + { + "epoch": 1.2111975116640747, + "grad_norm": 0.7742229104042053, + "learning_rate": 1.325e-06, + "log_odds_chosen": 13.105299949645996, + "log_odds_ratio": -0.09716056287288666, + "logits/chosen": 0.22029726207256317, + "logits/rejected": 0.7522600889205933, + "logps/chosen": -0.7733492255210876, + "logps/rejected": -13.077220916748047, + "loss": 2.6818, + "nll_loss": 2.67205810546875, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.077334925532341, + "rewards/margins": 1.2303872108459473, + "rewards/rejected": -1.3077220916748047, + "step": 1947 + }, + { + "epoch": 1.211819595645412, + "grad_norm": 0.5355544090270996, + "learning_rate": 1.3e-06, + "log_odds_chosen": 9.141685485839844, + "log_odds_ratio": -0.0110908392816782, + "logits/chosen": 0.20812612771987915, + "logits/rejected": 0.826041042804718, + "logps/chosen": -1.2029227018356323, + "logps/rejected": -9.956043243408203, + "loss": 3.0694, + "nll_loss": 3.0682947635650635, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12029227614402771, + "rewards/margins": 0.8753120303153992, + "rewards/rejected": -0.9956042766571045, + "step": 1948 + }, + { + "epoch": 1.2124416796267496, + "grad_norm": 0.5379774570465088, + "learning_rate": 1.275e-06, + "log_odds_chosen": 10.0809965133667, + "log_odds_ratio": -0.22488044202327728, + "logits/chosen": 0.3199789524078369, + "logits/rejected": 1.2801084518432617, + "logps/chosen": -1.104744791984558, + "logps/rejected": -10.602627754211426, + "loss": 3.0859, + "nll_loss": 3.06339168548584, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11047448217868805, + "rewards/margins": 0.9497882723808289, + "rewards/rejected": -1.0602627992630005, + "step": 1949 + }, + { + "epoch": 1.213063763608087, + "grad_norm": 0.47320687770843506, + "learning_rate": 1.25e-06, + "log_odds_chosen": 10.314563751220703, + "log_odds_ratio": -0.11364743113517761, + "logits/chosen": 0.17472714185714722, + "logits/rejected": 0.5548080801963806, + "logps/chosen": -1.1122039556503296, + "logps/rejected": -10.97630500793457, + "loss": 2.9175, + "nll_loss": 2.906121253967285, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11122038960456848, + "rewards/margins": 0.9864101409912109, + "rewards/rejected": -1.097630500793457, + "step": 1950 + }, + { + "epoch": 1.2136858475894245, + "grad_norm": 0.6643481850624084, + "learning_rate": 1.2250000000000001e-06, + "log_odds_chosen": 7.230688571929932, + "log_odds_ratio": -0.27975186705589294, + "logits/chosen": 0.20719948410987854, + "logits/rejected": 0.5046452283859253, + "logps/chosen": -0.9272645711898804, + "logps/rejected": -7.5637593269348145, + "loss": 3.4042, + "nll_loss": 3.3762478828430176, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0927264541387558, + "rewards/margins": 0.6636494994163513, + "rewards/rejected": -0.7563759088516235, + "step": 1951 + }, + { + "epoch": 1.2143079315707621, + "grad_norm": 0.47893887758255005, + "learning_rate": 1.2000000000000002e-06, + "log_odds_chosen": 10.240165710449219, + "log_odds_ratio": -0.08771882206201553, + "logits/chosen": 0.30709195137023926, + "logits/rejected": 0.836345911026001, + "logps/chosen": -0.8510648608207703, + "logps/rejected": -10.372858047485352, + "loss": 3.2679, + "nll_loss": 3.259153366088867, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0851064920425415, + "rewards/margins": 0.9521793127059937, + "rewards/rejected": -1.0372858047485352, + "step": 1952 + }, + { + "epoch": 1.2149300155520995, + "grad_norm": 0.4384036362171173, + "learning_rate": 1.175e-06, + "log_odds_chosen": 6.116580009460449, + "log_odds_ratio": -0.0858708992600441, + "logits/chosen": 0.20664840936660767, + "logits/rejected": 0.5166964530944824, + "logps/chosen": -0.8784169554710388, + "logps/rejected": -6.095085144042969, + "loss": 3.3228, + "nll_loss": 3.314185857772827, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08784169703722, + "rewards/margins": 0.5216668844223022, + "rewards/rejected": -0.6095085740089417, + "step": 1953 + }, + { + "epoch": 1.215552099533437, + "grad_norm": 0.47888606786727905, + "learning_rate": 1.15e-06, + "log_odds_chosen": 14.013835906982422, + "log_odds_ratio": -0.0009424424497410655, + "logits/chosen": 0.30866438150405884, + "logits/rejected": 1.1062043905258179, + "logps/chosen": -0.9635977745056152, + "logps/rejected": -14.372419357299805, + "loss": 3.1659, + "nll_loss": 3.1658222675323486, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09635977447032928, + "rewards/margins": 1.3408820629119873, + "rewards/rejected": -1.437242031097412, + "step": 1954 + }, + { + "epoch": 1.2161741835147746, + "grad_norm": 0.6387030482292175, + "learning_rate": 1.125e-06, + "log_odds_chosen": 11.080677032470703, + "log_odds_ratio": -0.12639693915843964, + "logits/chosen": 0.3598368763923645, + "logits/rejected": 1.24238121509552, + "logps/chosen": -1.0083693265914917, + "logps/rejected": -11.685609817504883, + "loss": 2.8974, + "nll_loss": 2.8847668170928955, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10083693265914917, + "rewards/margins": 1.0677239894866943, + "rewards/rejected": -1.1685608625411987, + "step": 1955 + }, + { + "epoch": 1.216796267496112, + "grad_norm": 0.46826833486557007, + "learning_rate": 1.1e-06, + "log_odds_chosen": 10.808345794677734, + "log_odds_ratio": -0.00998397171497345, + "logits/chosen": 0.2149408757686615, + "logits/rejected": 0.9323172569274902, + "logps/chosen": -1.1690503358840942, + "logps/rejected": -11.58592414855957, + "loss": 3.2139, + "nll_loss": 3.212857723236084, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11690502613782883, + "rewards/margins": 1.0416873693466187, + "rewards/rejected": -1.1585923433303833, + "step": 1956 + }, + { + "epoch": 1.2174183514774495, + "grad_norm": 0.5350275635719299, + "learning_rate": 1.0749999999999999e-06, + "log_odds_chosen": 7.807048797607422, + "log_odds_ratio": -0.2332705557346344, + "logits/chosen": 0.05367419868707657, + "logits/rejected": 0.641290545463562, + "logps/chosen": -1.2197027206420898, + "logps/rejected": -8.814544677734375, + "loss": 2.7277, + "nll_loss": 2.7044012546539307, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1219702661037445, + "rewards/margins": 0.7594842910766602, + "rewards/rejected": -0.8814545273780823, + "step": 1957 + }, + { + "epoch": 1.218040435458787, + "grad_norm": 0.4396865665912628, + "learning_rate": 1.0500000000000001e-06, + "log_odds_chosen": 7.691943645477295, + "log_odds_ratio": -0.11706163734197617, + "logits/chosen": 0.12572860717773438, + "logits/rejected": 0.6232350468635559, + "logps/chosen": -1.0413182973861694, + "logps/rejected": -8.103107452392578, + "loss": 3.0067, + "nll_loss": 2.994955539703369, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10413183271884918, + "rewards/margins": 0.7061789035797119, + "rewards/rejected": -0.8103107213973999, + "step": 1958 + }, + { + "epoch": 1.2186625194401244, + "grad_norm": 0.6320565342903137, + "learning_rate": 1.0250000000000001e-06, + "log_odds_chosen": 10.37856388092041, + "log_odds_ratio": -0.1980902999639511, + "logits/chosen": 0.23183494806289673, + "logits/rejected": 0.9997614622116089, + "logps/chosen": -1.0834324359893799, + "logps/rejected": -11.047297477722168, + "loss": 2.3947, + "nll_loss": 2.3749301433563232, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10834324359893799, + "rewards/margins": 0.9963865280151367, + "rewards/rejected": -1.1047298908233643, + "step": 1959 + }, + { + "epoch": 1.219284603421462, + "grad_norm": 0.6440889239311218, + "learning_rate": 1.0000000000000002e-06, + "log_odds_chosen": 9.363749504089355, + "log_odds_ratio": -0.009031450375914574, + "logits/chosen": 0.2922241687774658, + "logits/rejected": 1.0417684316635132, + "logps/chosen": -1.280368447303772, + "logps/rejected": -10.292003631591797, + "loss": 2.8072, + "nll_loss": 2.806274652481079, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12803685665130615, + "rewards/margins": 0.9011635184288025, + "rewards/rejected": -1.0292003154754639, + "step": 1960 + }, + { + "epoch": 1.2199066874027993, + "grad_norm": 0.5854726433753967, + "learning_rate": 9.75e-07, + "log_odds_chosen": 7.83726692199707, + "log_odds_ratio": -0.14151814579963684, + "logits/chosen": 0.02728712186217308, + "logits/rejected": 0.7834301590919495, + "logps/chosen": -1.2481005191802979, + "logps/rejected": -8.801592826843262, + "loss": 2.4193, + "nll_loss": 2.405181646347046, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12481004744768143, + "rewards/margins": 0.7553492188453674, + "rewards/rejected": -0.880159318447113, + "step": 1961 + }, + { + "epoch": 1.2205287713841368, + "grad_norm": 0.6133548021316528, + "learning_rate": 9.5e-07, + "log_odds_chosen": 17.47218894958496, + "log_odds_ratio": -0.0755654126405716, + "logits/chosen": 0.4042644202709198, + "logits/rejected": 1.29765784740448, + "logps/chosen": -0.8723544478416443, + "logps/rejected": -17.82417869567871, + "loss": 2.9475, + "nll_loss": 2.939979076385498, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08723544329404831, + "rewards/margins": 1.6951824426651, + "rewards/rejected": -1.782417893409729, + "step": 1962 + }, + { + "epoch": 1.2211508553654744, + "grad_norm": 0.6662535667419434, + "learning_rate": 9.25e-07, + "log_odds_chosen": 14.000879287719727, + "log_odds_ratio": -0.0005081897252239287, + "logits/chosen": 0.26021426916122437, + "logits/rejected": 1.3705577850341797, + "logps/chosen": -1.1540570259094238, + "logps/rejected": -14.73798942565918, + "loss": 2.3304, + "nll_loss": 2.3303232192993164, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11540570110082626, + "rewards/margins": 1.3583931922912598, + "rewards/rejected": -1.4737989902496338, + "step": 1963 + }, + { + "epoch": 1.2217729393468117, + "grad_norm": 0.5186629295349121, + "learning_rate": 9e-07, + "log_odds_chosen": 14.97154426574707, + "log_odds_ratio": -0.0003290826571173966, + "logits/chosen": 0.1332269012928009, + "logits/rejected": 0.7068179845809937, + "logps/chosen": -1.1427979469299316, + "logps/rejected": -15.563932418823242, + "loss": 3.0262, + "nll_loss": 3.0261826515197754, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11427979171276093, + "rewards/margins": 1.4421135187149048, + "rewards/rejected": -1.5563932657241821, + "step": 1964 + }, + { + "epoch": 1.2223950233281493, + "grad_norm": 0.644451916217804, + "learning_rate": 8.750000000000001e-07, + "log_odds_chosen": 8.28154182434082, + "log_odds_ratio": -0.18015088140964508, + "logits/chosen": 0.14118118584156036, + "logits/rejected": 0.36667898297309875, + "logps/chosen": -1.0873171091079712, + "logps/rejected": -8.973154067993164, + "loss": 2.7051, + "nll_loss": 2.6870474815368652, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1087317168712616, + "rewards/margins": 0.7885836958885193, + "rewards/rejected": -0.8973153829574585, + "step": 1965 + }, + { + "epoch": 1.2230171073094869, + "grad_norm": 0.6322017312049866, + "learning_rate": 8.500000000000001e-07, + "log_odds_chosen": 8.490213394165039, + "log_odds_ratio": -0.034700050950050354, + "logits/chosen": 0.14440450072288513, + "logits/rejected": 0.6193152666091919, + "logps/chosen": -1.045159101486206, + "logps/rejected": -9.040973663330078, + "loss": 2.6674, + "nll_loss": 2.66395902633667, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1045159101486206, + "rewards/margins": 0.7995815277099609, + "rewards/rejected": -0.9040974378585815, + "step": 1966 + }, + { + "epoch": 1.2236391912908242, + "grad_norm": 0.6075518727302551, + "learning_rate": 8.25e-07, + "log_odds_chosen": 14.548714637756348, + "log_odds_ratio": -0.07951350510120392, + "logits/chosen": 0.22518101334571838, + "logits/rejected": 1.286879062652588, + "logps/chosen": -1.033128261566162, + "logps/rejected": -15.159509658813477, + "loss": 2.7825, + "nll_loss": 2.7745676040649414, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10331282019615173, + "rewards/margins": 1.4126381874084473, + "rewards/rejected": -1.5159509181976318, + "step": 1967 + }, + { + "epoch": 1.2242612752721618, + "grad_norm": 0.45069652795791626, + "learning_rate": 8.000000000000001e-07, + "log_odds_chosen": 15.424966812133789, + "log_odds_ratio": -2.145773578376975e-06, + "logits/chosen": 0.35158199071884155, + "logits/rejected": 1.5012739896774292, + "logps/chosen": -1.0200140476226807, + "logps/rejected": -15.91152572631836, + "loss": 3.0081, + "nll_loss": 3.008110284805298, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10200141370296478, + "rewards/margins": 1.4891510009765625, + "rewards/rejected": -1.591152548789978, + "step": 1968 + }, + { + "epoch": 1.2248833592534991, + "grad_norm": 0.47687819600105286, + "learning_rate": 7.75e-07, + "log_odds_chosen": 7.983428001403809, + "log_odds_ratio": -0.028099507093429565, + "logits/chosen": 0.18631598353385925, + "logits/rejected": 0.5772290825843811, + "logps/chosen": -0.7105157971382141, + "logps/rejected": -7.801077365875244, + "loss": 3.1483, + "nll_loss": 3.145503520965576, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07105158269405365, + "rewards/margins": 0.7090562582015991, + "rewards/rejected": -0.7801077961921692, + "step": 1969 + }, + { + "epoch": 1.2255054432348367, + "grad_norm": 0.8003374338150024, + "learning_rate": 7.5e-07, + "log_odds_chosen": 12.410947799682617, + "log_odds_ratio": -0.00033352465834468603, + "logits/chosen": 0.07465855777263641, + "logits/rejected": 0.8042261004447937, + "logps/chosen": -1.0633723735809326, + "logps/rejected": -12.912806510925293, + "loss": 2.5147, + "nll_loss": 2.5146756172180176, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10633723437786102, + "rewards/margins": 1.1849435567855835, + "rewards/rejected": -1.2912808656692505, + "step": 1970 + }, + { + "epoch": 1.2261275272161742, + "grad_norm": 2.2342381477355957, + "learning_rate": 7.25e-07, + "log_odds_chosen": 10.86426067352295, + "log_odds_ratio": -0.13659894466400146, + "logits/chosen": 0.18673838675022125, + "logits/rejected": 0.6166978478431702, + "logps/chosen": -1.0592784881591797, + "logps/rejected": -11.352396011352539, + "loss": 3.137, + "nll_loss": 3.1233880519866943, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10592785477638245, + "rewards/margins": 1.0293117761611938, + "rewards/rejected": -1.135239601135254, + "step": 1971 + }, + { + "epoch": 1.2267496111975116, + "grad_norm": 0.6535879373550415, + "learning_rate": 7.000000000000001e-07, + "log_odds_chosen": 9.130402565002441, + "log_odds_ratio": -0.1539786159992218, + "logits/chosen": 0.32859697937965393, + "logits/rejected": 0.9583666920661926, + "logps/chosen": -0.966360330581665, + "logps/rejected": -9.597227096557617, + "loss": 2.797, + "nll_loss": 2.7815654277801514, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09663603454828262, + "rewards/margins": 0.8630867004394531, + "rewards/rejected": -0.9597226977348328, + "step": 1972 + }, + { + "epoch": 1.2273716951788491, + "grad_norm": 0.5714898109436035, + "learning_rate": 6.75e-07, + "log_odds_chosen": 13.186323165893555, + "log_odds_ratio": -0.04164155200123787, + "logits/chosen": 0.3834404945373535, + "logits/rejected": 1.092491626739502, + "logps/chosen": -1.2385993003845215, + "logps/rejected": -13.750805854797363, + "loss": 3.6435, + "nll_loss": 3.6393702030181885, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.12385992705821991, + "rewards/margins": 1.251220703125, + "rewards/rejected": -1.3750805854797363, + "step": 1973 + }, + { + "epoch": 1.2279937791601867, + "grad_norm": 0.48778393864631653, + "learning_rate": 6.5e-07, + "log_odds_chosen": 12.12580394744873, + "log_odds_ratio": -0.11388830840587616, + "logits/chosen": 0.17847785353660583, + "logits/rejected": 0.7780681252479553, + "logps/chosen": -0.8805215358734131, + "logps/rejected": -12.4000244140625, + "loss": 3.0103, + "nll_loss": 2.9988701343536377, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08805215358734131, + "rewards/margins": 1.1519503593444824, + "rewards/rejected": -1.2400025129318237, + "step": 1974 + }, + { + "epoch": 1.228615863141524, + "grad_norm": 0.649593710899353, + "learning_rate": 6.25e-07, + "log_odds_chosen": 6.819096565246582, + "log_odds_ratio": -0.25234541296958923, + "logits/chosen": 0.22128456830978394, + "logits/rejected": 0.7910210490226746, + "logps/chosen": -1.0413517951965332, + "logps/rejected": -7.474663257598877, + "loss": 2.6732, + "nll_loss": 2.647918939590454, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1041351780295372, + "rewards/margins": 0.6433311700820923, + "rewards/rejected": -0.7474663853645325, + "step": 1975 + }, + { + "epoch": 1.2292379471228616, + "grad_norm": 0.5235407948493958, + "learning_rate": 6.000000000000001e-07, + "log_odds_chosen": 11.304561614990234, + "log_odds_ratio": -0.0011457885848358274, + "logits/chosen": 0.27476418018341064, + "logits/rejected": 1.1056790351867676, + "logps/chosen": -0.9318090677261353, + "logps/rejected": -11.397740364074707, + "loss": 2.9588, + "nll_loss": 2.9587111473083496, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09318091720342636, + "rewards/margins": 1.046593189239502, + "rewards/rejected": -1.1397740840911865, + "step": 1976 + }, + { + "epoch": 1.2298600311041992, + "grad_norm": 0.6689901947975159, + "learning_rate": 5.75e-07, + "log_odds_chosen": 6.795727729797363, + "log_odds_ratio": -0.30927932262420654, + "logits/chosen": 0.027122098952531815, + "logits/rejected": 0.27755478024482727, + "logps/chosen": -1.076484203338623, + "logps/rejected": -7.535107612609863, + "loss": 2.4418, + "nll_loss": 2.410856246948242, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10764843225479126, + "rewards/margins": 0.645862340927124, + "rewards/rejected": -0.7535107731819153, + "step": 1977 + }, + { + "epoch": 1.2304821150855365, + "grad_norm": 0.6304493546485901, + "learning_rate": 5.5e-07, + "log_odds_chosen": 9.401277542114258, + "log_odds_ratio": -0.18153725564479828, + "logits/chosen": 0.11142933368682861, + "logits/rejected": 0.42696264386177063, + "logps/chosen": -0.8474681377410889, + "logps/rejected": -9.611903190612793, + "loss": 2.4952, + "nll_loss": 2.4770264625549316, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0847468227148056, + "rewards/margins": 0.8764435648918152, + "rewards/rejected": -0.961190402507782, + "step": 1978 + }, + { + "epoch": 1.231104199066874, + "grad_norm": 0.534700870513916, + "learning_rate": 5.250000000000001e-07, + "log_odds_chosen": 12.72781753540039, + "log_odds_ratio": -0.11465675383806229, + "logits/chosen": 0.3088584244251251, + "logits/rejected": 1.142822504043579, + "logps/chosen": -0.8340325355529785, + "logps/rejected": -12.834344863891602, + "loss": 2.9313, + "nll_loss": 2.9198219776153564, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08340325951576233, + "rewards/margins": 1.2000312805175781, + "rewards/rejected": -1.2834346294403076, + "step": 1979 + }, + { + "epoch": 1.2317262830482114, + "grad_norm": 0.6244094967842102, + "learning_rate": 5.000000000000001e-07, + "log_odds_chosen": 13.20551586151123, + "log_odds_ratio": -0.03865737095475197, + "logits/chosen": 0.2893320918083191, + "logits/rejected": 1.3345921039581299, + "logps/chosen": -1.0497782230377197, + "logps/rejected": -13.795721054077148, + "loss": 2.7889, + "nll_loss": 2.7850396633148193, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10497783124446869, + "rewards/margins": 1.2745941877365112, + "rewards/rejected": -1.3795721530914307, + "step": 1980 + }, + { + "epoch": 1.232348367029549, + "grad_norm": 0.532413899898529, + "learning_rate": 4.75e-07, + "log_odds_chosen": 10.777617454528809, + "log_odds_ratio": -0.010713006369769573, + "logits/chosen": 0.39420104026794434, + "logits/rejected": 1.2114235162734985, + "logps/chosen": -0.8548382520675659, + "logps/rejected": -10.988439559936523, + "loss": 3.2467, + "nll_loss": 3.2456629276275635, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08548382669687271, + "rewards/margins": 1.0133602619171143, + "rewards/rejected": -1.098844051361084, + "step": 1981 + }, + { + "epoch": 1.2329704510108865, + "grad_norm": 0.5516924262046814, + "learning_rate": 4.5e-07, + "log_odds_chosen": 12.285039901733398, + "log_odds_ratio": -0.08736379444599152, + "logits/chosen": 0.26371684670448303, + "logits/rejected": 0.9414895176887512, + "logps/chosen": -0.8180931806564331, + "logps/rejected": -12.407268524169922, + "loss": 2.9298, + "nll_loss": 2.921090602874756, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08180931955575943, + "rewards/margins": 1.1589176654815674, + "rewards/rejected": -1.2407269477844238, + "step": 1982 + }, + { + "epoch": 1.2335925349922239, + "grad_norm": 0.493353009223938, + "learning_rate": 4.2500000000000006e-07, + "log_odds_chosen": 13.17050552368164, + "log_odds_ratio": -0.005692049395292997, + "logits/chosen": 0.35018348693847656, + "logits/rejected": 1.103072166442871, + "logps/chosen": -1.3006165027618408, + "logps/rejected": -13.748285293579102, + "loss": 3.395, + "nll_loss": 3.394420623779297, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13006165623664856, + "rewards/margins": 1.2447669506072998, + "rewards/rejected": -1.374828577041626, + "step": 1983 + }, + { + "epoch": 1.2342146189735614, + "grad_norm": 0.4653708040714264, + "learning_rate": 4.0000000000000003e-07, + "log_odds_chosen": 8.631146430969238, + "log_odds_ratio": -0.19020092487335205, + "logits/chosen": 0.43207770586013794, + "logits/rejected": 1.0626914501190186, + "logps/chosen": -1.5182112455368042, + "logps/rejected": -9.975319862365723, + "loss": 3.8365, + "nll_loss": 3.8174636363983154, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15182113647460938, + "rewards/margins": 0.845710813999176, + "rewards/rejected": -0.9975318908691406, + "step": 1984 + }, + { + "epoch": 1.234836702954899, + "grad_norm": 0.4419538378715515, + "learning_rate": 3.75e-07, + "log_odds_chosen": 7.5447492599487305, + "log_odds_ratio": -0.01544712483882904, + "logits/chosen": 0.24856893718242645, + "logits/rejected": 0.7290024757385254, + "logps/chosen": -1.1332921981811523, + "logps/rejected": -8.262965202331543, + "loss": 3.1574, + "nll_loss": 3.1558544635772705, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.113329216837883, + "rewards/margins": 0.7129673361778259, + "rewards/rejected": -0.8262965083122253, + "step": 1985 + }, + { + "epoch": 1.2354587869362363, + "grad_norm": 0.4535614550113678, + "learning_rate": 3.5000000000000004e-07, + "log_odds_chosen": 12.693193435668945, + "log_odds_ratio": -0.017142467200756073, + "logits/chosen": 0.2979247272014618, + "logits/rejected": 1.086916208267212, + "logps/chosen": -0.9978163838386536, + "logps/rejected": -13.19540023803711, + "loss": 2.8112, + "nll_loss": 2.809511661529541, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09978163242340088, + "rewards/margins": 1.21975839138031, + "rewards/rejected": -1.319540023803711, + "step": 1986 + }, + { + "epoch": 1.236080870917574, + "grad_norm": 0.9630240797996521, + "learning_rate": 3.25e-07, + "log_odds_chosen": 4.148228645324707, + "log_odds_ratio": -0.3517554998397827, + "logits/chosen": 0.10793019831180573, + "logits/rejected": 0.37933433055877686, + "logps/chosen": -1.1543450355529785, + "logps/rejected": -5.036352157592773, + "loss": 2.5266, + "nll_loss": 2.4914731979370117, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11543451249599457, + "rewards/margins": 0.3882007300853729, + "rewards/rejected": -0.5036352276802063, + "step": 1987 + }, + { + "epoch": 1.2367029548989112, + "grad_norm": 0.5966618657112122, + "learning_rate": 3.0000000000000004e-07, + "log_odds_chosen": 12.570096015930176, + "log_odds_ratio": -0.004863132257014513, + "logits/chosen": 0.24723371863365173, + "logits/rejected": 1.2845591306686401, + "logps/chosen": -1.2328592538833618, + "logps/rejected": -13.443431854248047, + "loss": 2.7224, + "nll_loss": 2.7219624519348145, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1232859343290329, + "rewards/margins": 1.2210571765899658, + "rewards/rejected": -1.3443431854248047, + "step": 1988 + }, + { + "epoch": 1.2373250388802488, + "grad_norm": 0.6358088850975037, + "learning_rate": 2.75e-07, + "log_odds_chosen": 6.43398380279541, + "log_odds_ratio": -0.17213129997253418, + "logits/chosen": 0.13488197326660156, + "logits/rejected": 0.573233962059021, + "logps/chosen": -0.9811330437660217, + "logps/rejected": -6.960114002227783, + "loss": 2.6887, + "nll_loss": 2.6715219020843506, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0981132984161377, + "rewards/margins": 0.5978981256484985, + "rewards/rejected": -0.6960113644599915, + "step": 1989 + }, + { + "epoch": 1.2379471228615864, + "grad_norm": 0.6376218199729919, + "learning_rate": 2.5000000000000004e-07, + "log_odds_chosen": 8.221298217773438, + "log_odds_ratio": -0.13431409001350403, + "logits/chosen": 0.0446498841047287, + "logits/rejected": 0.5792117118835449, + "logps/chosen": -1.0903517007827759, + "logps/rejected": -8.90846061706543, + "loss": 2.1503, + "nll_loss": 2.136909008026123, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10903516411781311, + "rewards/margins": 0.7818108797073364, + "rewards/rejected": -0.8908460736274719, + "step": 1990 + }, + { + "epoch": 1.2385692068429237, + "grad_norm": 0.46342816948890686, + "learning_rate": 2.25e-07, + "log_odds_chosen": 9.396651268005371, + "log_odds_ratio": -0.19147804379463196, + "logits/chosen": 0.2942267060279846, + "logits/rejected": 0.8135102391242981, + "logps/chosen": -0.9783865213394165, + "logps/rejected": -9.955822944641113, + "loss": 3.0935, + "nll_loss": 3.0743513107299805, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09783865511417389, + "rewards/margins": 0.8977437019348145, + "rewards/rejected": -0.9955822825431824, + "step": 1991 + }, + { + "epoch": 1.2391912908242613, + "grad_norm": 0.6562243700027466, + "learning_rate": 2.0000000000000002e-07, + "log_odds_chosen": 7.678519248962402, + "log_odds_ratio": -0.046273984014987946, + "logits/chosen": 0.2911015748977661, + "logits/rejected": 0.81610107421875, + "logps/chosen": -1.0920604467391968, + "logps/rejected": -8.079513549804688, + "loss": 3.0501, + "nll_loss": 3.0454328060150146, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10920606553554535, + "rewards/margins": 0.6987452507019043, + "rewards/rejected": -0.8079513311386108, + "step": 1992 + }, + { + "epoch": 1.2398133748055988, + "grad_norm": 0.6405279636383057, + "learning_rate": 1.7500000000000002e-07, + "log_odds_chosen": 8.547779083251953, + "log_odds_ratio": -0.14488686621189117, + "logits/chosen": 0.279305100440979, + "logits/rejected": 0.7682068347930908, + "logps/chosen": -1.0256428718566895, + "logps/rejected": -9.043883323669434, + "loss": 3.1503, + "nll_loss": 3.135767936706543, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10256429016590118, + "rewards/margins": 0.8018239736557007, + "rewards/rejected": -0.9043882489204407, + "step": 1993 + }, + { + "epoch": 1.2404354587869362, + "grad_norm": 0.6466782689094543, + "learning_rate": 1.5000000000000002e-07, + "log_odds_chosen": 7.694208145141602, + "log_odds_ratio": -0.07817018032073975, + "logits/chosen": 0.18080022931098938, + "logits/rejected": 0.88742995262146, + "logps/chosen": -1.003135085105896, + "logps/rejected": -8.164618492126465, + "loss": 2.8495, + "nll_loss": 2.8416547775268555, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10031351447105408, + "rewards/margins": 0.716148316860199, + "rewards/rejected": -0.8164618015289307, + "step": 1994 + }, + { + "epoch": 1.2410575427682737, + "grad_norm": 0.5974647402763367, + "learning_rate": 1.2500000000000002e-07, + "log_odds_chosen": 12.203994750976562, + "log_odds_ratio": -0.028767168521881104, + "logits/chosen": 0.2811827063560486, + "logits/rejected": 0.8901752233505249, + "logps/chosen": -0.8630846738815308, + "logps/rejected": -12.416587829589844, + "loss": 3.0164, + "nll_loss": 3.0134806632995605, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08630846440792084, + "rewards/margins": 1.1553503274917603, + "rewards/rejected": -1.2416586875915527, + "step": 1995 + }, + { + "epoch": 1.2416796267496113, + "grad_norm": 0.5271542072296143, + "learning_rate": 1.0000000000000001e-07, + "log_odds_chosen": 15.869498252868652, + "log_odds_ratio": -0.0005326489335857332, + "logits/chosen": 0.31395959854125977, + "logits/rejected": 1.5588934421539307, + "logps/chosen": -0.9302768707275391, + "logps/rejected": -16.15066146850586, + "loss": 2.7436, + "nll_loss": 2.7435102462768555, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09302769601345062, + "rewards/margins": 1.522038459777832, + "rewards/rejected": -1.6150660514831543, + "step": 1996 + }, + { + "epoch": 1.2423017107309486, + "grad_norm": 2.5741851329803467, + "learning_rate": 7.500000000000001e-08, + "log_odds_chosen": 11.470212936401367, + "log_odds_ratio": -0.017528457567095757, + "logits/chosen": 0.3122308850288391, + "logits/rejected": 0.7689934968948364, + "logps/chosen": -1.033614993095398, + "logps/rejected": -11.780576705932617, + "loss": 3.5241, + "nll_loss": 3.5223326683044434, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10336150974035263, + "rewards/margins": 1.0746960639953613, + "rewards/rejected": -1.1780576705932617, + "step": 1997 + }, + { + "epoch": 1.2429237947122862, + "grad_norm": 0.49963220953941345, + "learning_rate": 5.0000000000000004e-08, + "log_odds_chosen": 13.224645614624023, + "log_odds_ratio": -3.3678723411867395e-05, + "logits/chosen": -0.03131323307752609, + "logits/rejected": 0.7888933420181274, + "logps/chosen": -0.924960732460022, + "logps/rejected": -13.508068084716797, + "loss": 2.3564, + "nll_loss": 2.3563878536224365, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09249607473611832, + "rewards/margins": 1.2583107948303223, + "rewards/rejected": -1.350806713104248, + "step": 1998 + }, + { + "epoch": 1.2435458786936238, + "grad_norm": 0.5515841245651245, + "learning_rate": 2.5000000000000002e-08, + "log_odds_chosen": 11.417040824890137, + "log_odds_ratio": -0.09382711350917816, + "logits/chosen": 0.32003253698349, + "logits/rejected": 0.9499678611755371, + "logps/chosen": -0.9657232761383057, + "logps/rejected": -11.697189331054688, + "loss": 3.106, + "nll_loss": 3.0966427326202393, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09657233208417892, + "rewards/margins": 1.0731468200683594, + "rewards/rejected": -1.1697189807891846, + "step": 1999 + }, + { + "epoch": 1.244167962674961, + "grad_norm": 0.48643431067466736, + "learning_rate": 0.0, + "log_odds_chosen": 6.229976654052734, + "log_odds_ratio": -0.18844850361347198, + "logits/chosen": 0.3436409831047058, + "logits/rejected": 0.5300962924957275, + "logps/chosen": -1.046073317527771, + "logps/rejected": -6.884661674499512, + "loss": 3.3064, + "nll_loss": 3.2875545024871826, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10460732877254486, + "rewards/margins": 0.583858847618103, + "rewards/rejected": -0.6884661912918091, + "step": 2000 + } + ], + "logging_steps": 1, + "max_steps": 2000, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}