{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.244167962674961,
  "eval_steps": 500,
  "global_step": 2000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0006220839813374805,
      "grad_norm": 19.98119354248047,
      "learning_rate": 4.9975e-05,
      "log_odds_chosen": -0.9538012742996216,
      "log_odds_ratio": -1.4484777450561523,
      "logits/chosen": -0.3500179052352905,
      "logits/rejected": -0.4888765215873718,
      "logps/chosen": -2.206789255142212,
      "logps/rejected": -1.3497388362884521,
      "loss": 7.9648,
      "nll_loss": 7.819947242736816,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.2206789255142212,
      "rewards/margins": -0.08570503443479538,
      "rewards/rejected": -0.13497388362884521,
      "step": 1
    },
    {
      "epoch": 0.001244167962674961,
      "grad_norm": 13.379537582397461,
      "learning_rate": 4.995e-05,
      "log_odds_chosen": -0.23575380444526672,
      "log_odds_ratio": -0.9271624088287354,
      "logits/chosen": -0.40814173221588135,
      "logits/rejected": -0.43905961513519287,
      "logps/chosen": -1.7428513765335083,
      "logps/rejected": -1.5162339210510254,
      "loss": 6.0609,
      "nll_loss": 5.96816873550415,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1742851585149765,
      "rewards/margins": -0.02266174554824829,
      "rewards/rejected": -0.15162339806556702,
      "step": 2
    },
    {
      "epoch": 0.0018662519440124418,
      "grad_norm": 18.385046005249023,
      "learning_rate": 4.992500000000001e-05,
      "log_odds_chosen": -1.152876853942871,
      "log_odds_ratio": -1.6186071634292603,
      "logits/chosen": -0.18745173513889313,
      "logits/rejected": -0.269758939743042,
      "logps/chosen": -2.3206677436828613,
      "logps/rejected": -1.3112003803253174,
      "loss": 6.6148,
      "nll_loss": 6.452923774719238,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.2320667803287506,
      "rewards/margins": -0.10094673931598663,
      "rewards/rejected": -0.13112004101276398,
      "step": 3
    },
    {
      "epoch": 0.002488335925349922,
      "grad_norm": 9.557136535644531,
      "learning_rate": 4.99e-05,
      "log_odds_chosen": -1.4854508638381958,
      "log_odds_ratio": -2.0142252445220947,
      "logits/chosen": -0.028350114822387695,
      "logits/rejected": -0.2540076971054077,
      "logps/chosen": -2.8356778621673584,
      "logps/rejected": -1.4353848695755005,
      "loss": 5.7077,
      "nll_loss": 5.506240367889404,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.28356778621673584,
      "rewards/margins": -0.14002928137779236,
      "rewards/rejected": -0.1435384899377823,
      "step": 4
    },
    {
      "epoch": 0.003110419906687403,
      "grad_norm": 1.9710413217544556,
      "learning_rate": 4.9875000000000006e-05,
      "log_odds_chosen": -0.064725860953331,
      "log_odds_ratio": -0.7509320974349976,
      "logits/chosen": -0.2338867336511612,
      "logits/rejected": -0.143581822514534,
      "logps/chosen": -1.6930336952209473,
      "logps/rejected": -1.6086996793746948,
      "loss": 3.6175,
      "nll_loss": 3.5423970222473145,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16930335760116577,
      "rewards/margins": -0.008433390408754349,
      "rewards/rejected": -0.16086997091770172,
      "step": 5
    },
    {
      "epoch": 0.0037325038880248835,
      "grad_norm": 4.176668643951416,
      "learning_rate": 4.9850000000000006e-05,
      "log_odds_chosen": -1.303108811378479,
      "log_odds_ratio": -1.8442931175231934,
      "logits/chosen": 0.178592711687088,
      "logits/rejected": -0.07974191009998322,
      "logps/chosen": -2.8217928409576416,
      "logps/rejected": -1.5813729763031006,
      "loss": 5.1449,
      "nll_loss": 4.96045446395874,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.2821792662143707,
      "rewards/margins": -0.12404197454452515,
      "rewards/rejected": -0.15813729166984558,
      "step": 6
    },
    {
      "epoch": 0.004354587869362364,
      "grad_norm": 3.5636956691741943,
      "learning_rate": 4.9825000000000005e-05,
      "log_odds_chosen": -1.0302621126174927,
      "log_odds_ratio": -1.533990502357483,
      "logits/chosen": 0.1397887021303177,
      "logits/rejected": -0.12103983759880066,
      "logps/chosen": -2.5798606872558594,
      "logps/rejected": -1.6471679210662842,
      "loss": 5.2595,
      "nll_loss": 5.1061482429504395,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2579860985279083,
      "rewards/margins": -0.09326928108930588,
      "rewards/rejected": -0.16471679508686066,
      "step": 7
    },
    {
      "epoch": 0.004976671850699844,
      "grad_norm": 1.424960732460022,
      "learning_rate": 4.9800000000000004e-05,
      "log_odds_chosen": -0.057843249291181564,
      "log_odds_ratio": -0.7315274477005005,
      "logits/chosen": -0.21581152081489563,
      "logits/rejected": -0.2254290133714676,
      "logps/chosen": -1.4299900531768799,
      "logps/rejected": -1.376258134841919,
      "loss": 3.5319,
      "nll_loss": 3.458702802658081,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14299900829792023,
      "rewards/margins": -0.00537317618727684,
      "rewards/rejected": -0.1376258283853531,
      "step": 8
    },
    {
      "epoch": 0.005598755832037325,
      "grad_norm": 2.506531000137329,
      "learning_rate": 4.9775000000000004e-05,
      "log_odds_chosen": -1.3545101881027222,
      "log_odds_ratio": -1.7200111150741577,
      "logits/chosen": 0.09172062575817108,
      "logits/rejected": -0.2259369194507599,
      "logps/chosen": -2.604728937149048,
      "logps/rejected": -1.421167016029358,
      "loss": 5.1963,
      "nll_loss": 5.0243120193481445,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.2604728937149048,
      "rewards/margins": -0.11835619807243347,
      "rewards/rejected": -0.1421166956424713,
      "step": 9
    },
    {
      "epoch": 0.006220839813374806,
      "grad_norm": 2.135577917098999,
      "learning_rate": 4.975e-05,
      "log_odds_chosen": -0.5466750264167786,
      "log_odds_ratio": -1.1493580341339111,
      "logits/chosen": -0.009926818311214447,
      "logits/rejected": -0.12821561098098755,
      "logps/chosen": -2.1568522453308105,
      "logps/rejected": -1.6440058946609497,
      "loss": 4.7799,
      "nll_loss": 4.664962291717529,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.21568521857261658,
      "rewards/margins": -0.05128464847803116,
      "rewards/rejected": -0.16440057754516602,
      "step": 10
    },
    {
      "epoch": 0.006842923794712286,
      "grad_norm": 2.0873756408691406,
      "learning_rate": 4.9725e-05,
      "log_odds_chosen": -0.22385576367378235,
      "log_odds_ratio": -0.8634535074234009,
      "logits/chosen": -0.07170040160417557,
      "logits/rejected": -0.28907641768455505,
      "logps/chosen": -1.4515858888626099,
      "logps/rejected": -1.2991254329681396,
      "loss": 4.4469,
      "nll_loss": 4.360559463500977,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.145158588886261,
      "rewards/margins": -0.015246042050421238,
      "rewards/rejected": -0.12991255521774292,
      "step": 11
    },
    {
      "epoch": 0.007465007776049767,
      "grad_norm": 1.0876375436782837,
      "learning_rate": 4.97e-05,
      "log_odds_chosen": -0.17661024630069733,
      "log_odds_ratio": -0.8128032088279724,
      "logits/chosen": -0.3114607334136963,
      "logits/rejected": -0.4201509356498718,
      "logps/chosen": -1.3225839138031006,
      "logps/rejected": -1.1948829889297485,
      "loss": 3.5573,
      "nll_loss": 3.476047992706299,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13225840032100677,
      "rewards/margins": -0.012770086526870728,
      "rewards/rejected": -0.11948829889297485,
      "step": 12
    },
    {
      "epoch": 0.008087091757387248,
      "grad_norm": 1.08712637424469,
      "learning_rate": 4.967500000000001e-05,
      "log_odds_chosen": -0.38383999466896057,
      "log_odds_ratio": -0.93343186378479,
      "logits/chosen": -0.17842403054237366,
      "logits/rejected": -0.4043556749820709,
      "logps/chosen": -1.5285923480987549,
      "logps/rejected": -1.2196223735809326,
      "loss": 4.0367,
      "nll_loss": 3.943343162536621,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.15285924077033997,
      "rewards/margins": -0.030896998941898346,
      "rewards/rejected": -0.12196224182844162,
      "step": 13
    },
    {
      "epoch": 0.008709175738724729,
      "grad_norm": 1.449515461921692,
      "learning_rate": 4.965e-05,
      "log_odds_chosen": -0.6121888160705566,
      "log_odds_ratio": -1.3804981708526611,
      "logits/chosen": -0.06745365262031555,
      "logits/rejected": -0.2206701636314392,
      "logps/chosen": -2.0546789169311523,
      "logps/rejected": -1.398402214050293,
      "loss": 4.3677,
      "nll_loss": 4.229663372039795,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20546789467334747,
      "rewards/margins": -0.06562765687704086,
      "rewards/rejected": -0.13984021544456482,
      "step": 14
    },
    {
      "epoch": 0.00933125972006221,
      "grad_norm": 0.8825786709785461,
      "learning_rate": 4.962500000000001e-05,
      "log_odds_chosen": -0.13495028018951416,
      "log_odds_ratio": -0.8009682893753052,
      "logits/chosen": -0.20614610612392426,
      "logits/rejected": -0.45154061913490295,
      "logps/chosen": -1.570542812347412,
      "logps/rejected": -1.476640224456787,
      "loss": 3.9433,
      "nll_loss": 3.863245964050293,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15705429017543793,
      "rewards/margins": -0.009390261955559254,
      "rewards/rejected": -0.14766402542591095,
      "step": 15
    },
    {
      "epoch": 0.009953343701399688,
      "grad_norm": 0.7910265922546387,
      "learning_rate": 4.96e-05,
      "log_odds_chosen": -0.36577627062797546,
      "log_odds_ratio": -0.9311540126800537,
      "logits/chosen": -0.22354616224765778,
      "logits/rejected": -0.29362282156944275,
      "logps/chosen": -1.65301513671875,
      "logps/rejected": -1.361577033996582,
      "loss": 3.606,
      "nll_loss": 3.5129306316375732,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.16530153155326843,
      "rewards/margins": -0.0291438065469265,
      "rewards/rejected": -0.13615772128105164,
      "step": 16
    },
    {
      "epoch": 0.010575427682737169,
      "grad_norm": 0.9194669127464294,
      "learning_rate": 4.9575000000000006e-05,
      "log_odds_chosen": -0.17109160125255585,
      "log_odds_ratio": -0.8144399523735046,
      "logits/chosen": -0.0705355554819107,
      "logits/rejected": -0.2670852541923523,
      "logps/chosen": -1.7514052391052246,
      "logps/rejected": -1.6155729293823242,
      "loss": 4.2639,
      "nll_loss": 4.182419776916504,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.17514052987098694,
      "rewards/margins": -0.013583235442638397,
      "rewards/rejected": -0.16155728697776794,
      "step": 17
    },
    {
      "epoch": 0.01119751166407465,
      "grad_norm": 0.7311789393424988,
      "learning_rate": 4.9550000000000005e-05,
      "log_odds_chosen": -0.04682855308055878,
      "log_odds_ratio": -0.7275374531745911,
      "logits/chosen": -0.08018438518047333,
      "logits/rejected": -0.20024976134300232,
      "logps/chosen": -1.3416996002197266,
      "logps/rejected": -1.3203401565551758,
      "loss": 3.7728,
      "nll_loss": 3.7000904083251953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13416996598243713,
      "rewards/margins": -0.00213593989610672,
      "rewards/rejected": -0.132034033536911,
      "step": 18
    },
    {
      "epoch": 0.01181959564541213,
      "grad_norm": 0.7463477253913879,
      "learning_rate": 4.9525000000000004e-05,
      "log_odds_chosen": -0.6258882284164429,
      "log_odds_ratio": -1.0997464656829834,
      "logits/chosen": -0.12337462604045868,
      "logits/rejected": -0.34859755635261536,
      "logps/chosen": -1.7963664531707764,
      "logps/rejected": -1.2740449905395508,
      "loss": 3.714,
      "nll_loss": 3.603994369506836,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.1796366274356842,
      "rewards/margins": -0.05223213881254196,
      "rewards/rejected": -0.12740451097488403,
      "step": 19
    },
    {
      "epoch": 0.012441679626749611,
      "grad_norm": 0.6245782375335693,
      "learning_rate": 4.9500000000000004e-05,
      "log_odds_chosen": -0.2411532998085022,
      "log_odds_ratio": -0.8302733898162842,
      "logits/chosen": -0.049967993050813675,
      "logits/rejected": -0.2759324908256531,
      "logps/chosen": -1.4726518392562866,
      "logps/rejected": -1.3111158609390259,
      "loss": 3.707,
      "nll_loss": 3.6239380836486816,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.14726518094539642,
      "rewards/margins": -0.01615358516573906,
      "rewards/rejected": -0.13111159205436707,
      "step": 20
    },
    {
      "epoch": 0.013063763608087092,
      "grad_norm": 0.7757321000099182,
      "learning_rate": 4.9475e-05,
      "log_odds_chosen": 0.0013154447078704834,
      "log_odds_ratio": -0.7458984851837158,
      "logits/chosen": -0.016477234661579132,
      "logits/rejected": -0.1882762759923935,
      "logps/chosen": -1.4737496376037598,
      "logps/rejected": -1.4141770601272583,
      "loss": 3.9635,
      "nll_loss": 3.8889150619506836,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1473749727010727,
      "rewards/margins": -0.005957265384495258,
      "rewards/rejected": -0.1414177119731903,
      "step": 21
    },
    {
      "epoch": 0.013685847589424573,
      "grad_norm": 0.5803574323654175,
      "learning_rate": 4.945e-05,
      "log_odds_chosen": -0.024330340325832367,
      "log_odds_ratio": -0.7314656972885132,
      "logits/chosen": -0.0876406729221344,
      "logits/rejected": -0.25811609625816345,
      "logps/chosen": -1.4882334470748901,
      "logps/rejected": -1.4596867561340332,
      "loss": 3.7711,
      "nll_loss": 3.6979050636291504,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1488233506679535,
      "rewards/margins": -0.002854675054550171,
      "rewards/rejected": -0.14596867561340332,
      "step": 22
    },
    {
      "epoch": 0.014307931570762053,
      "grad_norm": 0.5226195454597473,
      "learning_rate": 4.9425e-05,
      "log_odds_chosen": -0.0417926162481308,
      "log_odds_ratio": -0.7332164645195007,
      "logits/chosen": -0.12541602551937103,
      "logits/rejected": -0.2495168298482895,
      "logps/chosen": -1.5487079620361328,
      "logps/rejected": -1.5171284675598145,
      "loss": 3.6381,
      "nll_loss": 3.5647568702697754,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.15487080812454224,
      "rewards/margins": -0.0031579509377479553,
      "rewards/rejected": -0.15171284973621368,
      "step": 23
    },
    {
      "epoch": 0.014930015552099534,
      "grad_norm": 0.49786579608917236,
      "learning_rate": 4.94e-05,
      "log_odds_chosen": -0.24082081019878387,
      "log_odds_ratio": -0.8401607275009155,
      "logits/chosen": -0.15435853600502014,
      "logits/rejected": -0.3182606101036072,
      "logps/chosen": -1.4637329578399658,
      "logps/rejected": -1.2792115211486816,
      "loss": 3.6258,
      "nll_loss": 3.5417871475219727,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14637331664562225,
      "rewards/margins": -0.01845216006040573,
      "rewards/rejected": -0.12792114913463593,
      "step": 24
    },
    {
      "epoch": 0.015552099533437015,
      "grad_norm": 0.7141457200050354,
      "learning_rate": 4.937500000000001e-05,
      "log_odds_chosen": 0.11563286930322647,
      "log_odds_ratio": -0.6847242116928101,
      "logits/chosen": -0.05548834800720215,
      "logits/rejected": -0.25137871503829956,
      "logps/chosen": -1.3125858306884766,
      "logps/rejected": -1.3853249549865723,
      "loss": 3.9615,
      "nll_loss": 3.8930606842041016,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13125859200954437,
      "rewards/margins": 0.007273899391293526,
      "rewards/rejected": -0.13853248953819275,
      "step": 25
    },
    {
      "epoch": 0.016174183514774496,
      "grad_norm": 0.48132890462875366,
      "learning_rate": 4.935e-05,
      "log_odds_chosen": -0.3378981947898865,
      "log_odds_ratio": -1.048970341682434,
      "logits/chosen": -0.27093982696533203,
      "logits/rejected": -0.29352226853370667,
      "logps/chosen": -1.8172366619110107,
      "logps/rejected": -1.4850423336029053,
      "loss": 3.3072,
      "nll_loss": 3.2022593021392822,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18172365427017212,
      "rewards/margins": -0.03321942687034607,
      "rewards/rejected": -0.14850424230098724,
      "step": 26
    },
    {
      "epoch": 0.016796267496111975,
      "grad_norm": 0.465388685464859,
      "learning_rate": 4.9325000000000006e-05,
      "log_odds_chosen": 0.26982247829437256,
      "log_odds_ratio": -0.5840736031532288,
      "logits/chosen": -0.13838329911231995,
      "logits/rejected": -0.14500632882118225,
      "logps/chosen": -1.2994801998138428,
      "logps/rejected": -1.5056928396224976,
      "loss": 3.2789,
      "nll_loss": 3.220458984375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12994801998138428,
      "rewards/margins": 0.020621273666620255,
      "rewards/rejected": -0.15056928992271423,
      "step": 27
    },
    {
      "epoch": 0.017418351477449457,
      "grad_norm": 0.4851706922054291,
      "learning_rate": 4.93e-05,
      "log_odds_chosen": 0.0167313814163208,
      "log_odds_ratio": -0.7801838517189026,
      "logits/chosen": -0.12882889807224274,
      "logits/rejected": -0.27155885100364685,
      "logps/chosen": -1.5362133979797363,
      "logps/rejected": -1.5006215572357178,
      "loss": 3.7467,
      "nll_loss": 3.6686904430389404,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1536213457584381,
      "rewards/margins": -0.0035591907799243927,
      "rewards/rejected": -0.15006214380264282,
      "step": 28
    },
    {
      "epoch": 0.018040435458786936,
      "grad_norm": 0.4745447039604187,
      "learning_rate": 4.9275000000000005e-05,
      "log_odds_chosen": -0.4235059916973114,
      "log_odds_ratio": -0.944256067276001,
      "logits/chosen": -0.15738001465797424,
      "logits/rejected": -0.28892505168914795,
      "logps/chosen": -1.4211246967315674,
      "logps/rejected": -1.1277490854263306,
      "loss": 3.5365,
      "nll_loss": 3.4420742988586426,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.14211246371269226,
      "rewards/margins": -0.029337557032704353,
      "rewards/rejected": -0.11277490854263306,
      "step": 29
    },
    {
      "epoch": 0.01866251944012442,
      "grad_norm": 0.507114052772522,
      "learning_rate": 4.9250000000000004e-05,
      "log_odds_chosen": -0.3637525737285614,
      "log_odds_ratio": -0.9062818884849548,
      "logits/chosen": -0.2181130200624466,
      "logits/rejected": -0.37557047605514526,
      "logps/chosen": -1.2170381546020508,
      "logps/rejected": -0.9694937467575073,
      "loss": 2.9864,
      "nll_loss": 2.8957455158233643,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.12170383334159851,
      "rewards/margins": -0.024754449725151062,
      "rewards/rejected": -0.09694937616586685,
      "step": 30
    },
    {
      "epoch": 0.019284603421461897,
      "grad_norm": 0.40544337034225464,
      "learning_rate": 4.9225000000000004e-05,
      "log_odds_chosen": -0.1934043914079666,
      "log_odds_ratio": -0.8023362159729004,
      "logits/chosen": -0.21577346324920654,
      "logits/rejected": -0.29102498292922974,
      "logps/chosen": -1.170575737953186,
      "logps/rejected": -1.035292625427246,
      "loss": 3.3617,
      "nll_loss": 3.281477928161621,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.11705759167671204,
      "rewards/margins": -0.013528317213058472,
      "rewards/rejected": -0.10352927446365356,
      "step": 31
    },
    {
      "epoch": 0.019906687402799376,
      "grad_norm": 0.5559415221214294,
      "learning_rate": 4.92e-05,
      "log_odds_chosen": -0.3727276027202606,
      "log_odds_ratio": -0.9165624976158142,
      "logits/chosen": -0.17616789042949677,
      "logits/rejected": -0.41072139143943787,
      "logps/chosen": -1.4517570734024048,
      "logps/rejected": -1.1750273704528809,
      "loss": 3.7894,
      "nll_loss": 3.6977787017822266,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.14517571032047272,
      "rewards/margins": -0.027672961354255676,
      "rewards/rejected": -0.11750274896621704,
      "step": 32
    },
    {
      "epoch": 0.02052877138413686,
      "grad_norm": 0.49967700242996216,
      "learning_rate": 4.9175e-05,
      "log_odds_chosen": -0.37599754333496094,
      "log_odds_ratio": -1.035901427268982,
      "logits/chosen": -0.19352321326732635,
      "logits/rejected": -0.37395039200782776,
      "logps/chosen": -1.4342128038406372,
      "logps/rejected": -1.0744060277938843,
      "loss": 3.4439,
      "nll_loss": 3.3403570652008057,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.14342127740383148,
      "rewards/margins": -0.03598066791892052,
      "rewards/rejected": -0.10744060575962067,
      "step": 33
    },
    {
      "epoch": 0.021150855365474338,
      "grad_norm": 0.40192580223083496,
      "learning_rate": 4.915e-05,
      "log_odds_chosen": 0.06716141849756241,
      "log_odds_ratio": -0.68401700258255,
      "logits/chosen": -0.2316037267446518,
      "logits/rejected": -0.34950870275497437,
      "logps/chosen": -1.3879081010818481,
      "logps/rejected": -1.4275767803192139,
      "loss": 3.3671,
      "nll_loss": 3.2987279891967773,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1387908160686493,
      "rewards/margins": 0.003966865129768848,
      "rewards/rejected": -0.14275768399238586,
      "step": 34
    },
    {
      "epoch": 0.02177293934681182,
      "grad_norm": 0.6308751106262207,
      "learning_rate": 4.9125e-05,
      "log_odds_chosen": 0.10806269943714142,
      "log_odds_ratio": -0.6696207523345947,
      "logits/chosen": -0.3126291334629059,
      "logits/rejected": -0.3402310907840729,
      "logps/chosen": -1.4202438592910767,
      "logps/rejected": -1.5030903816223145,
      "loss": 3.0162,
      "nll_loss": 2.949201822280884,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14202438294887543,
      "rewards/margins": 0.008284655399620533,
      "rewards/rejected": -0.15030904114246368,
      "step": 35
    },
    {
      "epoch": 0.0223950233281493,
      "grad_norm": 0.47672948241233826,
      "learning_rate": 4.91e-05,
      "log_odds_chosen": 0.6212818026542664,
      "log_odds_ratio": -0.48152947425842285,
      "logits/chosen": -0.2364703118801117,
      "logits/rejected": -0.3322206139564514,
      "logps/chosen": -0.9610673189163208,
      "logps/rejected": -1.338320016860962,
      "loss": 3.2005,
      "nll_loss": 3.152357339859009,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09610673785209656,
      "rewards/margins": 0.03772527724504471,
      "rewards/rejected": -0.13383200764656067,
      "step": 36
    },
    {
      "epoch": 0.023017107309486782,
      "grad_norm": 0.44931307435035706,
      "learning_rate": 4.907500000000001e-05,
      "log_odds_chosen": -0.29742100834846497,
      "log_odds_ratio": -0.952938437461853,
      "logits/chosen": -0.16727347671985626,
      "logits/rejected": -0.3058803081512451,
      "logps/chosen": -1.608022689819336,
      "logps/rejected": -1.4014194011688232,
      "loss": 3.4746,
      "nll_loss": 3.3793392181396484,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.16080228984355927,
      "rewards/margins": -0.020660335198044777,
      "rewards/rejected": -0.14014194905757904,
      "step": 37
    },
    {
      "epoch": 0.02363919129082426,
      "grad_norm": 0.42830324172973633,
      "learning_rate": 4.905e-05,
      "log_odds_chosen": -0.4679420292377472,
      "log_odds_ratio": -1.0487958192825317,
      "logits/chosen": -0.1773906946182251,
      "logits/rejected": -0.206269308924675,
      "logps/chosen": -1.745017647743225,
      "logps/rejected": -1.3269473314285278,
      "loss": 3.3234,
      "nll_loss": 3.2185542583465576,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.17450176179409027,
      "rewards/margins": -0.04180702194571495,
      "rewards/rejected": -0.13269473612308502,
      "step": 38
    },
    {
      "epoch": 0.024261275272161743,
      "grad_norm": 0.4050178825855255,
      "learning_rate": 4.9025000000000006e-05,
      "log_odds_chosen": 0.10232503712177277,
      "log_odds_ratio": -0.6785426139831543,
      "logits/chosen": -0.18703876435756683,
      "logits/rejected": -0.3412337899208069,
      "logps/chosen": -1.0300055742263794,
      "logps/rejected": -1.0844731330871582,
      "loss": 3.1217,
      "nll_loss": 3.0538713932037354,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.10300055146217346,
      "rewards/margins": 0.0054467711597681046,
      "rewards/rejected": -0.10844732820987701,
      "step": 39
    },
    {
      "epoch": 0.024883359253499222,
      "grad_norm": 0.4841116666793823,
      "learning_rate": 4.9e-05,
      "log_odds_chosen": 0.02147568017244339,
      "log_odds_ratio": -0.6917948722839355,
      "logits/chosen": -0.10562430322170258,
      "logits/rejected": -0.30269837379455566,
      "logps/chosen": -1.4125561714172363,
      "logps/rejected": -1.440596580505371,
      "loss": 3.449,
      "nll_loss": 3.3798322677612305,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14125561714172363,
      "rewards/margins": 0.0028040409088134766,
      "rewards/rejected": -0.1440596580505371,
      "step": 40
    },
    {
      "epoch": 0.0255054432348367,
      "grad_norm": 0.4915813207626343,
      "learning_rate": 4.8975000000000005e-05,
      "log_odds_chosen": -0.11176743358373642,
      "log_odds_ratio": -0.7612941265106201,
      "logits/chosen": -0.08788780868053436,
      "logits/rejected": -0.3535291254520416,
      "logps/chosen": -1.260170578956604,
      "logps/rejected": -1.17002534866333,
      "loss": 3.5663,
      "nll_loss": 3.490180015563965,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12601706385612488,
      "rewards/margins": -0.009014511480927467,
      "rewards/rejected": -0.11700254678726196,
      "step": 41
    },
    {
      "epoch": 0.026127527216174184,
      "grad_norm": 0.4824346601963043,
      "learning_rate": 4.8950000000000004e-05,
      "log_odds_chosen": -0.18203996121883392,
      "log_odds_ratio": -0.80300372838974,
      "logits/chosen": -0.13452517986297607,
      "logits/rejected": -0.14837010204792023,
      "logps/chosen": -1.6312255859375,
      "logps/rejected": -1.490962266921997,
      "loss": 3.3183,
      "nll_loss": 3.2380266189575195,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.16312256455421448,
      "rewards/margins": -0.014026330783963203,
      "rewards/rejected": -0.14909623563289642,
      "step": 42
    },
    {
      "epoch": 0.026749611197511663,
      "grad_norm": 0.40107211470603943,
      "learning_rate": 4.8925e-05,
      "log_odds_chosen": 0.1854020357131958,
      "log_odds_ratio": -0.6462830305099487,
      "logits/chosen": -0.15604250133037567,
      "logits/rejected": -0.18608345091342926,
      "logps/chosen": -1.3835902214050293,
      "logps/rejected": -1.5467376708984375,
      "loss": 3.2676,
      "nll_loss": 3.203000545501709,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13835902512073517,
      "rewards/margins": 0.016314741224050522,
      "rewards/rejected": -0.154673770070076,
      "step": 43
    },
    {
      "epoch": 0.027371695178849145,
      "grad_norm": 0.516572117805481,
      "learning_rate": 4.89e-05,
      "log_odds_chosen": -0.15254731476306915,
      "log_odds_ratio": -0.9058659672737122,
      "logits/chosen": -0.07723084092140198,
      "logits/rejected": -0.3161468803882599,
      "logps/chosen": -1.5304224491119385,
      "logps/rejected": -1.3918745517730713,
      "loss": 3.5929,
      "nll_loss": 3.502323865890503,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1530422568321228,
      "rewards/margins": -0.013854792341589928,
      "rewards/rejected": -0.13918745517730713,
      "step": 44
    },
    {
      "epoch": 0.027993779160186624,
      "grad_norm": 0.4352788031101227,
      "learning_rate": 4.8875e-05,
      "log_odds_chosen": 0.12119769304990768,
      "log_odds_ratio": -0.6822906136512756,
      "logits/chosen": -0.08103634417057037,
      "logits/rejected": -0.23941874504089355,
      "logps/chosen": -1.2429063320159912,
      "logps/rejected": -1.3138731718063354,
      "loss": 3.4242,
      "nll_loss": 3.3560192584991455,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.12429064512252808,
      "rewards/margins": 0.007096678018569946,
      "rewards/rejected": -0.13138732314109802,
      "step": 45
    },
    {
      "epoch": 0.028615863141524107,
      "grad_norm": 0.45666632056236267,
      "learning_rate": 4.885e-05,
      "log_odds_chosen": 0.277486115694046,
      "log_odds_ratio": -0.6156935095787048,
      "logits/chosen": -0.16658517718315125,
      "logits/rejected": -0.2931288778781891,
      "logps/chosen": -1.0638997554779053,
      "logps/rejected": -1.2447154521942139,
      "loss": 2.8714,
      "nll_loss": 2.8098669052124023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10638997703790665,
      "rewards/margins": 0.018081560730934143,
      "rewards/rejected": -0.12447153031826019,
      "step": 46
    },
    {
      "epoch": 0.029237947122861586,
      "grad_norm": 0.4158986806869507,
      "learning_rate": 4.8825e-05,
      "log_odds_chosen": -0.03135260194540024,
      "log_odds_ratio": -0.7471761107444763,
      "logits/chosen": 0.007953077554702759,
      "logits/rejected": -0.1722358912229538,
      "logps/chosen": -1.3186936378479004,
      "logps/rejected": -1.2820833921432495,
      "loss": 3.5286,
      "nll_loss": 3.453866958618164,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.131869375705719,
      "rewards/margins": -0.0036610299721360207,
      "rewards/rejected": -0.12820833921432495,
      "step": 47
    },
    {
      "epoch": 0.029860031104199068,
      "grad_norm": 0.5239469408988953,
      "learning_rate": 4.88e-05,
      "log_odds_chosen": -0.14184284210205078,
      "log_odds_ratio": -0.7821545600891113,
      "logits/chosen": -0.00846000388264656,
      "logits/rejected": -0.24844269454479218,
      "logps/chosen": -1.4144936800003052,
      "logps/rejected": -1.2858200073242188,
      "loss": 3.157,
      "nll_loss": 3.07881236076355,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14144936203956604,
      "rewards/margins": -0.01286737434566021,
      "rewards/rejected": -0.12858200073242188,
      "step": 48
    },
    {
      "epoch": 0.030482115085536547,
      "grad_norm": 0.42745500802993774,
      "learning_rate": 4.8775000000000007e-05,
      "log_odds_chosen": -0.016907602548599243,
      "log_odds_ratio": -0.7335056066513062,
      "logits/chosen": 0.027836868539452553,
      "logits/rejected": -0.16093496978282928,
      "logps/chosen": -1.3329463005065918,
      "logps/rejected": -1.3012332916259766,
      "loss": 3.7355,
      "nll_loss": 3.6621429920196533,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13329462707042694,
      "rewards/margins": -0.00317130284383893,
      "rewards/rejected": -0.1301233321428299,
      "step": 49
    },
    {
      "epoch": 0.03110419906687403,
      "grad_norm": 0.5647901296615601,
      "learning_rate": 4.875e-05,
      "log_odds_chosen": 0.05693642795085907,
      "log_odds_ratio": -0.7423352003097534,
      "logits/chosen": 0.07700448483228683,
      "logits/rejected": -0.21037913858890533,
      "logps/chosen": -1.3298025131225586,
      "logps/rejected": -1.3227578401565552,
      "loss": 3.6339,
      "nll_loss": 3.55966854095459,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13298025727272034,
      "rewards/margins": -0.0007044710218906403,
      "rewards/rejected": -0.13227578997612,
      "step": 50
    },
    {
      "epoch": 0.031726283048211505,
      "grad_norm": 0.5185202360153198,
      "learning_rate": 4.8725000000000005e-05,
      "log_odds_chosen": 0.271283358335495,
      "log_odds_ratio": -0.5842689275741577,
      "logits/chosen": -0.0015755696222186089,
      "logits/rejected": -0.17162489891052246,
      "logps/chosen": -1.039259672164917,
      "logps/rejected": -1.1926697492599487,
      "loss": 3.2335,
      "nll_loss": 3.1750574111938477,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10392597317695618,
      "rewards/margins": 0.015341001562774181,
      "rewards/rejected": -0.11926697194576263,
      "step": 51
    },
    {
      "epoch": 0.03234836702954899,
      "grad_norm": 0.3914552927017212,
      "learning_rate": 4.87e-05,
      "log_odds_chosen": -0.09510594606399536,
      "log_odds_ratio": -0.7722680568695068,
      "logits/chosen": -0.11492118239402771,
      "logits/rejected": -0.1295318901538849,
      "logps/chosen": -1.2744344472885132,
      "logps/rejected": -1.1613863706588745,
      "loss": 2.7643,
      "nll_loss": 2.6871042251586914,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.12744343280792236,
      "rewards/margins": -0.011304810643196106,
      "rewards/rejected": -0.11613863706588745,
      "step": 52
    },
    {
      "epoch": 0.03297045101088647,
      "grad_norm": 0.42848122119903564,
      "learning_rate": 4.8675000000000004e-05,
      "log_odds_chosen": 0.13229113817214966,
      "log_odds_ratio": -0.659803569316864,
      "logits/chosen": -0.206315815448761,
      "logits/rejected": -0.24223698675632477,
      "logps/chosen": -1.1478495597839355,
      "logps/rejected": -1.2279183864593506,
      "loss": 2.5846,
      "nll_loss": 2.5186636447906494,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11478495597839355,
      "rewards/margins": 0.008006898686289787,
      "rewards/rejected": -0.12279185652732849,
      "step": 53
    },
    {
      "epoch": 0.03359253499222395,
      "grad_norm": 0.9718621373176575,
      "learning_rate": 4.8650000000000003e-05,
      "log_odds_chosen": 0.4774380326271057,
      "log_odds_ratio": -0.5322307348251343,
      "logits/chosen": 0.07056058198213577,
      "logits/rejected": -0.18487195670604706,
      "logps/chosen": -1.300857663154602,
      "logps/rejected": -1.635850429534912,
      "loss": 3.7447,
      "nll_loss": 3.6914896965026855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.130085751414299,
      "rewards/margins": 0.033499278128147125,
      "rewards/rejected": -0.16358503699302673,
      "step": 54
    },
    {
      "epoch": 0.03421461897356143,
      "grad_norm": 0.341932475566864,
      "learning_rate": 4.8625e-05,
      "log_odds_chosen": 0.007643640041351318,
      "log_odds_ratio": -0.7581207752227783,
      "logits/chosen": -0.09210430085659027,
      "logits/rejected": -0.0826970785856247,
      "logps/chosen": -1.4047452211380005,
      "logps/rejected": -1.3780653476715088,
      "loss": 3.1252,
      "nll_loss": 3.0493462085723877,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14047454297542572,
      "rewards/margins": -0.002668004482984543,
      "rewards/rejected": -0.13780653476715088,
      "step": 55
    },
    {
      "epoch": 0.034836702954898914,
      "grad_norm": 0.45145970582962036,
      "learning_rate": 4.86e-05,
      "log_odds_chosen": -0.24970951676368713,
      "log_odds_ratio": -0.8379786014556885,
      "logits/chosen": 0.09427239745855331,
      "logits/rejected": 0.03530137240886688,
      "logps/chosen": -1.4495787620544434,
      "logps/rejected": -1.284151554107666,
      "loss": 2.9932,
      "nll_loss": 2.90944242477417,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14495787024497986,
      "rewards/margins": -0.0165427103638649,
      "rewards/rejected": -0.12841516733169556,
      "step": 56
    },
    {
      "epoch": 0.03545878693623639,
      "grad_norm": 0.7668087482452393,
      "learning_rate": 4.8575e-05,
      "log_odds_chosen": -0.44778311252593994,
      "log_odds_ratio": -1.0893973112106323,
      "logits/chosen": -0.056043416261672974,
      "logits/rejected": -0.1815190464258194,
      "logps/chosen": -1.829887866973877,
      "logps/rejected": -1.4810354709625244,
      "loss": 3.1517,
      "nll_loss": 3.0427865982055664,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.18298880755901337,
      "rewards/margins": -0.03488525003194809,
      "rewards/rejected": -0.14810355007648468,
      "step": 57
    },
    {
      "epoch": 0.03608087091757387,
      "grad_norm": 0.49874478578567505,
      "learning_rate": 4.855e-05,
      "log_odds_chosen": 0.3892138600349426,
      "log_odds_ratio": -0.5497685670852661,
      "logits/chosen": -0.11005600541830063,
      "logits/rejected": -0.2566314935684204,
      "logps/chosen": -1.1769773960113525,
      "logps/rejected": -1.4603263139724731,
      "loss": 3.0813,
      "nll_loss": 3.026326894760132,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11769775301218033,
      "rewards/margins": 0.028334885835647583,
      "rewards/rejected": -0.14603263139724731,
      "step": 58
    },
    {
      "epoch": 0.03670295489891135,
      "grad_norm": 0.3849422335624695,
      "learning_rate": 4.8525e-05,
      "log_odds_chosen": -0.050915684551000595,
      "log_odds_ratio": -0.7625954151153564,
      "logits/chosen": -0.010294832289218903,
      "logits/rejected": -0.12012767791748047,
      "logps/chosen": -1.289703607559204,
      "logps/rejected": -1.2587721347808838,
      "loss": 3.4305,
      "nll_loss": 3.354198932647705,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12897036969661713,
      "rewards/margins": -0.0030931569635868073,
      "rewards/rejected": -0.12587721645832062,
      "step": 59
    },
    {
      "epoch": 0.03732503888024884,
      "grad_norm": 0.3733402192592621,
      "learning_rate": 4.85e-05,
      "log_odds_chosen": 0.9943278431892395,
      "log_odds_ratio": -0.46127861738204956,
      "logits/chosen": -0.16634932160377502,
      "logits/rejected": -0.24419239163398743,
      "logps/chosen": -0.8558648824691772,
      "logps/rejected": -1.3538252115249634,
      "loss": 3.0007,
      "nll_loss": 2.954594612121582,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08558649569749832,
      "rewards/margins": 0.049796026200056076,
      "rewards/rejected": -0.1353825181722641,
      "step": 60
    },
    {
      "epoch": 0.037947122861586316,
      "grad_norm": 0.47226205468177795,
      "learning_rate": 4.8475000000000006e-05,
      "log_odds_chosen": 0.36546510457992554,
      "log_odds_ratio": -0.5973978042602539,
      "logits/chosen": 0.09770157933235168,
      "logits/rejected": -0.09729360044002533,
      "logps/chosen": -1.3515472412109375,
      "logps/rejected": -1.606210470199585,
      "loss": 3.6102,
      "nll_loss": 3.5504493713378906,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13515472412109375,
      "rewards/margins": 0.025466332212090492,
      "rewards/rejected": -0.1606210470199585,
      "step": 61
    },
    {
      "epoch": 0.038569206842923795,
      "grad_norm": 0.38134610652923584,
      "learning_rate": 4.845e-05,
      "log_odds_chosen": 0.3819832503795624,
      "log_odds_ratio": -0.6043448448181152,
      "logits/chosen": 0.1279037743806839,
      "logits/rejected": 0.05211859941482544,
      "logps/chosen": -1.3050546646118164,
      "logps/rejected": -1.5339863300323486,
      "loss": 3.3049,
      "nll_loss": 3.244431972503662,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13050545752048492,
      "rewards/margins": 0.02289317362010479,
      "rewards/rejected": -0.15339863300323486,
      "step": 62
    },
    {
      "epoch": 0.039191290824261274,
      "grad_norm": 0.5376470685005188,
      "learning_rate": 4.8425000000000005e-05,
      "log_odds_chosen": 0.21296370029449463,
      "log_odds_ratio": -0.6956855058670044,
      "logits/chosen": 0.005937471985816956,
      "logits/rejected": -0.13821996748447418,
      "logps/chosen": -1.2709373235702515,
      "logps/rejected": -1.3110651969909668,
      "loss": 3.4905,
      "nll_loss": 3.4209353923797607,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12709373235702515,
      "rewards/margins": 0.004012777470052242,
      "rewards/rejected": -0.13110651075839996,
      "step": 63
    },
    {
      "epoch": 0.03981337480559875,
      "grad_norm": 0.45539331436157227,
      "learning_rate": 4.8400000000000004e-05,
      "log_odds_chosen": -0.19902189075946808,
      "log_odds_ratio": -0.8536033630371094,
      "logits/chosen": 0.059451624751091,
      "logits/rejected": -0.11644947528839111,
      "logps/chosen": -1.4663325548171997,
      "logps/rejected": -1.3308027982711792,
      "loss": 3.6228,
      "nll_loss": 3.5374748706817627,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14663325250148773,
      "rewards/margins": -0.013552968390285969,
      "rewards/rejected": -0.13308028876781464,
      "step": 64
    },
    {
      "epoch": 0.04043545878693624,
      "grad_norm": 0.3535555899143219,
      "learning_rate": 4.8375000000000004e-05,
      "log_odds_chosen": -0.03207793086767197,
      "log_odds_ratio": -0.724827766418457,
      "logits/chosen": 0.035917554050683975,
      "logits/rejected": -0.06365230679512024,
      "logps/chosen": -1.3977922201156616,
      "logps/rejected": -1.3612167835235596,
      "loss": 3.3147,
      "nll_loss": 3.2421815395355225,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1397792100906372,
      "rewards/margins": -0.0036575384438037872,
      "rewards/rejected": -0.13612167537212372,
      "step": 65
    },
    {
      "epoch": 0.04105754276827372,
      "grad_norm": 0.4546717405319214,
      "learning_rate": 4.835e-05,
      "log_odds_chosen": -0.03722512722015381,
      "log_odds_ratio": -0.7913417220115662,
      "logits/chosen": 0.09111925959587097,
      "logits/rejected": -0.1404401957988739,
      "logps/chosen": -1.3331866264343262,
      "logps/rejected": -1.2620258331298828,
      "loss": 3.2911,
      "nll_loss": 3.2119736671447754,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13331866264343262,
      "rewards/margins": -0.007116083987057209,
      "rewards/rejected": -0.12620258331298828,
      "step": 66
    },
    {
      "epoch": 0.0416796267496112,
      "grad_norm": 0.30916666984558105,
      "learning_rate": 4.8325e-05,
      "log_odds_chosen": -0.3806450664997101,
      "log_odds_ratio": -0.9495774507522583,
      "logits/chosen": 0.22228950262069702,
      "logits/rejected": 0.011185593903064728,
      "logps/chosen": -1.36622953414917,
      "logps/rejected": -1.1000258922576904,
      "loss": 3.9335,
      "nll_loss": 3.8385517597198486,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.13662296533584595,
      "rewards/margins": -0.026620371267199516,
      "rewards/rejected": -0.11000259220600128,
      "step": 67
    },
    {
      "epoch": 0.042301710730948676,
      "grad_norm": 0.5843622088432312,
      "learning_rate": 4.83e-05,
      "log_odds_chosen": -0.04287657141685486,
      "log_odds_ratio": -0.7503262758255005,
      "logits/chosen": 0.12020996958017349,
      "logits/rejected": -0.06287462264299393,
      "logps/chosen": -1.2391448020935059,
      "logps/rejected": -1.2382404804229736,
      "loss": 2.7952,
      "nll_loss": 2.72021484375,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12391449511051178,
      "rewards/margins": -9.043654426932335e-05,
      "rewards/rejected": -0.12382405996322632,
      "step": 68
    },
    {
      "epoch": 0.04292379471228616,
      "grad_norm": 0.40152105689048767,
      "learning_rate": 4.8275e-05,
      "log_odds_chosen": 0.48192113637924194,
      "log_odds_ratio": -0.5793935060501099,
      "logits/chosen": 0.024148011580109596,
      "logits/rejected": -0.028608618304133415,
      "logps/chosen": -1.2637081146240234,
      "logps/rejected": -1.5951621532440186,
      "loss": 3.0237,
      "nll_loss": 2.9657363891601562,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12637081742286682,
      "rewards/margins": 0.03314541280269623,
      "rewards/rejected": -0.15951621532440186,
      "step": 69
    },
    {
      "epoch": 0.04354587869362364,
      "grad_norm": 0.5005712509155273,
      "learning_rate": 4.825e-05,
      "log_odds_chosen": 0.16657370328903198,
      "log_odds_ratio": -0.6632782220840454,
      "logits/chosen": 0.0074752867221832275,
      "logits/rejected": -0.15818460285663605,
      "logps/chosen": -1.009189486503601,
      "logps/rejected": -1.074352502822876,
      "loss": 3.0056,
      "nll_loss": 2.939265012741089,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1009189561009407,
      "rewards/margins": 0.006516308058053255,
      "rewards/rejected": -0.10743525624275208,
      "step": 70
    },
    {
      "epoch": 0.04416796267496112,
      "grad_norm": 0.47246062755584717,
      "learning_rate": 4.822500000000001e-05,
      "log_odds_chosen": -0.008882712572813034,
      "log_odds_ratio": -0.799690842628479,
      "logits/chosen": 0.1656499207019806,
      "logits/rejected": 0.012578403577208519,
      "logps/chosen": -1.1688836812973022,
      "logps/rejected": -1.1197009086608887,
      "loss": 3.2172,
      "nll_loss": 3.137253999710083,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1168883666396141,
      "rewards/margins": -0.0049182698130607605,
      "rewards/rejected": -0.11197009682655334,
      "step": 71
    },
    {
      "epoch": 0.0447900466562986,
      "grad_norm": 0.5179511308670044,
      "learning_rate": 4.82e-05,
      "log_odds_chosen": -0.01626509428024292,
      "log_odds_ratio": -0.7597264051437378,
      "logits/chosen": -0.07426048815250397,
      "logits/rejected": -0.21345210075378418,
      "logps/chosen": -1.305432677268982,
      "logps/rejected": -1.3045635223388672,
      "loss": 2.7467,
      "nll_loss": 2.6707706451416016,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13054326176643372,
      "rewards/margins": -8.692499250173569e-05,
      "rewards/rejected": -0.1304563581943512,
      "step": 72
    },
    {
      "epoch": 0.04541213063763608,
      "grad_norm": 0.3978026211261749,
      "learning_rate": 4.8175000000000005e-05,
      "log_odds_chosen": -0.17721597850322723,
      "log_odds_ratio": -0.8433041572570801,
      "logits/chosen": -0.00016376003623008728,
      "logits/rejected": -0.1693696677684784,
      "logps/chosen": -1.6973767280578613,
      "logps/rejected": -1.5059082508087158,
      "loss": 3.4049,
      "nll_loss": 3.3205349445343018,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.16973768174648285,
      "rewards/margins": -0.019146855920553207,
      "rewards/rejected": -0.15059083700180054,
      "step": 73
    },
    {
      "epoch": 0.046034214618973564,
      "grad_norm": 0.44062405824661255,
      "learning_rate": 4.815e-05,
      "log_odds_chosen": 0.29165250062942505,
      "log_odds_ratio": -0.6030845642089844,
      "logits/chosen": 0.03590073063969612,
      "logits/rejected": -0.05357559770345688,
      "logps/chosen": -0.9286098480224609,
      "logps/rejected": -1.074052333831787,
      "loss": 3.1473,
      "nll_loss": 3.0869500637054443,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09286098182201385,
      "rewards/margins": 0.014544256031513214,
      "rewards/rejected": -0.10740524530410767,
      "step": 74
    },
    {
      "epoch": 0.04665629860031104,
      "grad_norm": 0.3442816436290741,
      "learning_rate": 4.8125000000000004e-05,
      "log_odds_chosen": 0.2888907194137573,
      "log_odds_ratio": -0.6076911687850952,
      "logits/chosen": -0.12151262164115906,
      "logits/rejected": -0.12172101438045502,
      "logps/chosen": -1.0215524435043335,
      "logps/rejected": -1.1889455318450928,
      "loss": 2.7404,
      "nll_loss": 2.6796624660491943,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10215524584054947,
      "rewards/margins": 0.016739321872591972,
      "rewards/rejected": -0.118894562125206,
      "step": 75
    },
    {
      "epoch": 0.04727838258164852,
      "grad_norm": 0.3842046558856964,
      "learning_rate": 4.8100000000000004e-05,
      "log_odds_chosen": -0.25257712602615356,
      "log_odds_ratio": -0.8657203912734985,
      "logits/chosen": 0.023503951728343964,
      "logits/rejected": -0.18105646967887878,
      "logps/chosen": -1.4399148225784302,
      "logps/rejected": -1.2502027750015259,
      "loss": 3.2649,
      "nll_loss": 3.178344488143921,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14399148523807526,
      "rewards/margins": -0.01897120475769043,
      "rewards/rejected": -0.12502028048038483,
      "step": 76
    },
    {
      "epoch": 0.047900466562986,
      "grad_norm": 0.3534882664680481,
      "learning_rate": 4.8075e-05,
      "log_odds_chosen": -0.21401476860046387,
      "log_odds_ratio": -0.8372653722763062,
      "logits/chosen": 0.14382417500019073,
      "logits/rejected": -0.052027709782123566,
      "logps/chosen": -1.4991471767425537,
      "logps/rejected": -1.312038540840149,
      "loss": 3.7097,
      "nll_loss": 3.625964403152466,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1499147266149521,
      "rewards/margins": -0.01871085911989212,
      "rewards/rejected": -0.13120386004447937,
      "step": 77
    },
    {
      "epoch": 0.04852255054432349,
      "grad_norm": 0.8299946188926697,
      "learning_rate": 4.805e-05,
      "log_odds_chosen": -0.24282418191432953,
      "log_odds_ratio": -0.8619076013565063,
      "logits/chosen": -0.18960991501808167,
      "logits/rejected": -0.34159693121910095,
      "logps/chosen": -1.1464121341705322,
      "logps/rejected": -0.992071270942688,
      "loss": 2.5854,
      "nll_loss": 2.4992449283599854,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1146412119269371,
      "rewards/margins": -0.015434084460139275,
      "rewards/rejected": -0.09920713305473328,
      "step": 78
    },
    {
      "epoch": 0.049144634525660966,
      "grad_norm": 0.5896576046943665,
      "learning_rate": 4.8025e-05,
      "log_odds_chosen": 0.14239203929901123,
      "log_odds_ratio": -0.6772485375404358,
      "logits/chosen": -0.02156006544828415,
      "logits/rejected": -0.23830567300319672,
      "logps/chosen": -1.3663321733474731,
      "logps/rejected": -1.4884802103042603,
      "loss": 3.0472,
      "nll_loss": 2.9794466495513916,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13663321733474731,
      "rewards/margins": 0.012214799411594868,
      "rewards/rejected": -0.1488480269908905,
      "step": 79
    },
    {
      "epoch": 0.049766718506998445,
      "grad_norm": 0.5111446976661682,
      "learning_rate": 4.8e-05,
      "log_odds_chosen": 0.06895725429058075,
      "log_odds_ratio": -0.6709849834442139,
      "logits/chosen": 0.03493687883019447,
      "logits/rejected": -0.12141556292772293,
      "logps/chosen": -1.3161884546279907,
      "logps/rejected": -1.3568629026412964,
      "loss": 2.9872,
      "nll_loss": 2.9200947284698486,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13161885738372803,
      "rewards/margins": 0.0040674470365047455,
      "rewards/rejected": -0.13568630814552307,
      "step": 80
    },
    {
      "epoch": 0.050388802488335924,
      "grad_norm": 0.5454034805297852,
      "learning_rate": 4.7975e-05,
      "log_odds_chosen": 0.013033639639616013,
      "log_odds_ratio": -0.7662529945373535,
      "logits/chosen": 0.09307534992694855,
      "logits/rejected": -0.13599663972854614,
      "logps/chosen": -1.2162799835205078,
      "logps/rejected": -1.1834042072296143,
      "loss": 3.0209,
      "nll_loss": 2.9443132877349854,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12162800133228302,
      "rewards/margins": -0.0032875724136829376,
      "rewards/rejected": -0.11834041774272919,
      "step": 81
    },
    {
      "epoch": 0.0510108864696734,
      "grad_norm": 0.4710577726364136,
      "learning_rate": 4.795e-05,
      "log_odds_chosen": 0.1277192234992981,
      "log_odds_ratio": -0.6501528024673462,
      "logits/chosen": 0.048733945935964584,
      "logits/rejected": -0.10845337063074112,
      "logps/chosen": -1.2608442306518555,
      "logps/rejected": -1.3538391590118408,
      "loss": 2.9852,
      "nll_loss": 2.920222759246826,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12608441710472107,
      "rewards/margins": 0.0092995036393404,
      "rewards/rejected": -0.13538393378257751,
      "step": 82
    },
    {
      "epoch": 0.05163297045101089,
      "grad_norm": 0.4011155068874359,
      "learning_rate": 4.7925000000000006e-05,
      "log_odds_chosen": 0.20483741164207458,
      "log_odds_ratio": -0.6441969275474548,
      "logits/chosen": 0.017529848963022232,
      "logits/rejected": -0.17869825661182404,
      "logps/chosen": -1.1669516563415527,
      "logps/rejected": -1.279728889465332,
      "loss": 2.7464,
      "nll_loss": 2.6819615364074707,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11669516563415527,
      "rewards/margins": 0.011277715675532818,
      "rewards/rejected": -0.12797288596630096,
      "step": 83
    },
    {
      "epoch": 0.05225505443234837,
      "grad_norm": 0.5642073750495911,
      "learning_rate": 4.79e-05,
      "log_odds_chosen": 0.16463297605514526,
      "log_odds_ratio": -0.6702374219894409,
      "logits/chosen": 0.25281310081481934,
      "logits/rejected": -0.022312596440315247,
      "logps/chosen": -1.236396074295044,
      "logps/rejected": -1.3540095090866089,
      "loss": 3.6312,
      "nll_loss": 3.564155101776123,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12363961338996887,
      "rewards/margins": 0.01176133006811142,
      "rewards/rejected": -0.1354009509086609,
      "step": 84
    },
    {
      "epoch": 0.05287713841368585,
      "grad_norm": 0.42271995544433594,
      "learning_rate": 4.7875000000000005e-05,
      "log_odds_chosen": 0.29148802161216736,
      "log_odds_ratio": -0.6428624987602234,
      "logits/chosen": 0.16957850754261017,
      "logits/rejected": -0.029354337602853775,
      "logps/chosen": -1.3640344142913818,
      "logps/rejected": -1.4766488075256348,
      "loss": 3.5189,
      "nll_loss": 3.45458984375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13640344142913818,
      "rewards/margins": 0.01126144826412201,
      "rewards/rejected": -0.1476649045944214,
      "step": 85
    },
    {
      "epoch": 0.053499222395023326,
      "grad_norm": 0.6085870265960693,
      "learning_rate": 4.785e-05,
      "log_odds_chosen": 0.020148158073425293,
      "log_odds_ratio": -0.6988396644592285,
      "logits/chosen": -0.01158811990171671,
      "logits/rejected": -0.15423518419265747,
      "logps/chosen": -1.4441418647766113,
      "logps/rejected": -1.4638988971710205,
      "loss": 2.5733,
      "nll_loss": 2.5033936500549316,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14441418647766113,
      "rewards/margins": 0.0019757067784667015,
      "rewards/rejected": -0.146389901638031,
      "step": 86
    },
    {
      "epoch": 0.05412130637636081,
      "grad_norm": 0.4892805814743042,
      "learning_rate": 4.7825000000000004e-05,
      "log_odds_chosen": 0.8066454529762268,
      "log_odds_ratio": -0.4088033139705658,
      "logits/chosen": 0.12198805809020996,
      "logits/rejected": -0.12190286070108414,
      "logps/chosen": -0.8199340105056763,
      "logps/rejected": -1.2292183637619019,
      "loss": 2.7766,
      "nll_loss": 2.73576021194458,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08199340105056763,
      "rewards/margins": 0.040928442031145096,
      "rewards/rejected": -0.12292183935642242,
      "step": 87
    },
    {
      "epoch": 0.05474339035769829,
      "grad_norm": 0.3810538351535797,
      "learning_rate": 4.78e-05,
      "log_odds_chosen": -0.106544628739357,
      "log_odds_ratio": -0.7797843217849731,
      "logits/chosen": 0.23251253366470337,
      "logits/rejected": 0.10770893096923828,
      "logps/chosen": -1.1218090057373047,
      "logps/rejected": -1.0476629734039307,
      "loss": 3.2626,
      "nll_loss": 3.1846585273742676,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11218090355396271,
      "rewards/margins": -0.007414607331156731,
      "rewards/rejected": -0.10476629436016083,
      "step": 88
    },
    {
      "epoch": 0.05536547433903577,
      "grad_norm": 0.5943082571029663,
      "learning_rate": 4.7775e-05,
      "log_odds_chosen": 0.18490558862686157,
      "log_odds_ratio": -0.6679292917251587,
      "logits/chosen": 0.14844851195812225,
      "logits/rejected": -0.0743369460105896,
      "logps/chosen": -1.268092155456543,
      "logps/rejected": -1.3130736351013184,
      "loss": 3.2991,
      "nll_loss": 3.232351779937744,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12680920958518982,
      "rewards/margins": 0.0044981567189097404,
      "rewards/rejected": -0.13130736351013184,
      "step": 89
    },
    {
      "epoch": 0.05598755832037325,
      "grad_norm": 0.5314677357673645,
      "learning_rate": 4.775e-05,
      "log_odds_chosen": 0.2689226269721985,
      "log_odds_ratio": -0.6226869225502014,
      "logits/chosen": 0.06861215829849243,
      "logits/rejected": -0.163130983710289,
      "logps/chosen": -1.3787274360656738,
      "logps/rejected": -1.6074514389038086,
      "loss": 2.6911,
      "nll_loss": 2.6288278102874756,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13787275552749634,
      "rewards/margins": 0.022872384637594223,
      "rewards/rejected": -0.16074511408805847,
      "step": 90
    },
    {
      "epoch": 0.05660964230171073,
      "grad_norm": 0.4301508367061615,
      "learning_rate": 4.7725e-05,
      "log_odds_chosen": 0.3461243510246277,
      "log_odds_ratio": -0.646808385848999,
      "logits/chosen": 0.24266819655895233,
      "logits/rejected": 0.029673846438527107,
      "logps/chosen": -1.2074958086013794,
      "logps/rejected": -1.411824345588684,
      "loss": 3.5601,
      "nll_loss": 3.495389938354492,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1207495778799057,
      "rewards/margins": 0.02043285220861435,
      "rewards/rejected": -0.14118242263793945,
      "step": 91
    },
    {
      "epoch": 0.05723172628304821,
      "grad_norm": 0.4627557694911957,
      "learning_rate": 4.77e-05,
      "log_odds_chosen": -0.28584611415863037,
      "log_odds_ratio": -0.9114346504211426,
      "logits/chosen": 0.21556691825389862,
      "logits/rejected": 0.014135261997580528,
      "logps/chosen": -1.367776870727539,
      "logps/rejected": -1.1591300964355469,
      "loss": 3.5889,
      "nll_loss": 3.497756004333496,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.13677769899368286,
      "rewards/margins": -0.02086469531059265,
      "rewards/rejected": -0.11591300368309021,
      "step": 92
    },
    {
      "epoch": 0.05785381026438569,
      "grad_norm": 0.37699708342552185,
      "learning_rate": 4.7675e-05,
      "log_odds_chosen": 0.3036189377307892,
      "log_odds_ratio": -0.5816516876220703,
      "logits/chosen": 0.01264739129692316,
      "logits/rejected": -0.06274904310703278,
      "logps/chosen": -0.9731736183166504,
      "logps/rejected": -1.1666101217269897,
      "loss": 3.0008,
      "nll_loss": 2.942596673965454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09731736779212952,
      "rewards/margins": 0.019343651831150055,
      "rewards/rejected": -0.11666101962327957,
      "step": 93
    },
    {
      "epoch": 0.05847589424572317,
      "grad_norm": 0.4950994551181793,
      "learning_rate": 4.765e-05,
      "log_odds_chosen": -0.003652891144156456,
      "log_odds_ratio": -0.7005108594894409,
      "logits/chosen": 0.1562429666519165,
      "logits/rejected": -0.07231743633747101,
      "logps/chosen": -1.5042771100997925,
      "logps/rejected": -1.506368637084961,
      "loss": 3.2415,
      "nll_loss": 3.1714537143707275,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1504277139902115,
      "rewards/margins": 0.0002091517671942711,
      "rewards/rejected": -0.15063685178756714,
      "step": 94
    },
    {
      "epoch": 0.05909797822706065,
      "grad_norm": 0.5913958549499512,
      "learning_rate": 4.7625000000000006e-05,
      "log_odds_chosen": -0.5333858132362366,
      "log_odds_ratio": -1.058704137802124,
      "logits/chosen": 0.2847288250923157,
      "logits/rejected": -0.04206930845975876,
      "logps/chosen": -2.0648934841156006,
      "logps/rejected": -1.5971152782440186,
      "loss": 3.879,
      "nll_loss": 3.7731385231018066,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.20648935437202454,
      "rewards/margins": -0.04677780717611313,
      "rewards/rejected": -0.1597115397453308,
      "step": 95
    },
    {
      "epoch": 0.059720062208398136,
      "grad_norm": 0.46030476689338684,
      "learning_rate": 4.76e-05,
      "log_odds_chosen": -0.05350463092327118,
      "log_odds_ratio": -0.7248205542564392,
      "logits/chosen": 0.07910023629665375,
      "logits/rejected": -0.06675226986408234,
      "logps/chosen": -1.3112492561340332,
      "logps/rejected": -1.2683042287826538,
      "loss": 2.5608,
      "nll_loss": 2.4883413314819336,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13112492859363556,
      "rewards/margins": -0.0042945025488734245,
      "rewards/rejected": -0.12683042883872986,
      "step": 96
    },
    {
      "epoch": 0.060342146189735615,
      "grad_norm": 0.8850117921829224,
      "learning_rate": 4.7575000000000004e-05,
      "log_odds_chosen": -0.16680529713630676,
      "log_odds_ratio": -0.918777585029602,
      "logits/chosen": 0.09566190838813782,
      "logits/rejected": -0.09266671538352966,
      "logps/chosen": -1.5960246324539185,
      "logps/rejected": -1.3469147682189941,
      "loss": 3.1828,
      "nll_loss": 3.0909605026245117,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15960246324539185,
      "rewards/margins": -0.024910978972911835,
      "rewards/rejected": -0.13469147682189941,
      "step": 97
    },
    {
      "epoch": 0.060964230171073094,
      "grad_norm": 0.43437162041664124,
      "learning_rate": 4.755e-05,
      "log_odds_chosen": 0.30574995279312134,
      "log_odds_ratio": -0.6281970143318176,
      "logits/chosen": 0.08859042823314667,
      "logits/rejected": -0.02086031809449196,
      "logps/chosen": -1.3456239700317383,
      "logps/rejected": -1.5943999290466309,
      "loss": 3.2651,
      "nll_loss": 3.2022864818573,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1345623880624771,
      "rewards/margins": 0.02487758919596672,
      "rewards/rejected": -0.15943998098373413,
      "step": 98
    },
    {
      "epoch": 0.06158631415241057,
      "grad_norm": 0.6766498684883118,
      "learning_rate": 4.7525e-05,
      "log_odds_chosen": -0.11186225712299347,
      "log_odds_ratio": -0.7736210823059082,
      "logits/chosen": 0.04021189361810684,
      "logits/rejected": 0.03945222869515419,
      "logps/chosen": -1.3682116270065308,
      "logps/rejected": -1.301282286643982,
      "loss": 2.6354,
      "nll_loss": 2.5580055713653564,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13682116568088531,
      "rewards/margins": -0.006692924536764622,
      "rewards/rejected": -0.13012823462486267,
      "step": 99
    },
    {
      "epoch": 0.06220839813374806,
      "grad_norm": 0.5133693218231201,
      "learning_rate": 4.75e-05,
      "log_odds_chosen": -0.17678174376487732,
      "log_odds_ratio": -0.8037921786308289,
      "logits/chosen": 0.03568641096353531,
      "logits/rejected": -0.08334266394376755,
      "logps/chosen": -1.4727808237075806,
      "logps/rejected": -1.3427841663360596,
      "loss": 2.8091,
      "nll_loss": 2.7287254333496094,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1472780704498291,
      "rewards/margins": -0.012999659404158592,
      "rewards/rejected": -0.13427841663360596,
      "step": 100
    },
    {
      "epoch": 0.06283048211508553,
      "grad_norm": 0.437914103269577,
      "learning_rate": 4.7475e-05,
      "log_odds_chosen": -0.055506929755210876,
      "log_odds_ratio": -0.7757474184036255,
      "logits/chosen": 0.13743720948696136,
      "logits/rejected": 0.028182677924633026,
      "logps/chosen": -1.2761411666870117,
      "logps/rejected": -1.158294677734375,
      "loss": 3.2391,
      "nll_loss": 3.161520481109619,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1276141107082367,
      "rewards/margins": -0.01178464200347662,
      "rewards/rejected": -0.1158294752240181,
      "step": 101
    },
    {
      "epoch": 0.06345256609642301,
      "grad_norm": 0.4476625919342041,
      "learning_rate": 4.745e-05,
      "log_odds_chosen": 0.2342412918806076,
      "log_odds_ratio": -0.6413729786872864,
      "logits/chosen": 0.168337881565094,
      "logits/rejected": -0.14545048773288727,
      "logps/chosen": -1.2778122425079346,
      "logps/rejected": -1.4140448570251465,
      "loss": 3.7524,
      "nll_loss": 3.6882784366607666,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1277812272310257,
      "rewards/margins": 0.013623261824250221,
      "rewards/rejected": -0.14140449464321136,
      "step": 102
    },
    {
      "epoch": 0.0640746500777605,
      "grad_norm": 0.5129625201225281,
      "learning_rate": 4.7425e-05,
      "log_odds_chosen": 0.08128762245178223,
      "log_odds_ratio": -0.6819002628326416,
      "logits/chosen": 0.10714941471815109,
      "logits/rejected": -0.07025566697120667,
      "logps/chosen": -1.333043098449707,
      "logps/rejected": -1.3708577156066895,
      "loss": 3.0904,
      "nll_loss": 3.0221877098083496,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13330431282520294,
      "rewards/margins": 0.0037814658135175705,
      "rewards/rejected": -0.13708576560020447,
      "step": 103
    },
    {
      "epoch": 0.06469673405909798,
      "grad_norm": 0.5524664521217346,
      "learning_rate": 4.74e-05,
      "log_odds_chosen": -0.28195393085479736,
      "log_odds_ratio": -0.8670526742935181,
      "logits/chosen": -0.04569420963525772,
      "logits/rejected": -0.1721382588148117,
      "logps/chosen": -1.4118688106536865,
      "logps/rejected": -1.197587490081787,
      "loss": 2.7953,
      "nll_loss": 2.7085652351379395,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1411868780851364,
      "rewards/margins": -0.021428123116493225,
      "rewards/rejected": -0.11975875496864319,
      "step": 104
    },
    {
      "epoch": 0.06531881804043546,
      "grad_norm": 0.4822681248188019,
      "learning_rate": 4.7375e-05,
      "log_odds_chosen": 0.38326454162597656,
      "log_odds_ratio": -0.5975579619407654,
      "logits/chosen": -0.007261446211487055,
      "logits/rejected": -0.030812345445156097,
      "logps/chosen": -1.179614782333374,
      "logps/rejected": -1.4915132522583008,
      "loss": 2.6517,
      "nll_loss": 2.5919699668884277,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11796149611473083,
      "rewards/margins": 0.031189851462841034,
      "rewards/rejected": -0.14915132522583008,
      "step": 105
    },
    {
      "epoch": 0.06594090202177294,
      "grad_norm": 0.5283992886543274,
      "learning_rate": 4.735e-05,
      "log_odds_chosen": 0.45969775319099426,
      "log_odds_ratio": -0.5704058408737183,
      "logits/chosen": -0.005958788096904755,
      "logits/rejected": -0.1403818130493164,
      "logps/chosen": -1.1262407302856445,
      "logps/rejected": -1.365699291229248,
      "loss": 3.3859,
      "nll_loss": 3.328857183456421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11262409389019012,
      "rewards/margins": 0.023945845663547516,
      "rewards/rejected": -0.13656993210315704,
      "step": 106
    },
    {
      "epoch": 0.06656298600311042,
      "grad_norm": 0.416676789522171,
      "learning_rate": 4.7325000000000005e-05,
      "log_odds_chosen": 0.1313173770904541,
      "log_odds_ratio": -0.6711844205856323,
      "logits/chosen": 0.157118558883667,
      "logits/rejected": 0.035575054585933685,
      "logps/chosen": -1.4387624263763428,
      "logps/rejected": -1.5632802248001099,
      "loss": 3.2781,
      "nll_loss": 3.210995674133301,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14387625455856323,
      "rewards/margins": 0.012451781891286373,
      "rewards/rejected": -0.156328022480011,
      "step": 107
    },
    {
      "epoch": 0.0671850699844479,
      "grad_norm": 0.5572922229766846,
      "learning_rate": 4.73e-05,
      "log_odds_chosen": -0.12381504476070404,
      "log_odds_ratio": -0.770376443862915,
      "logits/chosen": 0.04753262549638748,
      "logits/rejected": -0.051002245396375656,
      "logps/chosen": -1.5710325241088867,
      "logps/rejected": -1.46817147731781,
      "loss": 2.8265,
      "nll_loss": 2.749462127685547,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.15710324048995972,
      "rewards/margins": -0.010286085307598114,
      "rewards/rejected": -0.146817147731781,
      "step": 108
    },
    {
      "epoch": 0.06780715396578538,
      "grad_norm": 0.3513568937778473,
      "learning_rate": 4.7275000000000004e-05,
      "log_odds_chosen": 0.05175573006272316,
      "log_odds_ratio": -0.6735912561416626,
      "logits/chosen": 0.08347096294164658,
      "logits/rejected": 0.05597352236509323,
      "logps/chosen": -1.2763020992279053,
      "logps/rejected": -1.3162968158721924,
      "loss": 2.8374,
      "nll_loss": 2.77008056640625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12763021886348724,
      "rewards/margins": 0.0039994558319449425,
      "rewards/rejected": -0.13162967562675476,
      "step": 109
    },
    {
      "epoch": 0.06842923794712286,
      "grad_norm": 0.48008182644844055,
      "learning_rate": 4.7249999999999997e-05,
      "log_odds_chosen": 0.2633849084377289,
      "log_odds_ratio": -0.5829953551292419,
      "logits/chosen": 0.0013431366533041,
      "logits/rejected": -0.17736227810382843,
      "logps/chosen": -1.2399024963378906,
      "logps/rejected": -1.4264099597930908,
      "loss": 2.5525,
      "nll_loss": 2.4942123889923096,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1239902600646019,
      "rewards/margins": 0.018650764599442482,
      "rewards/rejected": -0.14264102280139923,
      "step": 110
    },
    {
      "epoch": 0.06905132192846034,
      "grad_norm": 0.4274824857711792,
      "learning_rate": 4.7225e-05,
      "log_odds_chosen": -0.1062813401222229,
      "log_odds_ratio": -0.809943675994873,
      "logits/chosen": 0.17697763442993164,
      "logits/rejected": -0.11846765875816345,
      "logps/chosen": -1.3306231498718262,
      "logps/rejected": -1.2187504768371582,
      "loss": 3.1534,
      "nll_loss": 3.0723817348480225,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13306231796741486,
      "rewards/margins": -0.011187261901795864,
      "rewards/rejected": -0.12187506258487701,
      "step": 111
    },
    {
      "epoch": 0.06967340590979783,
      "grad_norm": 0.44988536834716797,
      "learning_rate": 4.72e-05,
      "log_odds_chosen": 0.298068106174469,
      "log_odds_ratio": -0.5905066132545471,
      "logits/chosen": 0.060463353991508484,
      "logits/rejected": -0.10935623943805695,
      "logps/chosen": -1.1479036808013916,
      "logps/rejected": -1.342679738998413,
      "loss": 2.709,
      "nll_loss": 2.6499881744384766,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11479037255048752,
      "rewards/margins": 0.01947760209441185,
      "rewards/rejected": -0.13426797091960907,
      "step": 112
    },
    {
      "epoch": 0.07029548989113531,
      "grad_norm": 0.4988582134246826,
      "learning_rate": 4.7175e-05,
      "log_odds_chosen": -0.08553829789161682,
      "log_odds_ratio": -0.8603007793426514,
      "logits/chosen": 0.39943423867225647,
      "logits/rejected": 0.06061486154794693,
      "logps/chosen": -1.4535963535308838,
      "logps/rejected": -1.3087565898895264,
      "loss": 3.8892,
      "nll_loss": 3.8031439781188965,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14535963535308838,
      "rewards/margins": -0.014483977109193802,
      "rewards/rejected": -0.13087564706802368,
      "step": 113
    },
    {
      "epoch": 0.07091757387247279,
      "grad_norm": 0.46725013852119446,
      "learning_rate": 4.715e-05,
      "log_odds_chosen": 0.641747772693634,
      "log_odds_ratio": -0.5314986705780029,
      "logits/chosen": 0.09639215469360352,
      "logits/rejected": -0.023075614124536514,
      "logps/chosen": -0.903424859046936,
      "logps/rejected": -1.2310090065002441,
      "loss": 3.0129,
      "nll_loss": 2.9597768783569336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09034248441457748,
      "rewards/margins": 0.032758425921201706,
      "rewards/rejected": -0.12310090661048889,
      "step": 114
    },
    {
      "epoch": 0.07153965785381027,
      "grad_norm": 0.4860602617263794,
      "learning_rate": 4.7125e-05,
      "log_odds_chosen": -0.0031689107418060303,
      "log_odds_ratio": -0.7295066118240356,
      "logits/chosen": 0.21932338178157806,
      "logits/rejected": 0.0026386789977550507,
      "logps/chosen": -1.5008599758148193,
      "logps/rejected": -1.5008916854858398,
      "loss": 3.3598,
      "nll_loss": 3.2868006229400635,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.15008598566055298,
      "rewards/margins": 3.1907111406326294e-06,
      "rewards/rejected": -0.15008917450904846,
      "step": 115
    },
    {
      "epoch": 0.07216174183514774,
      "grad_norm": 0.8297399282455444,
      "learning_rate": 4.71e-05,
      "log_odds_chosen": -0.01504303514957428,
      "log_odds_ratio": -0.7895686030387878,
      "logits/chosen": 0.08432838320732117,
      "logits/rejected": -0.08494340628385544,
      "logps/chosen": -1.3483591079711914,
      "logps/rejected": -1.3111402988433838,
      "loss": 3.046,
      "nll_loss": 2.9669947624206543,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13483591377735138,
      "rewards/margins": -0.0037218770012259483,
      "rewards/rejected": -0.13111403584480286,
      "step": 116
    },
    {
      "epoch": 0.07278382581648522,
      "grad_norm": 0.4088709354400635,
      "learning_rate": 4.7075e-05,
      "log_odds_chosen": 0.20638790726661682,
      "log_odds_ratio": -0.6793928146362305,
      "logits/chosen": 0.2052862048149109,
      "logits/rejected": 0.044292062520980835,
      "logps/chosen": -1.2724294662475586,
      "logps/rejected": -1.4322636127471924,
      "loss": 3.539,
      "nll_loss": 3.471055030822754,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12724295258522034,
      "rewards/margins": 0.01598340831696987,
      "rewards/rejected": -0.14322635531425476,
      "step": 117
    },
    {
      "epoch": 0.0734059097978227,
      "grad_norm": 0.42515477538108826,
      "learning_rate": 4.705e-05,
      "log_odds_chosen": -0.019400358200073242,
      "log_odds_ratio": -0.7594976425170898,
      "logits/chosen": 0.27322930097579956,
      "logits/rejected": 0.07737819850444794,
      "logps/chosen": -1.2583709955215454,
      "logps/rejected": -1.2143146991729736,
      "loss": 3.5821,
      "nll_loss": 3.5061068534851074,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12583710253238678,
      "rewards/margins": -0.004405634477734566,
      "rewards/rejected": -0.12143146991729736,
      "step": 118
    },
    {
      "epoch": 0.07402799377916018,
      "grad_norm": 0.5038117170333862,
      "learning_rate": 4.7025000000000005e-05,
      "log_odds_chosen": 0.38098299503326416,
      "log_odds_ratio": -0.5279546976089478,
      "logits/chosen": 0.15353046357631683,
      "logits/rejected": -0.20058253407478333,
      "logps/chosen": -1.2580475807189941,
      "logps/rejected": -1.5402312278747559,
      "loss": 3.1463,
      "nll_loss": 3.0934572219848633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12580475211143494,
      "rewards/margins": 0.028218373656272888,
      "rewards/rejected": -0.15402314066886902,
      "step": 119
    },
    {
      "epoch": 0.07465007776049767,
      "grad_norm": 0.4213177561759949,
      "learning_rate": 4.7e-05,
      "log_odds_chosen": 0.2627849876880646,
      "log_odds_ratio": -0.6001940965652466,
      "logits/chosen": 0.3041577935218811,
      "logits/rejected": -0.03829241171479225,
      "logps/chosen": -1.1501599550247192,
      "logps/rejected": -1.3457529544830322,
      "loss": 3.6689,
      "nll_loss": 3.6088767051696777,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11501598358154297,
      "rewards/margins": 0.019559307023882866,
      "rewards/rejected": -0.13457529246807098,
      "step": 120
    },
    {
      "epoch": 0.07527216174183515,
      "grad_norm": 0.5030676126480103,
      "learning_rate": 4.6975000000000003e-05,
      "log_odds_chosen": 0.1841212958097458,
      "log_odds_ratio": -0.6212969422340393,
      "logits/chosen": 0.10191065073013306,
      "logits/rejected": 0.07480275630950928,
      "logps/chosen": -1.2450635433197021,
      "logps/rejected": -1.3654481172561646,
      "loss": 2.7331,
      "nll_loss": 2.670966625213623,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12450635433197021,
      "rewards/margins": 0.012038461863994598,
      "rewards/rejected": -0.13654480874538422,
      "step": 121
    },
    {
      "epoch": 0.07589424572317263,
      "grad_norm": 0.47793930768966675,
      "learning_rate": 4.695e-05,
      "log_odds_chosen": 0.22587494552135468,
      "log_odds_ratio": -0.6521732211112976,
      "logits/chosen": 0.14298784732818604,
      "logits/rejected": 0.07297507673501968,
      "logps/chosen": -1.1125988960266113,
      "logps/rejected": -1.2532191276550293,
      "loss": 3.0589,
      "nll_loss": 2.993687391281128,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11125989258289337,
      "rewards/margins": 0.014062018133699894,
      "rewards/rejected": -0.12532192468643188,
      "step": 122
    },
    {
      "epoch": 0.07651632970451011,
      "grad_norm": 0.585175096988678,
      "learning_rate": 4.6925e-05,
      "log_odds_chosen": 0.2622927725315094,
      "log_odds_ratio": -0.598949670791626,
      "logits/chosen": 0.06829071789979935,
      "logits/rejected": -0.06984008848667145,
      "logps/chosen": -1.279144287109375,
      "logps/rejected": -1.4879951477050781,
      "loss": 2.6114,
      "nll_loss": 2.5515384674072266,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1279144287109375,
      "rewards/margins": 0.020885098725557327,
      "rewards/rejected": -0.14879952371120453,
      "step": 123
    },
    {
      "epoch": 0.07713841368584759,
      "grad_norm": 0.5636870861053467,
      "learning_rate": 4.69e-05,
      "log_odds_chosen": 0.9784868955612183,
      "log_odds_ratio": -0.40459758043289185,
      "logits/chosen": 0.18325799703598022,
      "logits/rejected": -0.04458653926849365,
      "logps/chosen": -0.9027924537658691,
      "logps/rejected": -1.5085422992706299,
      "loss": 3.4475,
      "nll_loss": 3.4070000648498535,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09027925133705139,
      "rewards/margins": 0.060574986040592194,
      "rewards/rejected": -0.150854229927063,
      "step": 124
    },
    {
      "epoch": 0.07776049766718507,
      "grad_norm": 0.5631504654884338,
      "learning_rate": 4.6875e-05,
      "log_odds_chosen": 0.8957180976867676,
      "log_odds_ratio": -0.46610015630722046,
      "logits/chosen": 0.13964581489562988,
      "logits/rejected": 0.029128514230251312,
      "logps/chosen": -0.8781135678291321,
      "logps/rejected": -1.3816094398498535,
      "loss": 2.6724,
      "nll_loss": 2.6258134841918945,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08781135827302933,
      "rewards/margins": 0.05034959316253662,
      "rewards/rejected": -0.13816094398498535,
      "step": 125
    },
    {
      "epoch": 0.07838258164852255,
      "grad_norm": 0.3702596426010132,
      "learning_rate": 4.685000000000001e-05,
      "log_odds_chosen": 0.15894243121147156,
      "log_odds_ratio": -0.6334323883056641,
      "logits/chosen": 0.2041868418455124,
      "logits/rejected": 0.06562945991754532,
      "logps/chosen": -1.257232427597046,
      "logps/rejected": -1.3767375946044922,
      "loss": 3.3947,
      "nll_loss": 3.3313791751861572,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12572325766086578,
      "rewards/margins": 0.011950517073273659,
      "rewards/rejected": -0.1376737654209137,
      "step": 126
    },
    {
      "epoch": 0.07900466562986003,
      "grad_norm": 0.43313437700271606,
      "learning_rate": 4.6825e-05,
      "log_odds_chosen": 0.21152283251285553,
      "log_odds_ratio": -0.6254419684410095,
      "logits/chosen": -0.06763223558664322,
      "logits/rejected": -0.05511503666639328,
      "logps/chosen": -1.2304375171661377,
      "logps/rejected": -1.3703246116638184,
      "loss": 2.5647,
      "nll_loss": 2.502199411392212,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12304375320672989,
      "rewards/margins": 0.013988707214593887,
      "rewards/rejected": -0.13703244924545288,
      "step": 127
    },
    {
      "epoch": 0.0796267496111975,
      "grad_norm": 0.4836374521255493,
      "learning_rate": 4.6800000000000006e-05,
      "log_odds_chosen": -0.18248885869979858,
      "log_odds_ratio": -0.8107894659042358,
      "logits/chosen": 0.10594216734170914,
      "logits/rejected": -0.056509390473365784,
      "logps/chosen": -1.2352960109710693,
      "logps/rejected": -1.1191729307174683,
      "loss": 3.1493,
      "nll_loss": 3.0681915283203125,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1235295981168747,
      "rewards/margins": -0.011612308211624622,
      "rewards/rejected": -0.11191728711128235,
      "step": 128
    },
    {
      "epoch": 0.080248833592535,
      "grad_norm": 0.41119441390037537,
      "learning_rate": 4.6775000000000005e-05,
      "log_odds_chosen": 0.48518961668014526,
      "log_odds_ratio": -0.49193328619003296,
      "logits/chosen": 0.09460698068141937,
      "logits/rejected": 0.13982588052749634,
      "logps/chosen": -1.2239277362823486,
      "logps/rejected": -1.5707978010177612,
      "loss": 2.8458,
      "nll_loss": 2.796567916870117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12239278107881546,
      "rewards/margins": 0.03468699753284454,
      "rewards/rejected": -0.1570797860622406,
      "step": 129
    },
    {
      "epoch": 0.08087091757387248,
      "grad_norm": 0.5971326231956482,
      "learning_rate": 4.6750000000000005e-05,
      "log_odds_chosen": 0.3527142405509949,
      "log_odds_ratio": -0.6361883878707886,
      "logits/chosen": 0.3607098460197449,
      "logits/rejected": -0.14897310733795166,
      "logps/chosen": -1.3266918659210205,
      "logps/rejected": -1.5898045301437378,
      "loss": 3.5671,
      "nll_loss": 3.5034427642822266,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13266919553279877,
      "rewards/margins": 0.02631126344203949,
      "rewards/rejected": -0.15898045897483826,
      "step": 130
    },
    {
      "epoch": 0.08149300155520996,
      "grad_norm": 0.33660298585891724,
      "learning_rate": 4.6725000000000004e-05,
      "log_odds_chosen": 0.866208553314209,
      "log_odds_ratio": -0.4489937126636505,
      "logits/chosen": 0.11533968150615692,
      "logits/rejected": 0.0789806991815567,
      "logps/chosen": -0.9988769888877869,
      "logps/rejected": -1.605279803276062,
      "loss": 2.9545,
      "nll_loss": 2.9095656871795654,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09988771378993988,
      "rewards/margins": 0.06064027547836304,
      "rewards/rejected": -0.16052797436714172,
      "step": 131
    },
    {
      "epoch": 0.08211508553654744,
      "grad_norm": 0.31415626406669617,
      "learning_rate": 4.6700000000000003e-05,
      "log_odds_chosen": 0.2750133275985718,
      "log_odds_ratio": -0.668391227722168,
      "logits/chosen": 0.2666417360305786,
      "logits/rejected": -0.017590831965208054,
      "logps/chosen": -1.0113110542297363,
      "logps/rejected": -1.1601898670196533,
      "loss": 3.5387,
      "nll_loss": 3.471817970275879,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10113111138343811,
      "rewards/margins": 0.014887874014675617,
      "rewards/rejected": -0.11601898819208145,
      "step": 132
    },
    {
      "epoch": 0.08273716951788491,
      "grad_norm": 0.5030128955841064,
      "learning_rate": 4.6675e-05,
      "log_odds_chosen": 1.185779094696045,
      "log_odds_ratio": -0.3972381353378296,
      "logits/chosen": 0.14357468485832214,
      "logits/rejected": 0.00014878623187541962,
      "logps/chosen": -1.0277682542800903,
      "logps/rejected": -1.820441722869873,
      "loss": 3.2187,
      "nll_loss": 3.178988456726074,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10277682542800903,
      "rewards/margins": 0.07926735281944275,
      "rewards/rejected": -0.1820441633462906,
      "step": 133
    },
    {
      "epoch": 0.0833592534992224,
      "grad_norm": 0.4640537202358246,
      "learning_rate": 4.665e-05,
      "log_odds_chosen": -0.1519235223531723,
      "log_odds_ratio": -0.8262673616409302,
      "logits/chosen": 0.11444417387247086,
      "logits/rejected": -0.012055326253175735,
      "logps/chosen": -1.376497745513916,
      "logps/rejected": -1.2670477628707886,
      "loss": 2.9043,
      "nll_loss": 2.821722984313965,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1376497745513916,
      "rewards/margins": -0.010944999754428864,
      "rewards/rejected": -0.12670478224754333,
      "step": 134
    },
    {
      "epoch": 0.08398133748055987,
      "grad_norm": 0.4529246389865875,
      "learning_rate": 4.6625e-05,
      "log_odds_chosen": 0.1725572943687439,
      "log_odds_ratio": -0.6657087206840515,
      "logits/chosen": 0.08027800917625427,
      "logits/rejected": -0.09482064098119736,
      "logps/chosen": -1.1149572134017944,
      "logps/rejected": -1.2249038219451904,
      "loss": 2.7318,
      "nll_loss": 2.6652512550354004,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1114957183599472,
      "rewards/margins": 0.01099465787410736,
      "rewards/rejected": -0.12249037623405457,
      "step": 135
    },
    {
      "epoch": 0.08460342146189735,
      "grad_norm": 0.4171338677406311,
      "learning_rate": 4.660000000000001e-05,
      "log_odds_chosen": -0.09017062187194824,
      "log_odds_ratio": -0.7511317133903503,
      "logits/chosen": 0.2360968291759491,
      "logits/rejected": -0.010965634137392044,
      "logps/chosen": -1.1789023876190186,
      "logps/rejected": -1.0944397449493408,
      "loss": 3.3572,
      "nll_loss": 3.2820589542388916,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.11789023876190186,
      "rewards/margins": -0.008446259424090385,
      "rewards/rejected": -0.10944397747516632,
      "step": 136
    },
    {
      "epoch": 0.08522550544323483,
      "grad_norm": 0.547868013381958,
      "learning_rate": 4.6575e-05,
      "log_odds_chosen": 0.7603722810745239,
      "log_odds_ratio": -0.49443089962005615,
      "logits/chosen": -0.03645198792219162,
      "logits/rejected": -0.07120641320943832,
      "logps/chosen": -1.1619699001312256,
      "logps/rejected": -1.7555570602416992,
      "loss": 2.7566,
      "nll_loss": 2.707111120223999,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11619699001312256,
      "rewards/margins": 0.05935873091220856,
      "rewards/rejected": -0.17555572092533112,
      "step": 137
    },
    {
      "epoch": 0.08584758942457232,
      "grad_norm": 0.3483281433582306,
      "learning_rate": 4.655000000000001e-05,
      "log_odds_chosen": 0.3946763873100281,
      "log_odds_ratio": -0.6060354113578796,
      "logits/chosen": 0.09304636716842651,
      "logits/rejected": 0.01444307342171669,
      "logps/chosen": -1.247818946838379,
      "logps/rejected": -1.4207770824432373,
      "loss": 3.0866,
      "nll_loss": 3.025946855545044,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12478190660476685,
      "rewards/margins": 0.017295803874731064,
      "rewards/rejected": -0.1420777142047882,
      "step": 138
    },
    {
      "epoch": 0.0864696734059098,
      "grad_norm": 0.4377744495868683,
      "learning_rate": 4.6525e-05,
      "log_odds_chosen": 0.5676741003990173,
      "log_odds_ratio": -0.4999235272407532,
      "logits/chosen": 0.18979693949222565,
      "logits/rejected": -0.09224162995815277,
      "logps/chosen": -1.0275253057479858,
      "logps/rejected": -1.4091662168502808,
      "loss": 3.1186,
      "nll_loss": 3.068612575531006,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10275252908468246,
      "rewards/margins": 0.03816410154104233,
      "rewards/rejected": -0.140916645526886,
      "step": 139
    },
    {
      "epoch": 0.08709175738724728,
      "grad_norm": 0.5015956163406372,
      "learning_rate": 4.6500000000000005e-05,
      "log_odds_chosen": 0.40913963317871094,
      "log_odds_ratio": -0.5399808883666992,
      "logits/chosen": 0.13208572566509247,
      "logits/rejected": -0.042870864272117615,
      "logps/chosen": -1.4589378833770752,
      "logps/rejected": -1.76136314868927,
      "loss": 2.8749,
      "nll_loss": 2.8209028244018555,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14589379727840424,
      "rewards/margins": 0.03024252876639366,
      "rewards/rejected": -0.176136314868927,
      "step": 140
    },
    {
      "epoch": 0.08771384136858476,
      "grad_norm": 0.5315511226654053,
      "learning_rate": 4.6475000000000005e-05,
      "log_odds_chosen": 0.23443233966827393,
      "log_odds_ratio": -0.6079375743865967,
      "logits/chosen": 0.10196913778781891,
      "logits/rejected": -0.09439704567193985,
      "logps/chosen": -1.2660562992095947,
      "logps/rejected": -1.440688133239746,
      "loss": 2.9088,
      "nll_loss": 2.8479607105255127,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12660562992095947,
      "rewards/margins": 0.01746317744255066,
      "rewards/rejected": -0.14406880736351013,
      "step": 141
    },
    {
      "epoch": 0.08833592534992224,
      "grad_norm": 0.48670142889022827,
      "learning_rate": 4.6450000000000004e-05,
      "log_odds_chosen": 0.20135235786437988,
      "log_odds_ratio": -0.6504536271095276,
      "logits/chosen": 0.057795051485300064,
      "logits/rejected": -0.16002613306045532,
      "logps/chosen": -1.202336072921753,
      "logps/rejected": -1.336153268814087,
      "loss": 2.9102,
      "nll_loss": 2.8451108932495117,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12023360282182693,
      "rewards/margins": 0.01338173821568489,
      "rewards/rejected": -0.13361534476280212,
      "step": 142
    },
    {
      "epoch": 0.08895800933125972,
      "grad_norm": 0.4415438771247864,
      "learning_rate": 4.6425000000000004e-05,
      "log_odds_chosen": 0.61316978931427,
      "log_odds_ratio": -0.4622770845890045,
      "logits/chosen": 0.04670334979891777,
      "logits/rejected": 0.007671605795621872,
      "logps/chosen": -1.3000093698501587,
      "logps/rejected": -1.7552378177642822,
      "loss": 2.9155,
      "nll_loss": 2.869236707687378,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13000094890594482,
      "rewards/margins": 0.045522838830947876,
      "rewards/rejected": -0.1755237877368927,
      "step": 143
    },
    {
      "epoch": 0.0895800933125972,
      "grad_norm": 0.5169774889945984,
      "learning_rate": 4.64e-05,
      "log_odds_chosen": 0.6011465787887573,
      "log_odds_ratio": -0.5140647292137146,
      "logits/chosen": 0.007771043106913567,
      "logits/rejected": -0.15149809420108795,
      "logps/chosen": -1.184515118598938,
      "logps/rejected": -1.645982027053833,
      "loss": 2.7975,
      "nll_loss": 2.7460548877716064,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11845151335000992,
      "rewards/margins": 0.0461466908454895,
      "rewards/rejected": -0.16459819674491882,
      "step": 144
    },
    {
      "epoch": 0.09020217729393468,
      "grad_norm": 0.6221879720687866,
      "learning_rate": 4.6375e-05,
      "log_odds_chosen": 0.6371638774871826,
      "log_odds_ratio": -0.5189849138259888,
      "logits/chosen": 0.051749397069215775,
      "logits/rejected": -0.1522921919822693,
      "logps/chosen": -0.8362572193145752,
      "logps/rejected": -1.1811851263046265,
      "loss": 3.1062,
      "nll_loss": 3.0543196201324463,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08362571895122528,
      "rewards/margins": 0.034492794424295425,
      "rewards/rejected": -0.1181185245513916,
      "step": 145
    },
    {
      "epoch": 0.09082426127527216,
      "grad_norm": 0.4802875518798828,
      "learning_rate": 4.635e-05,
      "log_odds_chosen": 0.3668875992298126,
      "log_odds_ratio": -0.5806620717048645,
      "logits/chosen": 0.10094377398490906,
      "logits/rejected": -0.02433057874441147,
      "logps/chosen": -1.2760828733444214,
      "logps/rejected": -1.5276360511779785,
      "loss": 2.9177,
      "nll_loss": 2.859609842300415,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1276082843542099,
      "rewards/margins": 0.02515532448887825,
      "rewards/rejected": -0.15276360511779785,
      "step": 146
    },
    {
      "epoch": 0.09144634525660965,
      "grad_norm": 0.4567488133907318,
      "learning_rate": 4.6325e-05,
      "log_odds_chosen": 0.6026850938796997,
      "log_odds_ratio": -0.4573545753955841,
      "logits/chosen": 0.19558559358119965,
      "logits/rejected": 0.03107052482664585,
      "logps/chosen": -1.185868263244629,
      "logps/rejected": -1.6299797296524048,
      "loss": 3.3654,
      "nll_loss": 3.3196253776550293,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11858682334423065,
      "rewards/margins": 0.044411152601242065,
      "rewards/rejected": -0.16299797594547272,
      "step": 147
    },
    {
      "epoch": 0.09206842923794713,
      "grad_norm": 0.5302126407623291,
      "learning_rate": 4.630000000000001e-05,
      "log_odds_chosen": -0.01642867922782898,
      "log_odds_ratio": -0.7874202728271484,
      "logits/chosen": 0.11570745706558228,
      "logits/rejected": -0.1371677815914154,
      "logps/chosen": -1.3408689498901367,
      "logps/rejected": -1.3910057544708252,
      "loss": 3.5827,
      "nll_loss": 3.5039186477661133,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13408689200878143,
      "rewards/margins": 0.005013669840991497,
      "rewards/rejected": -0.1391005516052246,
      "step": 148
    },
    {
      "epoch": 0.0926905132192846,
      "grad_norm": 0.4184373617172241,
      "learning_rate": 4.6275e-05,
      "log_odds_chosen": 0.05463185906410217,
      "log_odds_ratio": -0.7645725607872009,
      "logits/chosen": 0.1671661138534546,
      "logits/rejected": 0.034170206636190414,
      "logps/chosen": -1.3616355657577515,
      "logps/rejected": -1.3897101879119873,
      "loss": 3.3596,
      "nll_loss": 3.283134698867798,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13616356253623962,
      "rewards/margins": 0.002807457000017166,
      "rewards/rejected": -0.13897103071212769,
      "step": 149
    },
    {
      "epoch": 0.09331259720062209,
      "grad_norm": 0.42127570509910583,
      "learning_rate": 4.6250000000000006e-05,
      "log_odds_chosen": 0.08022980391979218,
      "log_odds_ratio": -0.7030846476554871,
      "logits/chosen": 0.0035371780395507812,
      "logits/rejected": 0.003839358687400818,
      "logps/chosen": -1.3736772537231445,
      "logps/rejected": -1.378920078277588,
      "loss": 2.8468,
      "nll_loss": 2.7764601707458496,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13736772537231445,
      "rewards/margins": 0.0005242684856057167,
      "rewards/rejected": -0.1378920078277588,
      "step": 150
    },
    {
      "epoch": 0.09393468118195956,
      "grad_norm": 0.4007570147514343,
      "learning_rate": 4.6225e-05,
      "log_odds_chosen": 0.06232370063662529,
      "log_odds_ratio": -0.6894880533218384,
      "logits/chosen": 0.028553307056427002,
      "logits/rejected": 0.13209117949008942,
      "logps/chosen": -1.5964021682739258,
      "logps/rejected": -1.6478145122528076,
      "loss": 2.7767,
      "nll_loss": 2.707792282104492,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15964022278785706,
      "rewards/margins": 0.005141226574778557,
      "rewards/rejected": -0.16478145122528076,
      "step": 151
    },
    {
      "epoch": 0.09455676516329704,
      "grad_norm": 0.506729006767273,
      "learning_rate": 4.6200000000000005e-05,
      "log_odds_chosen": 0.44813084602355957,
      "log_odds_ratio": -0.6032027006149292,
      "logits/chosen": 0.19062553346157074,
      "logits/rejected": -0.01674717850983143,
      "logps/chosen": -1.1239427328109741,
      "logps/rejected": -1.383901834487915,
      "loss": 3.1411,
      "nll_loss": 3.0807762145996094,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11239427328109741,
      "rewards/margins": 0.025995904579758644,
      "rewards/rejected": -0.1383901834487915,
      "step": 152
    },
    {
      "epoch": 0.09517884914463452,
      "grad_norm": 0.4609549641609192,
      "learning_rate": 4.6175000000000004e-05,
      "log_odds_chosen": 0.4586949646472931,
      "log_odds_ratio": -0.5538212060928345,
      "logits/chosen": 0.25490209460258484,
      "logits/rejected": 0.08342747390270233,
      "logps/chosen": -1.3676748275756836,
      "logps/rejected": -1.6974760293960571,
      "loss": 3.281,
      "nll_loss": 3.2256217002868652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13676749169826508,
      "rewards/margins": 0.032980117946863174,
      "rewards/rejected": -0.16974762082099915,
      "step": 153
    },
    {
      "epoch": 0.095800933125972,
      "grad_norm": 0.3721259832382202,
      "learning_rate": 4.6150000000000004e-05,
      "log_odds_chosen": 0.5167128443717957,
      "log_odds_ratio": -0.5634645223617554,
      "logits/chosen": 0.06376560777425766,
      "logits/rejected": -0.0771709680557251,
      "logps/chosen": -1.1290371417999268,
      "logps/rejected": -1.432267189025879,
      "loss": 2.8284,
      "nll_loss": 2.7720093727111816,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11290371417999268,
      "rewards/margins": 0.03032299503684044,
      "rewards/rejected": -0.1432267129421234,
      "step": 154
    },
    {
      "epoch": 0.09642301710730948,
      "grad_norm": 0.43300989270210266,
      "learning_rate": 4.6125e-05,
      "log_odds_chosen": 0.20810005068778992,
      "log_odds_ratio": -0.7468218207359314,
      "logits/chosen": 0.2738891839981079,
      "logits/rejected": 0.0035995468497276306,
      "logps/chosen": -1.1184593439102173,
      "logps/rejected": -1.2356008291244507,
      "loss": 3.6973,
      "nll_loss": 3.6226534843444824,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1118459403514862,
      "rewards/margins": 0.011714148335158825,
      "rewards/rejected": -0.12356008589267731,
      "step": 155
    },
    {
      "epoch": 0.09704510108864697,
      "grad_norm": 0.4422686994075775,
      "learning_rate": 4.61e-05,
      "log_odds_chosen": 0.38565486669540405,
      "log_odds_ratio": -0.5663627982139587,
      "logits/chosen": 0.1406802535057068,
      "logits/rejected": -0.10811451077461243,
      "logps/chosen": -1.2533131837844849,
      "logps/rejected": -1.501025915145874,
      "loss": 2.9378,
      "nll_loss": 2.881122589111328,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.125331312417984,
      "rewards/margins": 0.02477126568555832,
      "rewards/rejected": -0.15010258555412292,
      "step": 156
    },
    {
      "epoch": 0.09766718506998445,
      "grad_norm": 0.43218180537223816,
      "learning_rate": 4.6075e-05,
      "log_odds_chosen": 0.7944599390029907,
      "log_odds_ratio": -0.4741981029510498,
      "logits/chosen": 0.13778428733348846,
      "logits/rejected": 0.048661183565855026,
      "logps/chosen": -1.2979130744934082,
      "logps/rejected": -1.8968315124511719,
      "loss": 3.1609,
      "nll_loss": 3.1134564876556396,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12979131937026978,
      "rewards/margins": 0.05989184230566025,
      "rewards/rejected": -0.18968316912651062,
      "step": 157
    },
    {
      "epoch": 0.09828926905132193,
      "grad_norm": 0.3956831991672516,
      "learning_rate": 4.605e-05,
      "log_odds_chosen": 0.27946868538856506,
      "log_odds_ratio": -0.6237690448760986,
      "logits/chosen": 0.3799353837966919,
      "logits/rejected": 0.11462019383907318,
      "logps/chosen": -1.2597386837005615,
      "logps/rejected": -1.4242260456085205,
      "loss": 3.4372,
      "nll_loss": 3.374814987182617,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1259738653898239,
      "rewards/margins": 0.016448739916086197,
      "rewards/rejected": -0.142422616481781,
      "step": 158
    },
    {
      "epoch": 0.09891135303265941,
      "grad_norm": 0.40490007400512695,
      "learning_rate": 4.6025e-05,
      "log_odds_chosen": 1.3628696203231812,
      "log_odds_ratio": -0.38135695457458496,
      "logits/chosen": 0.16640058159828186,
      "logits/rejected": 0.0424039289355278,
      "logps/chosen": -0.7394822835922241,
      "logps/rejected": -1.6354302167892456,
      "loss": 3.0489,
      "nll_loss": 3.010765552520752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07394823431968689,
      "rewards/margins": 0.08959478884935379,
      "rewards/rejected": -0.16354301571846008,
      "step": 159
    },
    {
      "epoch": 0.09953343701399689,
      "grad_norm": 0.522266685962677,
      "learning_rate": 4.600000000000001e-05,
      "log_odds_chosen": 0.31891077756881714,
      "log_odds_ratio": -0.6262969374656677,
      "logits/chosen": 0.2351851910352707,
      "logits/rejected": 0.0422259159386158,
      "logps/chosen": -1.1578633785247803,
      "logps/rejected": -1.3393428325653076,
      "loss": 2.8211,
      "nll_loss": 2.7584638595581055,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11578632891178131,
      "rewards/margins": 0.018147945404052734,
      "rewards/rejected": -0.13393428921699524,
      "step": 160
    },
    {
      "epoch": 0.10015552099533437,
      "grad_norm": 0.4992579221725464,
      "learning_rate": 4.5975e-05,
      "log_odds_chosen": 0.36979615688323975,
      "log_odds_ratio": -0.5988181829452515,
      "logits/chosen": 0.2722373604774475,
      "logits/rejected": 0.13797958195209503,
      "logps/chosen": -1.223292589187622,
      "logps/rejected": -1.4450644254684448,
      "loss": 3.065,
      "nll_loss": 3.0051493644714355,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12232926487922668,
      "rewards/margins": 0.022177184000611305,
      "rewards/rejected": -0.14450645446777344,
      "step": 161
    },
    {
      "epoch": 0.10077760497667185,
      "grad_norm": 0.48701006174087524,
      "learning_rate": 4.5950000000000006e-05,
      "log_odds_chosen": 0.34209126234054565,
      "log_odds_ratio": -0.550736665725708,
      "logits/chosen": 0.14484034478664398,
      "logits/rejected": -0.13392439484596252,
      "logps/chosen": -1.1791408061981201,
      "logps/rejected": -1.4220118522644043,
      "loss": 2.7731,
      "nll_loss": 2.718043088912964,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11791408061981201,
      "rewards/margins": 0.02428712509572506,
      "rewards/rejected": -0.14220120012760162,
      "step": 162
    },
    {
      "epoch": 0.10139968895800933,
      "grad_norm": 0.42111936211586,
      "learning_rate": 4.5925e-05,
      "log_odds_chosen": 0.09586916118860245,
      "log_odds_ratio": -0.6500096321105957,
      "logits/chosen": 0.3419564366340637,
      "logits/rejected": 0.21585996448993683,
      "logps/chosen": -1.4152202606201172,
      "logps/rejected": -1.4942808151245117,
      "loss": 3.3547,
      "nll_loss": 3.2897045612335205,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14152203500270844,
      "rewards/margins": 0.007906057871878147,
      "rewards/rejected": -0.1494280993938446,
      "step": 163
    },
    {
      "epoch": 0.1020217729393468,
      "grad_norm": 0.651195228099823,
      "learning_rate": 4.5900000000000004e-05,
      "log_odds_chosen": 0.007867768406867981,
      "log_odds_ratio": -0.8247443437576294,
      "logits/chosen": 0.15761840343475342,
      "logits/rejected": 0.04952532798051834,
      "logps/chosen": -1.4861036539077759,
      "logps/rejected": -1.4350836277008057,
      "loss": 2.6309,
      "nll_loss": 2.548461437225342,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14861036837100983,
      "rewards/margins": -0.005102001130580902,
      "rewards/rejected": -0.14350835978984833,
      "step": 164
    },
    {
      "epoch": 0.1026438569206843,
      "grad_norm": 0.47373661398887634,
      "learning_rate": 4.5875000000000004e-05,
      "log_odds_chosen": 0.15654607117176056,
      "log_odds_ratio": -0.67220139503479,
      "logits/chosen": 0.42479315400123596,
      "logits/rejected": 0.2729908227920532,
      "logps/chosen": -1.3848153352737427,
      "logps/rejected": -1.4821150302886963,
      "loss": 3.1219,
      "nll_loss": 3.054720878601074,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1384815275669098,
      "rewards/margins": 0.009729975834488869,
      "rewards/rejected": -0.1482115089893341,
      "step": 165
    },
    {
      "epoch": 0.10326594090202178,
      "grad_norm": 0.39424991607666016,
      "learning_rate": 4.585e-05,
      "log_odds_chosen": 0.8976173400878906,
      "log_odds_ratio": -0.5067555904388428,
      "logits/chosen": 0.12819889187812805,
      "logits/rejected": -0.08246054500341415,
      "logps/chosen": -0.9714441299438477,
      "logps/rejected": -1.3971306085586548,
      "loss": 3.0076,
      "nll_loss": 2.9568872451782227,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09714441001415253,
      "rewards/margins": 0.04256865009665489,
      "rewards/rejected": -0.13971306383609772,
      "step": 166
    },
    {
      "epoch": 0.10388802488335926,
      "grad_norm": 0.4359923303127289,
      "learning_rate": 4.5825e-05,
      "log_odds_chosen": 0.13466989994049072,
      "log_odds_ratio": -0.7068646550178528,
      "logits/chosen": 0.29034674167633057,
      "logits/rejected": -0.1330055147409439,
      "logps/chosen": -1.3547630310058594,
      "logps/rejected": -1.426397442817688,
      "loss": 3.2544,
      "nll_loss": 3.1836836338043213,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13547630608081818,
      "rewards/margins": 0.007163448259234428,
      "rewards/rejected": -0.14263975620269775,
      "step": 167
    },
    {
      "epoch": 0.10451010886469674,
      "grad_norm": 0.47172248363494873,
      "learning_rate": 4.58e-05,
      "log_odds_chosen": 0.8191059827804565,
      "log_odds_ratio": -0.5032205581665039,
      "logits/chosen": 0.17938898503780365,
      "logits/rejected": 0.05944395065307617,
      "logps/chosen": -1.067983627319336,
      "logps/rejected": -1.5608168840408325,
      "loss": 2.9386,
      "nll_loss": 2.888256311416626,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10679835081100464,
      "rewards/margins": 0.04928332567214966,
      "rewards/rejected": -0.1560816764831543,
      "step": 168
    },
    {
      "epoch": 0.10513219284603421,
      "grad_norm": 0.5837737321853638,
      "learning_rate": 4.5775e-05,
      "log_odds_chosen": 0.21645092964172363,
      "log_odds_ratio": -0.6055108904838562,
      "logits/chosen": 0.42070725560188293,
      "logits/rejected": 0.0440496951341629,
      "logps/chosen": -1.0943289995193481,
      "logps/rejected": -1.2313555479049683,
      "loss": 3.6424,
      "nll_loss": 3.5818536281585693,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1094328984618187,
      "rewards/margins": 0.01370265707373619,
      "rewards/rejected": -0.12313555181026459,
      "step": 169
    },
    {
      "epoch": 0.1057542768273717,
      "grad_norm": 0.5199751257896423,
      "learning_rate": 4.575e-05,
      "log_odds_chosen": 0.17023658752441406,
      "log_odds_ratio": -0.7053340077400208,
      "logits/chosen": 0.16789722442626953,
      "logits/rejected": 0.13283823430538177,
      "logps/chosen": -1.2800236940383911,
      "logps/rejected": -1.3655457496643066,
      "loss": 2.931,
      "nll_loss": 2.860494613647461,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1280023753643036,
      "rewards/margins": 0.008552204817533493,
      "rewards/rejected": -0.1365545690059662,
      "step": 170
    },
    {
      "epoch": 0.10637636080870917,
      "grad_norm": 0.6239449381828308,
      "learning_rate": 4.5725e-05,
      "log_odds_chosen": -0.17279069125652313,
      "log_odds_ratio": -0.8778814077377319,
      "logits/chosen": 0.43515917658805847,
      "logits/rejected": 0.1489274650812149,
      "logps/chosen": -1.5174874067306519,
      "logps/rejected": -1.3432010412216187,
      "loss": 3.8779,
      "nll_loss": 3.790090799331665,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15174873173236847,
      "rewards/margins": -0.017428629100322723,
      "rewards/rejected": -0.13432011008262634,
      "step": 171
    },
    {
      "epoch": 0.10699844479004665,
      "grad_norm": 0.5014814734458923,
      "learning_rate": 4.5700000000000006e-05,
      "log_odds_chosen": 0.5290875434875488,
      "log_odds_ratio": -0.5955926179885864,
      "logits/chosen": 0.05028457194566727,
      "logits/rejected": -0.058185793459415436,
      "logps/chosen": -1.0905635356903076,
      "logps/rejected": -1.361629605293274,
      "loss": 2.4669,
      "nll_loss": 2.407294273376465,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10905636847019196,
      "rewards/margins": 0.027106598019599915,
      "rewards/rejected": -0.13616296648979187,
      "step": 172
    },
    {
      "epoch": 0.10762052877138413,
      "grad_norm": 0.45314520597457886,
      "learning_rate": 4.5675e-05,
      "log_odds_chosen": 0.11316041648387909,
      "log_odds_ratio": -0.6723555326461792,
      "logits/chosen": 0.1640755981206894,
      "logits/rejected": -0.034357067197561264,
      "logps/chosen": -1.294669508934021,
      "logps/rejected": -1.3866937160491943,
      "loss": 2.9407,
      "nll_loss": 2.8734371662139893,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1294669657945633,
      "rewards/margins": 0.009202406741678715,
      "rewards/rejected": -0.13866937160491943,
      "step": 173
    },
    {
      "epoch": 0.10824261275272162,
      "grad_norm": 0.44885244965553284,
      "learning_rate": 4.5650000000000005e-05,
      "log_odds_chosen": 0.3160461485385895,
      "log_odds_ratio": -0.5713240504264832,
      "logits/chosen": 0.23067131638526917,
      "logits/rejected": 0.04281052201986313,
      "logps/chosen": -1.31607186794281,
      "logps/rejected": -1.5356450080871582,
      "loss": 3.2553,
      "nll_loss": 3.1981256008148193,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13160717487335205,
      "rewards/margins": 0.021957319229841232,
      "rewards/rejected": -0.15356451272964478,
      "step": 174
    },
    {
      "epoch": 0.1088646967340591,
      "grad_norm": 0.49723100662231445,
      "learning_rate": 4.5625e-05,
      "log_odds_chosen": 0.9637750387191772,
      "log_odds_ratio": -0.4667288661003113,
      "logits/chosen": 0.13240934908390045,
      "logits/rejected": -0.04236384481191635,
      "logps/chosen": -1.2129688262939453,
      "logps/rejected": -1.8437656164169312,
      "loss": 3.156,
      "nll_loss": 3.109340190887451,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12129689753055573,
      "rewards/margins": 0.06307967007160187,
      "rewards/rejected": -0.1843765676021576,
      "step": 175
    },
    {
      "epoch": 0.10948678071539658,
      "grad_norm": 0.5128828883171082,
      "learning_rate": 4.5600000000000004e-05,
      "log_odds_chosen": 0.3789896070957184,
      "log_odds_ratio": -0.5478941798210144,
      "logits/chosen": 0.15509214997291565,
      "logits/rejected": 0.07832697033882141,
      "logps/chosen": -1.4777848720550537,
      "logps/rejected": -1.7933158874511719,
      "loss": 2.8832,
      "nll_loss": 2.8284482955932617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1477784812450409,
      "rewards/margins": 0.031553104519844055,
      "rewards/rejected": -0.17933160066604614,
      "step": 176
    },
    {
      "epoch": 0.11010886469673406,
      "grad_norm": 0.41308772563934326,
      "learning_rate": 4.5575e-05,
      "log_odds_chosen": -0.020543716847896576,
      "log_odds_ratio": -0.8699101209640503,
      "logits/chosen": 0.18716076016426086,
      "logits/rejected": -0.06990818679332733,
      "logps/chosen": -1.4498722553253174,
      "logps/rejected": -1.3525209426879883,
      "loss": 3.5567,
      "nll_loss": 3.469757556915283,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14498722553253174,
      "rewards/margins": -0.009735142812132835,
      "rewards/rejected": -0.13525208830833435,
      "step": 177
    },
    {
      "epoch": 0.11073094867807154,
      "grad_norm": 0.48727282881736755,
      "learning_rate": 4.555e-05,
      "log_odds_chosen": 0.26913127303123474,
      "log_odds_ratio": -0.6672435998916626,
      "logits/chosen": 0.163034588098526,
      "logits/rejected": -0.04085027799010277,
      "logps/chosen": -1.3829305171966553,
      "logps/rejected": -1.5941439867019653,
      "loss": 3.2551,
      "nll_loss": 3.1884212493896484,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13829305768013,
      "rewards/margins": 0.021121343597769737,
      "rewards/rejected": -0.1594144105911255,
      "step": 178
    },
    {
      "epoch": 0.11135303265940902,
      "grad_norm": 0.3962324857711792,
      "learning_rate": 4.5525e-05,
      "log_odds_chosen": 0.2522580623626709,
      "log_odds_ratio": -0.6461670994758606,
      "logits/chosen": 0.006648369133472443,
      "logits/rejected": -0.11574047803878784,
      "logps/chosen": -1.311452865600586,
      "logps/rejected": -1.4216885566711426,
      "loss": 2.9589,
      "nll_loss": 2.894331693649292,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13114528357982635,
      "rewards/margins": 0.011023584753274918,
      "rewards/rejected": -0.14216886460781097,
      "step": 179
    },
    {
      "epoch": 0.1119751166407465,
      "grad_norm": 0.4192509353160858,
      "learning_rate": 4.55e-05,
      "log_odds_chosen": 0.3250114917755127,
      "log_odds_ratio": -0.6104202270507812,
      "logits/chosen": 0.20732258260250092,
      "logits/rejected": 0.005627447739243507,
      "logps/chosen": -1.2976369857788086,
      "logps/rejected": -1.5418976545333862,
      "loss": 3.5311,
      "nll_loss": 3.4701075553894043,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12976369261741638,
      "rewards/margins": 0.02442607283592224,
      "rewards/rejected": -0.15418976545333862,
      "step": 180
    },
    {
      "epoch": 0.11259720062208398,
      "grad_norm": 0.3853883445262909,
      "learning_rate": 4.5475e-05,
      "log_odds_chosen": 0.11205227673053741,
      "log_odds_ratio": -0.65590500831604,
      "logits/chosen": 0.15651991963386536,
      "logits/rejected": -0.04156707227230072,
      "logps/chosen": -1.3088514804840088,
      "logps/rejected": -1.4056215286254883,
      "loss": 3.1677,
      "nll_loss": 3.102067470550537,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13088513910770416,
      "rewards/margins": 0.009677015244960785,
      "rewards/rejected": -0.14056214690208435,
      "step": 181
    },
    {
      "epoch": 0.11321928460342146,
      "grad_norm": 0.560540497303009,
      "learning_rate": 4.545000000000001e-05,
      "log_odds_chosen": 0.7333866953849792,
      "log_odds_ratio": -0.4243454039096832,
      "logits/chosen": 0.22672435641288757,
      "logits/rejected": -0.04371669888496399,
      "logps/chosen": -0.94305020570755,
      "logps/rejected": -1.453209638595581,
      "loss": 2.9935,
      "nll_loss": 2.951089859008789,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09430501610040665,
      "rewards/margins": 0.0510159507393837,
      "rewards/rejected": -0.14532096683979034,
      "step": 182
    },
    {
      "epoch": 0.11384136858475895,
      "grad_norm": 0.3617045283317566,
      "learning_rate": 4.5425e-05,
      "log_odds_chosen": 1.3186687231063843,
      "log_odds_ratio": -0.3125315308570862,
      "logits/chosen": 0.09166554361581802,
      "logits/rejected": 0.04617012292146683,
      "logps/chosen": -0.8101480007171631,
      "logps/rejected": -1.4856460094451904,
      "loss": 3.1663,
      "nll_loss": 3.1350221633911133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08101479709148407,
      "rewards/margins": 0.06754979491233826,
      "rewards/rejected": -0.14856459200382233,
      "step": 183
    },
    {
      "epoch": 0.11446345256609643,
      "grad_norm": 0.43170198798179626,
      "learning_rate": 4.5400000000000006e-05,
      "log_odds_chosen": 0.6821365356445312,
      "log_odds_ratio": -0.5054466724395752,
      "logits/chosen": 0.12610091269016266,
      "logits/rejected": 0.06542984396219254,
      "logps/chosen": -1.140608310699463,
      "logps/rejected": -1.5901718139648438,
      "loss": 3.0324,
      "nll_loss": 2.9818594455718994,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11406083405017853,
      "rewards/margins": 0.04495634511113167,
      "rewards/rejected": -0.1590171754360199,
      "step": 184
    },
    {
      "epoch": 0.1150855365474339,
      "grad_norm": 0.3195982277393341,
      "learning_rate": 4.5375e-05,
      "log_odds_chosen": 0.5379209518432617,
      "log_odds_ratio": -0.49095743894577026,
      "logits/chosen": 0.18578393757343292,
      "logits/rejected": 0.09170369803905487,
      "logps/chosen": -1.0872983932495117,
      "logps/rejected": -1.476131558418274,
      "loss": 3.5126,
      "nll_loss": 3.4635496139526367,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10872984677553177,
      "rewards/margins": 0.03888332098722458,
      "rewards/rejected": -0.14761316776275635,
      "step": 185
    },
    {
      "epoch": 0.11570762052877138,
      "grad_norm": 0.38618817925453186,
      "learning_rate": 4.5350000000000005e-05,
      "log_odds_chosen": 0.27880579233169556,
      "log_odds_ratio": -0.5841034054756165,
      "logits/chosen": 0.22152912616729736,
      "logits/rejected": 0.037873830646276474,
      "logps/chosen": -1.1515580415725708,
      "logps/rejected": -1.3237394094467163,
      "loss": 2.9728,
      "nll_loss": 2.914379835128784,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11515580117702484,
      "rewards/margins": 0.01721813902258873,
      "rewards/rejected": -0.13237394392490387,
      "step": 186
    },
    {
      "epoch": 0.11632970451010886,
      "grad_norm": 0.4079546332359314,
      "learning_rate": 4.5325000000000004e-05,
      "log_odds_chosen": -0.15632662177085876,
      "log_odds_ratio": -0.7896630764007568,
      "logits/chosen": 0.14888915419578552,
      "logits/rejected": 0.015266455709934235,
      "logps/chosen": -1.393578290939331,
      "logps/rejected": -1.2698719501495361,
      "loss": 2.9801,
      "nll_loss": 2.9011240005493164,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1393578201532364,
      "rewards/margins": -0.012370622716844082,
      "rewards/rejected": -0.12698720395565033,
      "step": 187
    },
    {
      "epoch": 0.11695178849144634,
      "grad_norm": 0.41905397176742554,
      "learning_rate": 4.53e-05,
      "log_odds_chosen": -0.014257103204727173,
      "log_odds_ratio": -0.7734930515289307,
      "logits/chosen": 0.22645491361618042,
      "logits/rejected": 0.10181404650211334,
      "logps/chosen": -1.4420051574707031,
      "logps/rejected": -1.3750616312026978,
      "loss": 3.305,
      "nll_loss": 3.2276558876037598,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14420051872730255,
      "rewards/margins": -0.006694357842206955,
      "rewards/rejected": -0.1375061571598053,
      "step": 188
    },
    {
      "epoch": 0.11757387247278382,
      "grad_norm": 0.4148879647254944,
      "learning_rate": 4.5275e-05,
      "log_odds_chosen": 0.3161711096763611,
      "log_odds_ratio": -0.5657863616943359,
      "logits/chosen": 0.09662674367427826,
      "logits/rejected": -0.03456714004278183,
      "logps/chosen": -1.122739553451538,
      "logps/rejected": -1.358430027961731,
      "loss": 2.703,
      "nll_loss": 2.6464426517486572,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11227396130561829,
      "rewards/margins": 0.023569051176309586,
      "rewards/rejected": -0.13584300875663757,
      "step": 189
    },
    {
      "epoch": 0.1181959564541213,
      "grad_norm": 1.207576870918274,
      "learning_rate": 4.525e-05,
      "log_odds_chosen": -0.09153705835342407,
      "log_odds_ratio": -0.7909804582595825,
      "logits/chosen": 0.0819367915391922,
      "logits/rejected": 0.03701075166463852,
      "logps/chosen": -1.6013344526290894,
      "logps/rejected": -1.5169143676757812,
      "loss": 3.0113,
      "nll_loss": 2.932208776473999,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1601334512233734,
      "rewards/margins": -0.008442009799182415,
      "rewards/rejected": -0.15169143676757812,
      "step": 190
    },
    {
      "epoch": 0.1188180404354588,
      "grad_norm": 0.42340028285980225,
      "learning_rate": 4.5225e-05,
      "log_odds_chosen": 0.7335370779037476,
      "log_odds_ratio": -0.4513997435569763,
      "logits/chosen": 0.15320263803005219,
      "logits/rejected": -0.06381958723068237,
      "logps/chosen": -0.8546745181083679,
      "logps/rejected": -1.3147962093353271,
      "loss": 3.2561,
      "nll_loss": 3.210986375808716,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08546745777130127,
      "rewards/margins": 0.046012166887521744,
      "rewards/rejected": -0.13147962093353271,
      "step": 191
    },
    {
      "epoch": 0.11944012441679627,
      "grad_norm": 0.4323185980319977,
      "learning_rate": 4.52e-05,
      "log_odds_chosen": 0.43490371108055115,
      "log_odds_ratio": -0.5183134078979492,
      "logits/chosen": 0.3056216835975647,
      "logits/rejected": 0.032028887420892715,
      "logps/chosen": -1.260435938835144,
      "logps/rejected": -1.577862024307251,
      "loss": 3.3018,
      "nll_loss": 3.249936819076538,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12604360282421112,
      "rewards/margins": 0.031742602586746216,
      "rewards/rejected": -0.15778620541095734,
      "step": 192
    },
    {
      "epoch": 0.12006220839813375,
      "grad_norm": 0.3796927332878113,
      "learning_rate": 4.5175e-05,
      "log_odds_chosen": 0.5728176236152649,
      "log_odds_ratio": -0.4924409091472626,
      "logits/chosen": 0.09786956757307053,
      "logits/rejected": -0.08450242131948471,
      "logps/chosen": -0.922787070274353,
      "logps/rejected": -1.3142082691192627,
      "loss": 2.9078,
      "nll_loss": 2.858553886413574,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09227870404720306,
      "rewards/margins": 0.039142120629549026,
      "rewards/rejected": -0.1314208209514618,
      "step": 193
    },
    {
      "epoch": 0.12068429237947123,
      "grad_norm": 0.7153462171554565,
      "learning_rate": 4.5150000000000006e-05,
      "log_odds_chosen": 0.5174391865730286,
      "log_odds_ratio": -0.5253512263298035,
      "logits/chosen": 0.3215792179107666,
      "logits/rejected": 0.12055753916501999,
      "logps/chosen": -1.3720208406448364,
      "logps/rejected": -1.7549231052398682,
      "loss": 3.0374,
      "nll_loss": 2.98486328125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13720208406448364,
      "rewards/margins": 0.03829022869467735,
      "rewards/rejected": -0.1754923164844513,
      "step": 194
    },
    {
      "epoch": 0.12130637636080871,
      "grad_norm": 0.4331601560115814,
      "learning_rate": 4.5125e-05,
      "log_odds_chosen": 0.7227539420127869,
      "log_odds_ratio": -0.4814952313899994,
      "logits/chosen": 0.2730720341205597,
      "logits/rejected": 0.1367463618516922,
      "logps/chosen": -1.1998332738876343,
      "logps/rejected": -1.7204265594482422,
      "loss": 3.2678,
      "nll_loss": 3.21962833404541,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11998332291841507,
      "rewards/margins": 0.05205933377146721,
      "rewards/rejected": -0.17204266786575317,
      "step": 195
    },
    {
      "epoch": 0.12192846034214619,
      "grad_norm": 0.4868512749671936,
      "learning_rate": 4.5100000000000005e-05,
      "log_odds_chosen": -0.11273391544818878,
      "log_odds_ratio": -0.7624410390853882,
      "logits/chosen": 0.162788987159729,
      "logits/rejected": 0.038346655666828156,
      "logps/chosen": -1.2809481620788574,
      "logps/rejected": -1.187037706375122,
      "loss": 2.6403,
      "nll_loss": 2.5640499591827393,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12809482216835022,
      "rewards/margins": -0.009391050785779953,
      "rewards/rejected": -0.11870376765727997,
      "step": 196
    },
    {
      "epoch": 0.12255054432348367,
      "grad_norm": 0.4865812659263611,
      "learning_rate": 4.5075e-05,
      "log_odds_chosen": -0.08828195929527283,
      "log_odds_ratio": -0.826860249042511,
      "logits/chosen": 0.2504577338695526,
      "logits/rejected": 0.20617207884788513,
      "logps/chosen": -1.288416862487793,
      "logps/rejected": -1.2700859308242798,
      "loss": 3.1873,
      "nll_loss": 3.1046361923217773,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12884169816970825,
      "rewards/margins": -0.001833103597164154,
      "rewards/rejected": -0.1270085871219635,
      "step": 197
    },
    {
      "epoch": 0.12317262830482115,
      "grad_norm": 0.605975866317749,
      "learning_rate": 4.5050000000000004e-05,
      "log_odds_chosen": 0.6784669756889343,
      "log_odds_ratio": -0.4767940044403076,
      "logits/chosen": 0.038463614881038666,
      "logits/rejected": 0.1368597447872162,
      "logps/chosen": -1.371410608291626,
      "logps/rejected": -1.9200092554092407,
      "loss": 2.0605,
      "nll_loss": 2.012791633605957,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13714107871055603,
      "rewards/margins": 0.054859861731529236,
      "rewards/rejected": -0.19200092554092407,
      "step": 198
    },
    {
      "epoch": 0.12379471228615863,
      "grad_norm": 0.3905656635761261,
      "learning_rate": 4.5025000000000003e-05,
      "log_odds_chosen": 0.27600833773612976,
      "log_odds_ratio": -0.5758750438690186,
      "logits/chosen": 0.24896469712257385,
      "logits/rejected": 0.1932651847600937,
      "logps/chosen": -1.2160751819610596,
      "logps/rejected": -1.3880665302276611,
      "loss": 3.6016,
      "nll_loss": 3.5440330505371094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12160752713680267,
      "rewards/margins": 0.01719912886619568,
      "rewards/rejected": -0.13880664110183716,
      "step": 199
    },
    {
      "epoch": 0.12441679626749612,
      "grad_norm": 0.9062671065330505,
      "learning_rate": 4.5e-05,
      "log_odds_chosen": 0.029399242252111435,
      "log_odds_ratio": -0.7444265484809875,
      "logits/chosen": 0.16195963323116302,
      "logits/rejected": -0.002658102661371231,
      "logps/chosen": -1.62650465965271,
      "logps/rejected": -1.6063220500946045,
      "loss": 3.017,
      "nll_loss": 2.9425477981567383,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.162650465965271,
      "rewards/margins": -0.0020182570442557335,
      "rewards/rejected": -0.16063222289085388,
      "step": 200
    },
    {
      "epoch": 0.12503888024883358,
      "grad_norm": 0.7090647220611572,
      "learning_rate": 4.4975e-05,
      "log_odds_chosen": -0.061332136392593384,
      "log_odds_ratio": -0.7987865209579468,
      "logits/chosen": 0.1813337355852127,
      "logits/rejected": -0.05886400490999222,
      "logps/chosen": -1.4757061004638672,
      "logps/rejected": -1.383807897567749,
      "loss": 3.2882,
      "nll_loss": 3.2082958221435547,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14757061004638672,
      "rewards/margins": -0.009189811535179615,
      "rewards/rejected": -0.13838079571723938,
      "step": 201
    },
    {
      "epoch": 0.12566096423017106,
      "grad_norm": 0.37017297744750977,
      "learning_rate": 4.495e-05,
      "log_odds_chosen": 0.6404499411582947,
      "log_odds_ratio": -0.5468106269836426,
      "logits/chosen": 0.2760981321334839,
      "logits/rejected": 0.07182542979717255,
      "logps/chosen": -1.1483453512191772,
      "logps/rejected": -1.6027138233184814,
      "loss": 3.5294,
      "nll_loss": 3.47471284866333,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11483453214168549,
      "rewards/margins": 0.04543685540556908,
      "rewards/rejected": -0.16027137637138367,
      "step": 202
    },
    {
      "epoch": 0.12628304821150854,
      "grad_norm": 0.3965308368206024,
      "learning_rate": 4.4925e-05,
      "log_odds_chosen": -0.044021353125572205,
      "log_odds_ratio": -0.7529421448707581,
      "logits/chosen": 0.12207438051700592,
      "logits/rejected": 0.07484577596187592,
      "logps/chosen": -1.0992047786712646,
      "logps/rejected": -1.0575580596923828,
      "loss": 2.8192,
      "nll_loss": 2.7438852787017822,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.10992048680782318,
      "rewards/margins": -0.004164676181972027,
      "rewards/rejected": -0.10575580596923828,
      "step": 203
    },
    {
      "epoch": 0.12690513219284602,
      "grad_norm": 0.3560815155506134,
      "learning_rate": 4.49e-05,
      "log_odds_chosen": 0.6957288384437561,
      "log_odds_ratio": -0.4497440457344055,
      "logits/chosen": 0.4506039619445801,
      "logits/rejected": 0.015391908586025238,
      "logps/chosen": -1.1910122632980347,
      "logps/rejected": -1.6560842990875244,
      "loss": 4.0035,
      "nll_loss": 3.9584884643554688,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11910124123096466,
      "rewards/margins": 0.04650719463825226,
      "rewards/rejected": -0.16560842096805573,
      "step": 204
    },
    {
      "epoch": 0.12752721617418353,
      "grad_norm": 0.6403685212135315,
      "learning_rate": 4.4875e-05,
      "log_odds_chosen": 0.2122831642627716,
      "log_odds_ratio": -0.8761559724807739,
      "logits/chosen": 0.014004663564264774,
      "logits/rejected": -0.14548785984516144,
      "logps/chosen": -1.689424753189087,
      "logps/rejected": -1.6934268474578857,
      "loss": 2.4198,
      "nll_loss": 2.332214117050171,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16894248127937317,
      "rewards/margins": 0.00040020793676376343,
      "rewards/rejected": -0.16934269666671753,
      "step": 205
    },
    {
      "epoch": 0.128149300155521,
      "grad_norm": 0.4000180661678314,
      "learning_rate": 4.4850000000000006e-05,
      "log_odds_chosen": 1.1184585094451904,
      "log_odds_ratio": -0.37039902806282043,
      "logits/chosen": 0.2634533941745758,
      "logits/rejected": 0.16103744506835938,
      "logps/chosen": -1.027080774307251,
      "logps/rejected": -1.6481044292449951,
      "loss": 3.1462,
      "nll_loss": 3.1091270446777344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10270807147026062,
      "rewards/margins": 0.06210235506296158,
      "rewards/rejected": -0.1648104339838028,
      "step": 206
    },
    {
      "epoch": 0.12877138413685849,
      "grad_norm": 0.5040024518966675,
      "learning_rate": 4.4825e-05,
      "log_odds_chosen": 0.2705846130847931,
      "log_odds_ratio": -0.59270179271698,
      "logits/chosen": 0.15955251455307007,
      "logits/rejected": -0.05338404327630997,
      "logps/chosen": -1.2470922470092773,
      "logps/rejected": -1.4498063325881958,
      "loss": 2.6152,
      "nll_loss": 2.555975914001465,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12470922619104385,
      "rewards/margins": 0.020271409302949905,
      "rewards/rejected": -0.14498063921928406,
      "step": 207
    },
    {
      "epoch": 0.12939346811819596,
      "grad_norm": 0.594912052154541,
      "learning_rate": 4.4800000000000005e-05,
      "log_odds_chosen": 0.7347403764724731,
      "log_odds_ratio": -0.44468581676483154,
      "logits/chosen": 0.14959490299224854,
      "logits/rejected": -0.027468346059322357,
      "logps/chosen": -1.372584581375122,
      "logps/rejected": -1.9416903257369995,
      "loss": 2.7127,
      "nll_loss": 2.6681857109069824,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13725847005844116,
      "rewards/margins": 0.056910570710897446,
      "rewards/rejected": -0.1941690444946289,
      "step": 208
    },
    {
      "epoch": 0.13001555209953344,
      "grad_norm": 0.5162251591682434,
      "learning_rate": 4.4775e-05,
      "log_odds_chosen": 0.7629825472831726,
      "log_odds_ratio": -0.48512592911720276,
      "logits/chosen": 0.2162114381790161,
      "logits/rejected": 0.039904482662677765,
      "logps/chosen": -1.1675233840942383,
      "logps/rejected": -1.7713046073913574,
      "loss": 2.8182,
      "nll_loss": 2.769730806350708,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11675234138965607,
      "rewards/margins": 0.06037811189889908,
      "rewards/rejected": -0.17713046073913574,
      "step": 209
    },
    {
      "epoch": 0.13063763608087092,
      "grad_norm": 0.4357577860355377,
      "learning_rate": 4.4750000000000004e-05,
      "log_odds_chosen": 0.16853998601436615,
      "log_odds_ratio": -0.6546946167945862,
      "logits/chosen": 0.28680700063705444,
      "logits/rejected": 0.13408944010734558,
      "logps/chosen": -1.4316234588623047,
      "logps/rejected": -1.5849097967147827,
      "loss": 3.1143,
      "nll_loss": 3.0488011837005615,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.143162339925766,
      "rewards/margins": 0.015328639186918736,
      "rewards/rejected": -0.15849095582962036,
      "step": 210
    },
    {
      "epoch": 0.1312597200622084,
      "grad_norm": 0.48053115606307983,
      "learning_rate": 4.4725e-05,
      "log_odds_chosen": 0.6203852891921997,
      "log_odds_ratio": -0.47376835346221924,
      "logits/chosen": 0.30184412002563477,
      "logits/rejected": 0.14279696345329285,
      "logps/chosen": -1.20229971408844,
      "logps/rejected": -1.6811708211898804,
      "loss": 3.1291,
      "nll_loss": 3.0817553997039795,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12022997438907623,
      "rewards/margins": 0.04788711294531822,
      "rewards/rejected": -0.16811709105968475,
      "step": 211
    },
    {
      "epoch": 0.13188180404354588,
      "grad_norm": 0.3863763213157654,
      "learning_rate": 4.47e-05,
      "log_odds_chosen": 0.23602965474128723,
      "log_odds_ratio": -0.6337170004844666,
      "logits/chosen": 0.14647379517555237,
      "logits/rejected": 0.13201558589935303,
      "logps/chosen": -1.5172404050827026,
      "logps/rejected": -1.723448634147644,
      "loss": 2.9906,
      "nll_loss": 2.9272663593292236,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15172404050827026,
      "rewards/margins": 0.02062082849442959,
      "rewards/rejected": -0.1723448783159256,
      "step": 212
    },
    {
      "epoch": 0.13250388802488336,
      "grad_norm": 0.5771374106407166,
      "learning_rate": 4.4675e-05,
      "log_odds_chosen": 0.6309947371482849,
      "log_odds_ratio": -0.5399520397186279,
      "logits/chosen": 0.12432387471199036,
      "logits/rejected": 0.20462198555469513,
      "logps/chosen": -1.5274455547332764,
      "logps/rejected": -2.06624436378479,
      "loss": 2.5767,
      "nll_loss": 2.5227041244506836,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15274454653263092,
      "rewards/margins": 0.053879883140325546,
      "rewards/rejected": -0.20662443339824677,
      "step": 213
    },
    {
      "epoch": 0.13312597200622084,
      "grad_norm": 0.41249603033065796,
      "learning_rate": 4.465e-05,
      "log_odds_chosen": -0.003160417079925537,
      "log_odds_ratio": -0.777141273021698,
      "logits/chosen": 0.14705267548561096,
      "logits/rejected": -0.06464418023824692,
      "logps/chosen": -1.5321584939956665,
      "logps/rejected": -1.5345534086227417,
      "loss": 3.1204,
      "nll_loss": 3.042703866958618,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.15321585536003113,
      "rewards/margins": 0.00023948121815919876,
      "rewards/rejected": -0.15345533192157745,
      "step": 214
    },
    {
      "epoch": 0.13374805598755832,
      "grad_norm": 0.4820147752761841,
      "learning_rate": 4.4625e-05,
      "log_odds_chosen": 0.2911366820335388,
      "log_odds_ratio": -0.5818198323249817,
      "logits/chosen": 0.2889520227909088,
      "logits/rejected": 0.08205986022949219,
      "logps/chosen": -1.4115300178527832,
      "logps/rejected": -1.6441384553909302,
      "loss": 2.9628,
      "nll_loss": 2.904639959335327,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1411530077457428,
      "rewards/margins": 0.02326083928346634,
      "rewards/rejected": -0.16441385447978973,
      "step": 215
    },
    {
      "epoch": 0.1343701399688958,
      "grad_norm": 0.41194218397140503,
      "learning_rate": 4.46e-05,
      "log_odds_chosen": 0.3945164382457733,
      "log_odds_ratio": -0.6375629305839539,
      "logits/chosen": 0.421595960855484,
      "logits/rejected": 0.30017945170402527,
      "logps/chosen": -1.3539952039718628,
      "logps/rejected": -1.5963020324707031,
      "loss": 3.5731,
      "nll_loss": 3.509347915649414,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13539952039718628,
      "rewards/margins": 0.024230685085058212,
      "rewards/rejected": -0.1596302092075348,
      "step": 216
    },
    {
      "epoch": 0.13499222395023328,
      "grad_norm": 0.503413200378418,
      "learning_rate": 4.4575e-05,
      "log_odds_chosen": 0.5858572721481323,
      "log_odds_ratio": -0.5386984944343567,
      "logits/chosen": 0.19849129021167755,
      "logits/rejected": 0.03486599028110504,
      "logps/chosen": -1.2306039333343506,
      "logps/rejected": -1.630487322807312,
      "loss": 3.1597,
      "nll_loss": 3.105804443359375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12306039035320282,
      "rewards/margins": 0.03998832777142525,
      "rewards/rejected": -0.16304872930049896,
      "step": 217
    },
    {
      "epoch": 0.13561430793157075,
      "grad_norm": 0.5384315848350525,
      "learning_rate": 4.4550000000000005e-05,
      "log_odds_chosen": 1.3037192821502686,
      "log_odds_ratio": -0.36916741728782654,
      "logits/chosen": 0.14214058220386505,
      "logits/rejected": 0.01107453927397728,
      "logps/chosen": -0.9095080494880676,
      "logps/rejected": -1.8086912631988525,
      "loss": 2.5382,
      "nll_loss": 2.5013184547424316,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09095080196857452,
      "rewards/margins": 0.08991833031177521,
      "rewards/rejected": -0.18086913228034973,
      "step": 218
    },
    {
      "epoch": 0.13623639191290823,
      "grad_norm": 0.46685314178466797,
      "learning_rate": 4.4525e-05,
      "log_odds_chosen": 0.5023396611213684,
      "log_odds_ratio": -0.5046614408493042,
      "logits/chosen": 0.2844086289405823,
      "logits/rejected": 0.1384527087211609,
      "logps/chosen": -1.252253532409668,
      "logps/rejected": -1.6022236347198486,
      "loss": 2.6639,
      "nll_loss": 2.613450527191162,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12522533535957336,
      "rewards/margins": 0.034997016191482544,
      "rewards/rejected": -0.1602223664522171,
      "step": 219
    },
    {
      "epoch": 0.1368584758942457,
      "grad_norm": 0.37732696533203125,
      "learning_rate": 4.4500000000000004e-05,
      "log_odds_chosen": 0.475115031003952,
      "log_odds_ratio": -0.5874269008636475,
      "logits/chosen": 0.18952935934066772,
      "logits/rejected": -0.05404677242040634,
      "logps/chosen": -1.0435460805892944,
      "logps/rejected": -1.356130599975586,
      "loss": 3.2866,
      "nll_loss": 3.227907419204712,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1043546050786972,
      "rewards/margins": 0.03125845640897751,
      "rewards/rejected": -0.13561305403709412,
      "step": 220
    },
    {
      "epoch": 0.1374805598755832,
      "grad_norm": 0.39852970838546753,
      "learning_rate": 4.4475e-05,
      "log_odds_chosen": 0.40951022505760193,
      "log_odds_ratio": -0.6001676917076111,
      "logits/chosen": 0.1417674422264099,
      "logits/rejected": -0.023162055760622025,
      "logps/chosen": -1.1433157920837402,
      "logps/rejected": -1.3727480173110962,
      "loss": 2.8523,
      "nll_loss": 2.7923316955566406,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11433158069849014,
      "rewards/margins": 0.022943219169974327,
      "rewards/rejected": -0.13727480173110962,
      "step": 221
    },
    {
      "epoch": 0.13810264385692067,
      "grad_norm": 0.4464816153049469,
      "learning_rate": 4.445e-05,
      "log_odds_chosen": 0.983132541179657,
      "log_odds_ratio": -0.3711632490158081,
      "logits/chosen": 0.22909504175186157,
      "logits/rejected": -0.061455175280570984,
      "logps/chosen": -1.0717201232910156,
      "logps/rejected": -1.7862846851348877,
      "loss": 3.3108,
      "nll_loss": 3.2737152576446533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10717201232910156,
      "rewards/margins": 0.07145645469427109,
      "rewards/rejected": -0.17862847447395325,
      "step": 222
    },
    {
      "epoch": 0.13872472783825818,
      "grad_norm": 0.5403879880905151,
      "learning_rate": 4.4425e-05,
      "log_odds_chosen": 0.5938290357589722,
      "log_odds_ratio": -0.4913523495197296,
      "logits/chosen": 0.09688084572553635,
      "logits/rejected": -0.05809571593999863,
      "logps/chosen": -1.2423458099365234,
      "logps/rejected": -1.6755822896957397,
      "loss": 2.842,
      "nll_loss": 2.7928619384765625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12423457205295563,
      "rewards/margins": 0.04332365095615387,
      "rewards/rejected": -0.1675582379102707,
      "step": 223
    },
    {
      "epoch": 0.13934681181959566,
      "grad_norm": 0.5137689113616943,
      "learning_rate": 4.44e-05,
      "log_odds_chosen": 0.9574803113937378,
      "log_odds_ratio": -0.6414791345596313,
      "logits/chosen": 0.41440945863723755,
      "logits/rejected": 0.10038256645202637,
      "logps/chosen": -1.3024613857269287,
      "logps/rejected": -1.8441886901855469,
      "loss": 4.0097,
      "nll_loss": 3.94557523727417,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1302461475133896,
      "rewards/margins": 0.05417274311184883,
      "rewards/rejected": -0.18441888689994812,
      "step": 224
    },
    {
      "epoch": 0.13996889580093314,
      "grad_norm": 0.34440910816192627,
      "learning_rate": 4.4375e-05,
      "log_odds_chosen": 0.7401940822601318,
      "log_odds_ratio": -0.5135765671730042,
      "logits/chosen": 0.34442248940467834,
      "logits/rejected": -0.011198656633496284,
      "logps/chosen": -1.2954169511795044,
      "logps/rejected": -1.7026612758636475,
      "loss": 3.9705,
      "nll_loss": 3.9191346168518066,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12954169511795044,
      "rewards/margins": 0.040724433958530426,
      "rewards/rejected": -0.17026613652706146,
      "step": 225
    },
    {
      "epoch": 0.14059097978227061,
      "grad_norm": 0.4693191349506378,
      "learning_rate": 4.435e-05,
      "log_odds_chosen": 0.46523332595825195,
      "log_odds_ratio": -0.5510789752006531,
      "logits/chosen": 0.09288059175014496,
      "logits/rejected": -0.01915971003472805,
      "logps/chosen": -1.0227751731872559,
      "logps/rejected": -1.2773354053497314,
      "loss": 2.7217,
      "nll_loss": 2.6666271686553955,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10227751731872559,
      "rewards/margins": 0.0254560187458992,
      "rewards/rejected": -0.1277335286140442,
      "step": 226
    },
    {
      "epoch": 0.1412130637636081,
      "grad_norm": 0.3790217936038971,
      "learning_rate": 4.4325e-05,
      "log_odds_chosen": 0.7664673924446106,
      "log_odds_ratio": -0.4955465793609619,
      "logits/chosen": 0.2821044921875,
      "logits/rejected": 0.1695287972688675,
      "logps/chosen": -1.0519059896469116,
      "logps/rejected": -1.5946249961853027,
      "loss": 3.3048,
      "nll_loss": 3.2552666664123535,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10519059747457504,
      "rewards/margins": 0.054271895438432693,
      "rewards/rejected": -0.15946249663829803,
      "step": 227
    },
    {
      "epoch": 0.14183514774494557,
      "grad_norm": 0.43475788831710815,
      "learning_rate": 4.43e-05,
      "log_odds_chosen": 1.3139894008636475,
      "log_odds_ratio": -0.3370991349220276,
      "logits/chosen": 0.12189202010631561,
      "logits/rejected": -0.0020540207624435425,
      "logps/chosen": -1.0322346687316895,
      "logps/rejected": -2.0027456283569336,
      "loss": 3.0615,
      "nll_loss": 3.0277514457702637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10322347283363342,
      "rewards/margins": 0.09705111384391785,
      "rewards/rejected": -0.20027457177639008,
      "step": 228
    },
    {
      "epoch": 0.14245723172628305,
      "grad_norm": 0.4523204565048218,
      "learning_rate": 4.4275e-05,
      "log_odds_chosen": 0.5393589735031128,
      "log_odds_ratio": -0.5129015445709229,
      "logits/chosen": 0.3066309988498688,
      "logits/rejected": 0.01438068225979805,
      "logps/chosen": -1.1355011463165283,
      "logps/rejected": -1.4902321100234985,
      "loss": 3.3797,
      "nll_loss": 3.328399181365967,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1135501116514206,
      "rewards/margins": 0.035473089665174484,
      "rewards/rejected": -0.14902320504188538,
      "step": 229
    },
    {
      "epoch": 0.14307931570762053,
      "grad_norm": 0.45688357949256897,
      "learning_rate": 4.4250000000000005e-05,
      "log_odds_chosen": 1.0163792371749878,
      "log_odds_ratio": -0.42969611287117004,
      "logits/chosen": 0.2794167995452881,
      "logits/rejected": 0.0069199977442622185,
      "logps/chosen": -1.1130561828613281,
      "logps/rejected": -1.8443264961242676,
      "loss": 3.1188,
      "nll_loss": 3.0758509635925293,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11130562424659729,
      "rewards/margins": 0.07312701642513275,
      "rewards/rejected": -0.18443265557289124,
      "step": 230
    },
    {
      "epoch": 0.143701399688958,
      "grad_norm": 0.46562471985816956,
      "learning_rate": 4.4225e-05,
      "log_odds_chosen": 0.051547348499298096,
      "log_odds_ratio": -0.6991626024246216,
      "logits/chosen": 0.3503286838531494,
      "logits/rejected": 0.16541802883148193,
      "logps/chosen": -1.2530078887939453,
      "logps/rejected": -1.2871830463409424,
      "loss": 3.1336,
      "nll_loss": 3.0636978149414062,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12530077993869781,
      "rewards/margins": 0.003417517989873886,
      "rewards/rejected": -0.128718301653862,
      "step": 231
    },
    {
      "epoch": 0.1443234836702955,
      "grad_norm": 0.5583508610725403,
      "learning_rate": 4.4200000000000004e-05,
      "log_odds_chosen": 0.4175459146499634,
      "log_odds_ratio": -0.5737831592559814,
      "logits/chosen": 0.4354173243045807,
      "logits/rejected": 0.12491598725318909,
      "logps/chosen": -1.5473692417144775,
      "logps/rejected": -1.9231562614440918,
      "loss": 3.3368,
      "nll_loss": 3.279399871826172,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15473692119121552,
      "rewards/margins": 0.03757869452238083,
      "rewards/rejected": -0.19231562316417694,
      "step": 232
    },
    {
      "epoch": 0.14494556765163297,
      "grad_norm": 0.5220412611961365,
      "learning_rate": 4.4174999999999996e-05,
      "log_odds_chosen": 0.49191123247146606,
      "log_odds_ratio": -0.6103449463844299,
      "logits/chosen": 0.17081604897975922,
      "logits/rejected": 0.05159229040145874,
      "logps/chosen": -1.402343988418579,
      "logps/rejected": -1.7350009679794312,
      "loss": 3.3047,
      "nll_loss": 3.2436368465423584,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14023439586162567,
      "rewards/margins": 0.03326570987701416,
      "rewards/rejected": -0.17350010573863983,
      "step": 233
    },
    {
      "epoch": 0.14556765163297045,
      "grad_norm": 0.7865017652511597,
      "learning_rate": 4.415e-05,
      "log_odds_chosen": -0.3338541090488434,
      "log_odds_ratio": -1.0038307905197144,
      "logits/chosen": 0.20039094984531403,
      "logits/rejected": 0.13230621814727783,
      "logps/chosen": -1.9984074831008911,
      "logps/rejected": -1.6774605512619019,
      "loss": 2.6104,
      "nll_loss": 2.5099916458129883,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1998407542705536,
      "rewards/margins": -0.03209469094872475,
      "rewards/rejected": -0.16774605214595795,
      "step": 234
    },
    {
      "epoch": 0.14618973561430793,
      "grad_norm": 0.5441064834594727,
      "learning_rate": 4.4125e-05,
      "log_odds_chosen": 0.19742441177368164,
      "log_odds_ratio": -0.6543236970901489,
      "logits/chosen": 0.2758321762084961,
      "logits/rejected": 0.2791656255722046,
      "logps/chosen": -1.3114479780197144,
      "logps/rejected": -1.4452178478240967,
      "loss": 2.6863,
      "nll_loss": 2.620828866958618,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13114479184150696,
      "rewards/margins": 0.013376971706748009,
      "rewards/rejected": -0.1445217728614807,
      "step": 235
    },
    {
      "epoch": 0.1468118195956454,
      "grad_norm": 0.4621172845363617,
      "learning_rate": 4.41e-05,
      "log_odds_chosen": 0.35999900102615356,
      "log_odds_ratio": -0.6286050081253052,
      "logits/chosen": 0.3370401859283447,
      "logits/rejected": 0.1895410120487213,
      "logps/chosen": -1.0965385437011719,
      "logps/rejected": -1.2593486309051514,
      "loss": 3.3825,
      "nll_loss": 3.3196167945861816,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10965386033058167,
      "rewards/margins": 0.016280999407172203,
      "rewards/rejected": -0.12593486905097961,
      "step": 236
    },
    {
      "epoch": 0.14743390357698288,
      "grad_norm": 0.4633600115776062,
      "learning_rate": 4.4075e-05,
      "log_odds_chosen": 0.15592439472675323,
      "log_odds_ratio": -0.6589779853820801,
      "logits/chosen": 0.24592751264572144,
      "logits/rejected": 0.05784950405359268,
      "logps/chosen": -1.350083827972412,
      "logps/rejected": -1.4974408149719238,
      "loss": 3.17,
      "nll_loss": 3.1041297912597656,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13500836491584778,
      "rewards/margins": 0.014735697768628597,
      "rewards/rejected": -0.14974406361579895,
      "step": 237
    },
    {
      "epoch": 0.14805598755832036,
      "grad_norm": 0.5822882652282715,
      "learning_rate": 4.405e-05,
      "log_odds_chosen": 0.5478257536888123,
      "log_odds_ratio": -0.6282358169555664,
      "logits/chosen": 0.28536713123321533,
      "logits/rejected": 0.10193793475627899,
      "logps/chosen": -1.4900511503219604,
      "logps/rejected": -1.923351526260376,
      "loss": 3.1027,
      "nll_loss": 3.039842128753662,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14900511503219604,
      "rewards/margins": 0.043330058455467224,
      "rewards/rejected": -0.19233515858650208,
      "step": 238
    },
    {
      "epoch": 0.14867807153965784,
      "grad_norm": 0.4006459712982178,
      "learning_rate": 4.4025e-05,
      "log_odds_chosen": 0.30393242835998535,
      "log_odds_ratio": -0.5825948715209961,
      "logits/chosen": 0.30995607376098633,
      "logits/rejected": 0.09291841834783554,
      "logps/chosen": -1.4452520608901978,
      "logps/rejected": -1.6686973571777344,
      "loss": 3.2763,
      "nll_loss": 3.218071937561035,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1445252150297165,
      "rewards/margins": 0.022344522178173065,
      "rewards/rejected": -0.16686972975730896,
      "step": 239
    },
    {
      "epoch": 0.14930015552099535,
      "grad_norm": 0.37317949533462524,
      "learning_rate": 4.4000000000000006e-05,
      "log_odds_chosen": 1.5197196006774902,
      "log_odds_ratio": -0.4212367534637451,
      "logits/chosen": 0.3777592182159424,
      "logits/rejected": 0.2529478073120117,
      "logps/chosen": -0.9898640513420105,
      "logps/rejected": -2.145714282989502,
      "loss": 3.4547,
      "nll_loss": 3.4126036167144775,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09898640215396881,
      "rewards/margins": 0.11558501422405243,
      "rewards/rejected": -0.21457141637802124,
      "step": 240
    },
    {
      "epoch": 0.14992223950233283,
      "grad_norm": 0.5220414996147156,
      "learning_rate": 4.3975e-05,
      "log_odds_chosen": 0.9724363684654236,
      "log_odds_ratio": -0.39547187089920044,
      "logits/chosen": 0.18093883991241455,
      "logits/rejected": 0.09247509390115738,
      "logps/chosen": -1.3090280294418335,
      "logps/rejected": -1.9929441213607788,
      "loss": 2.5084,
      "nll_loss": 2.4688820838928223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13090281188488007,
      "rewards/margins": 0.06839160621166229,
      "rewards/rejected": -0.19929441809654236,
      "step": 241
    },
    {
      "epoch": 0.1505443234836703,
      "grad_norm": 0.41178998351097107,
      "learning_rate": 4.3950000000000004e-05,
      "log_odds_chosen": 0.03777070343494415,
      "log_odds_ratio": -0.7433863282203674,
      "logits/chosen": 0.18827003240585327,
      "logits/rejected": 0.17734594643115997,
      "logps/chosen": -1.3193129301071167,
      "logps/rejected": -1.3846216201782227,
      "loss": 2.8552,
      "nll_loss": 2.7808141708374023,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13193130493164062,
      "rewards/margins": 0.006530864164233208,
      "rewards/rejected": -0.1384621560573578,
      "step": 242
    },
    {
      "epoch": 0.15116640746500778,
      "grad_norm": 0.41735777258872986,
      "learning_rate": 4.3925e-05,
      "log_odds_chosen": 0.5820766687393188,
      "log_odds_ratio": -0.4971098303794861,
      "logits/chosen": 0.3844646215438843,
      "logits/rejected": 0.1357293277978897,
      "logps/chosen": -1.285773515701294,
      "logps/rejected": -1.6937378644943237,
      "loss": 3.6206,
      "nll_loss": 3.5709385871887207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1285773366689682,
      "rewards/margins": 0.0407964326441288,
      "rewards/rejected": -0.1693737804889679,
      "step": 243
    },
    {
      "epoch": 0.15178849144634526,
      "grad_norm": 0.4252638816833496,
      "learning_rate": 4.39e-05,
      "log_odds_chosen": 0.9820137023925781,
      "log_odds_ratio": -0.5139923095703125,
      "logits/chosen": 0.09503781795501709,
      "logits/rejected": 0.09257042407989502,
      "logps/chosen": -1.1998246908187866,
      "logps/rejected": -1.9771127700805664,
      "loss": 2.4225,
      "nll_loss": 2.371094226837158,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11998246610164642,
      "rewards/margins": 0.07772879302501678,
      "rewards/rejected": -0.1977112740278244,
      "step": 244
    },
    {
      "epoch": 0.15241057542768274,
      "grad_norm": 0.35691797733306885,
      "learning_rate": 4.3875e-05,
      "log_odds_chosen": 0.15277954936027527,
      "log_odds_ratio": -0.6356155872344971,
      "logits/chosen": 0.1442718803882599,
      "logits/rejected": 0.08599300682544708,
      "logps/chosen": -1.3047629594802856,
      "logps/rejected": -1.41819167137146,
      "loss": 2.9185,
      "nll_loss": 2.8548996448516846,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13047629594802856,
      "rewards/margins": 0.011342879384756088,
      "rewards/rejected": -0.14181917905807495,
      "step": 245
    },
    {
      "epoch": 0.15303265940902022,
      "grad_norm": 0.3984750509262085,
      "learning_rate": 4.385e-05,
      "log_odds_chosen": 1.2786288261413574,
      "log_odds_ratio": -0.36163750290870667,
      "logits/chosen": 0.1432109922170639,
      "logits/rejected": 0.06168469786643982,
      "logps/chosen": -0.9494646191596985,
      "logps/rejected": -1.8683538436889648,
      "loss": 3.0516,
      "nll_loss": 3.0154216289520264,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0949464663863182,
      "rewards/margins": 0.0918889045715332,
      "rewards/rejected": -0.1868353635072708,
      "step": 246
    },
    {
      "epoch": 0.1536547433903577,
      "grad_norm": 0.35005396604537964,
      "learning_rate": 4.3825e-05,
      "log_odds_chosen": 0.36924123764038086,
      "log_odds_ratio": -0.6463004946708679,
      "logits/chosen": 0.08100152760744095,
      "logits/rejected": -0.09407303482294083,
      "logps/chosen": -1.0215667486190796,
      "logps/rejected": -1.2288739681243896,
      "loss": 2.9297,
      "nll_loss": 2.865115165710449,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10215667635202408,
      "rewards/margins": 0.020730726420879364,
      "rewards/rejected": -0.12288740277290344,
      "step": 247
    },
    {
      "epoch": 0.15427682737169518,
      "grad_norm": 0.5391296744346619,
      "learning_rate": 4.38e-05,
      "log_odds_chosen": 0.5213131904602051,
      "log_odds_ratio": -0.5025854110717773,
      "logits/chosen": 0.06299649924039841,
      "logits/rejected": -0.09286680817604065,
      "logps/chosen": -1.111344337463379,
      "logps/rejected": -1.5201202630996704,
      "loss": 3.0356,
      "nll_loss": 2.9853503704071045,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11113443225622177,
      "rewards/margins": 0.04087759926915169,
      "rewards/rejected": -0.15201203525066376,
      "step": 248
    },
    {
      "epoch": 0.15489891135303266,
      "grad_norm": 0.454863578081131,
      "learning_rate": 4.3775e-05,
      "log_odds_chosen": 1.0419408082962036,
      "log_odds_ratio": -0.47779956459999084,
      "logits/chosen": 0.06458957493305206,
      "logits/rejected": -0.05411160737276077,
      "logps/chosen": -1.1926398277282715,
      "logps/rejected": -1.8617669343948364,
      "loss": 3.2576,
      "nll_loss": 3.2098097801208496,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11926397681236267,
      "rewards/margins": 0.0669127106666565,
      "rewards/rejected": -0.18617668747901917,
      "step": 249
    },
    {
      "epoch": 0.15552099533437014,
      "grad_norm": 0.5558252334594727,
      "learning_rate": 4.375e-05,
      "log_odds_chosen": 1.1860463619232178,
      "log_odds_ratio": -0.41251182556152344,
      "logits/chosen": -0.08973684906959534,
      "logits/rejected": -0.16550233960151672,
      "logps/chosen": -1.0720642805099487,
      "logps/rejected": -1.8928790092468262,
      "loss": 2.5241,
      "nll_loss": 2.4828810691833496,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10720643401145935,
      "rewards/margins": 0.08208145946264267,
      "rewards/rejected": -0.18928790092468262,
      "step": 250
    },
    {
      "epoch": 0.15614307931570762,
      "grad_norm": 0.46578970551490784,
      "learning_rate": 4.3725000000000006e-05,
      "log_odds_chosen": 0.8737031817436218,
      "log_odds_ratio": -0.45753180980682373,
      "logits/chosen": 0.10488224774599075,
      "logits/rejected": 0.002302175387740135,
      "logps/chosen": -1.141048550605774,
      "logps/rejected": -1.7917754650115967,
      "loss": 2.7639,
      "nll_loss": 2.718146562576294,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11410486698150635,
      "rewards/margins": 0.0650726854801178,
      "rewards/rejected": -0.17917755246162415,
      "step": 251
    },
    {
      "epoch": 0.1567651632970451,
      "grad_norm": 0.5264093279838562,
      "learning_rate": 4.3700000000000005e-05,
      "log_odds_chosen": 1.2083812952041626,
      "log_odds_ratio": -0.46954599022865295,
      "logits/chosen": 0.10020960867404938,
      "logits/rejected": 0.05089284107089043,
      "logps/chosen": -1.2586607933044434,
      "logps/rejected": -2.2547106742858887,
      "loss": 2.6152,
      "nll_loss": 2.568207025527954,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12586607038974762,
      "rewards/margins": 0.09960497915744781,
      "rewards/rejected": -0.22547104954719543,
      "step": 252
    },
    {
      "epoch": 0.15738724727838257,
      "grad_norm": 0.39940425753593445,
      "learning_rate": 4.3675000000000005e-05,
      "log_odds_chosen": 0.8237320780754089,
      "log_odds_ratio": -0.42615777254104614,
      "logits/chosen": 0.12673884630203247,
      "logits/rejected": 0.04520142823457718,
      "logps/chosen": -1.2019363641738892,
      "logps/rejected": -1.8468737602233887,
      "loss": 3.0343,
      "nll_loss": 2.991687536239624,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12019363045692444,
      "rewards/margins": 0.06449373066425323,
      "rewards/rejected": -0.18468736112117767,
      "step": 253
    },
    {
      "epoch": 0.15800933125972005,
      "grad_norm": 0.36626678705215454,
      "learning_rate": 4.3650000000000004e-05,
      "log_odds_chosen": 0.9984500408172607,
      "log_odds_ratio": -0.3665887117385864,
      "logits/chosen": 0.18472406268119812,
      "logits/rejected": 0.037876978516578674,
      "logps/chosen": -1.0484715700149536,
      "logps/rejected": -1.8369355201721191,
      "loss": 3.5121,
      "nll_loss": 3.4753966331481934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10484716296195984,
      "rewards/margins": 0.07884638756513596,
      "rewards/rejected": -0.1836935579776764,
      "step": 254
    },
    {
      "epoch": 0.15863141524105753,
      "grad_norm": 0.4521494507789612,
      "learning_rate": 4.3625e-05,
      "log_odds_chosen": 1.2302688360214233,
      "log_odds_ratio": -0.3208959698677063,
      "logits/chosen": 0.1981630027294159,
      "logits/rejected": -0.019413426518440247,
      "logps/chosen": -0.9359973669052124,
      "logps/rejected": -1.7934815883636475,
      "loss": 3.3395,
      "nll_loss": 3.3074443340301514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09359973669052124,
      "rewards/margins": 0.08574841916561127,
      "rewards/rejected": -0.1793481707572937,
      "step": 255
    },
    {
      "epoch": 0.159253499222395,
      "grad_norm": 0.4002397060394287,
      "learning_rate": 4.36e-05,
      "log_odds_chosen": 0.2759625315666199,
      "log_odds_ratio": -0.5995438098907471,
      "logits/chosen": 0.0834539383649826,
      "logits/rejected": -0.03273439407348633,
      "logps/chosen": -1.3060052394866943,
      "logps/rejected": -1.5258723497390747,
      "loss": 3.0473,
      "nll_loss": 2.9873416423797607,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13060052692890167,
      "rewards/margins": 0.021986715495586395,
      "rewards/rejected": -0.15258723497390747,
      "step": 256
    },
    {
      "epoch": 0.1598755832037325,
      "grad_norm": 0.6363208293914795,
      "learning_rate": 4.3575e-05,
      "log_odds_chosen": 0.5167827606201172,
      "log_odds_ratio": -0.5308623909950256,
      "logits/chosen": 0.23719561100006104,
      "logits/rejected": 0.009174295701086521,
      "logps/chosen": -1.3490158319473267,
      "logps/rejected": -1.7873324155807495,
      "loss": 3.2075,
      "nll_loss": 3.1543946266174316,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13490158319473267,
      "rewards/margins": 0.04383165389299393,
      "rewards/rejected": -0.1787332445383072,
      "step": 257
    },
    {
      "epoch": 0.16049766718507,
      "grad_norm": 0.4826381504535675,
      "learning_rate": 4.355e-05,
      "log_odds_chosen": 0.10458363592624664,
      "log_odds_ratio": -0.709639310836792,
      "logits/chosen": 0.22416506707668304,
      "logits/rejected": 0.1090393140912056,
      "logps/chosen": -1.410514235496521,
      "logps/rejected": -1.4754109382629395,
      "loss": 3.4102,
      "nll_loss": 3.339193820953369,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14105141162872314,
      "rewards/margins": 0.006489668972790241,
      "rewards/rejected": -0.1475410908460617,
      "step": 258
    },
    {
      "epoch": 0.16111975116640748,
      "grad_norm": 0.41199758648872375,
      "learning_rate": 4.352500000000001e-05,
      "log_odds_chosen": 1.1602838039398193,
      "log_odds_ratio": -0.3441213369369507,
      "logits/chosen": 0.12007400393486023,
      "logits/rejected": 0.005203016102313995,
      "logps/chosen": -0.8913068175315857,
      "logps/rejected": -1.666398286819458,
      "loss": 3.228,
      "nll_loss": 3.1936206817626953,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08913067728281021,
      "rewards/margins": 0.07750913500785828,
      "rewards/rejected": -0.16663981974124908,
      "step": 259
    },
    {
      "epoch": 0.16174183514774496,
      "grad_norm": 0.5916374325752258,
      "learning_rate": 4.35e-05,
      "log_odds_chosen": 0.4282585680484772,
      "log_odds_ratio": -0.5398960709571838,
      "logits/chosen": 0.22822090983390808,
      "logits/rejected": 0.1819199174642563,
      "logps/chosen": -1.363776445388794,
      "logps/rejected": -1.7082788944244385,
      "loss": 2.7722,
      "nll_loss": 2.7182321548461914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13637763261795044,
      "rewards/margins": 0.03445024788379669,
      "rewards/rejected": -0.17082789540290833,
      "step": 260
    },
    {
      "epoch": 0.16236391912908243,
      "grad_norm": 0.5163818001747131,
      "learning_rate": 4.3475000000000006e-05,
      "log_odds_chosen": 0.7817611694335938,
      "log_odds_ratio": -0.419194757938385,
      "logits/chosen": 0.11984409391880035,
      "logits/rejected": -0.05257415026426315,
      "logps/chosen": -1.1561753749847412,
      "logps/rejected": -1.754962682723999,
      "loss": 2.795,
      "nll_loss": 2.7531259059906006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1156175285577774,
      "rewards/margins": 0.05987873673439026,
      "rewards/rejected": -0.17549628019332886,
      "step": 261
    },
    {
      "epoch": 0.1629860031104199,
      "grad_norm": 0.3384513854980469,
      "learning_rate": 4.345e-05,
      "log_odds_chosen": 0.48024481534957886,
      "log_odds_ratio": -0.5129582285881042,
      "logits/chosen": 0.38080763816833496,
      "logits/rejected": 0.12893861532211304,
      "logps/chosen": -1.1857705116271973,
      "logps/rejected": -1.510347604751587,
      "loss": 3.62,
      "nll_loss": 3.5687360763549805,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11857704818248749,
      "rewards/margins": 0.032457705587148666,
      "rewards/rejected": -0.15103475749492645,
      "step": 262
    },
    {
      "epoch": 0.1636080870917574,
      "grad_norm": 0.3694182336330414,
      "learning_rate": 4.3425000000000005e-05,
      "log_odds_chosen": 1.5465469360351562,
      "log_odds_ratio": -0.2594394385814667,
      "logits/chosen": 0.19702796638011932,
      "logits/rejected": 0.1825985312461853,
      "logps/chosen": -1.0576568841934204,
      "logps/rejected": -2.280040740966797,
      "loss": 2.7614,
      "nll_loss": 2.735440492630005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1057656854391098,
      "rewards/margins": 0.12223838269710541,
      "rewards/rejected": -0.2280040681362152,
      "step": 263
    },
    {
      "epoch": 0.16423017107309487,
      "grad_norm": 0.4737184941768646,
      "learning_rate": 4.3400000000000005e-05,
      "log_odds_chosen": 1.012281060218811,
      "log_odds_ratio": -0.45542672276496887,
      "logits/chosen": 0.18417447805404663,
      "logits/rejected": 0.2393220067024231,
      "logps/chosen": -1.3665863275527954,
      "logps/rejected": -2.137876272201538,
      "loss": 3.0061,
      "nll_loss": 2.9605910778045654,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13665863871574402,
      "rewards/margins": 0.07712897658348083,
      "rewards/rejected": -0.21378763020038605,
      "step": 264
    },
    {
      "epoch": 0.16485225505443235,
      "grad_norm": 0.5989867448806763,
      "learning_rate": 4.3375000000000004e-05,
      "log_odds_chosen": 0.47888749837875366,
      "log_odds_ratio": -0.5475578904151917,
      "logits/chosen": 0.34955573081970215,
      "logits/rejected": -0.01235947385430336,
      "logps/chosen": -1.1995731592178345,
      "logps/rejected": -1.5407181978225708,
      "loss": 3.5055,
      "nll_loss": 3.4507837295532227,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1199573203921318,
      "rewards/margins": 0.034114498645067215,
      "rewards/rejected": -0.15407180786132812,
      "step": 265
    },
    {
      "epoch": 0.16547433903576983,
      "grad_norm": 0.45694050192832947,
      "learning_rate": 4.335e-05,
      "log_odds_chosen": 1.0779105424880981,
      "log_odds_ratio": -0.4133705198764801,
      "logits/chosen": 0.31415730714797974,
      "logits/rejected": 0.17435882985591888,
      "logps/chosen": -1.2905559539794922,
      "logps/rejected": -2.1517629623413086,
      "loss": 3.4923,
      "nll_loss": 3.4509427547454834,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12905558943748474,
      "rewards/margins": 0.08612069487571716,
      "rewards/rejected": -0.2151763141155243,
      "step": 266
    },
    {
      "epoch": 0.1660964230171073,
      "grad_norm": 0.5878811478614807,
      "learning_rate": 4.3325e-05,
      "log_odds_chosen": 0.61550372838974,
      "log_odds_ratio": -0.5280717015266418,
      "logits/chosen": 0.13362732529640198,
      "logits/rejected": -0.045731253921985626,
      "logps/chosen": -1.3563334941864014,
      "logps/rejected": -1.7919440269470215,
      "loss": 2.8887,
      "nll_loss": 2.8359363079071045,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13563336431980133,
      "rewards/margins": 0.04356103017926216,
      "rewards/rejected": -0.1791943907737732,
      "step": 267
    },
    {
      "epoch": 0.1667185069984448,
      "grad_norm": 0.32133936882019043,
      "learning_rate": 4.33e-05,
      "log_odds_chosen": 0.21047142148017883,
      "log_odds_ratio": -0.7247399687767029,
      "logits/chosen": 0.40043243765830994,
      "logits/rejected": 0.05317821353673935,
      "logps/chosen": -1.4496357440948486,
      "logps/rejected": -1.6929826736450195,
      "loss": 3.9373,
      "nll_loss": 3.8648571968078613,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1449635773897171,
      "rewards/margins": 0.02433469332754612,
      "rewards/rejected": -0.16929826140403748,
      "step": 268
    },
    {
      "epoch": 0.16734059097978227,
      "grad_norm": 0.39708441495895386,
      "learning_rate": 4.3275e-05,
      "log_odds_chosen": 0.2173278033733368,
      "log_odds_ratio": -0.659453272819519,
      "logits/chosen": 0.16504698991775513,
      "logits/rejected": 0.015571564435958862,
      "logps/chosen": -1.2936235666275024,
      "logps/rejected": -1.4728864431381226,
      "loss": 3.2912,
      "nll_loss": 3.225303888320923,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12936235964298248,
      "rewards/margins": 0.0179262924939394,
      "rewards/rejected": -0.14728865027427673,
      "step": 269
    },
    {
      "epoch": 0.16796267496111975,
      "grad_norm": 0.4461729824542999,
      "learning_rate": 4.325e-05,
      "log_odds_chosen": 0.7989851236343384,
      "log_odds_ratio": -0.5427728295326233,
      "logits/chosen": -0.0695042535662651,
      "logits/rejected": -0.15135838091373444,
      "logps/chosen": -1.2412259578704834,
      "logps/rejected": -1.7757339477539062,
      "loss": 2.8043,
      "nll_loss": 2.7499935626983643,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12412260472774506,
      "rewards/margins": 0.053450800478458405,
      "rewards/rejected": -0.17757341265678406,
      "step": 270
    },
    {
      "epoch": 0.16858475894245722,
      "grad_norm": 0.5164592266082764,
      "learning_rate": 4.322500000000001e-05,
      "log_odds_chosen": 0.5904808640480042,
      "log_odds_ratio": -0.5027241110801697,
      "logits/chosen": 0.006964411586523056,
      "logits/rejected": -0.11874233931303024,
      "logps/chosen": -1.0042153596878052,
      "logps/rejected": -1.401916265487671,
      "loss": 2.8845,
      "nll_loss": 2.834200143814087,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10042153298854828,
      "rewards/margins": 0.039770081639289856,
      "rewards/rejected": -0.14019162952899933,
      "step": 271
    },
    {
      "epoch": 0.1692068429237947,
      "grad_norm": 0.5214595198631287,
      "learning_rate": 4.32e-05,
      "log_odds_chosen": 0.24742233753204346,
      "log_odds_ratio": -0.7152416110038757,
      "logits/chosen": 0.1348581463098526,
      "logits/rejected": -0.040653035044670105,
      "logps/chosen": -1.2521984577178955,
      "logps/rejected": -1.3138203620910645,
      "loss": 3.3,
      "nll_loss": 3.2285189628601074,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12521985173225403,
      "rewards/margins": 0.006162186153233051,
      "rewards/rejected": -0.1313820332288742,
      "step": 272
    },
    {
      "epoch": 0.16982892690513218,
      "grad_norm": 0.40921929478645325,
      "learning_rate": 4.3175000000000006e-05,
      "log_odds_chosen": 0.8957703113555908,
      "log_odds_ratio": -0.4180378317832947,
      "logits/chosen": 0.04939873516559601,
      "logits/rejected": -0.06932801753282547,
      "logps/chosen": -0.9343599677085876,
      "logps/rejected": -1.509351134300232,
      "loss": 3.069,
      "nll_loss": 3.027172088623047,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09343599528074265,
      "rewards/margins": 0.05749912187457085,
      "rewards/rejected": -0.1509351134300232,
      "step": 273
    },
    {
      "epoch": 0.17045101088646966,
      "grad_norm": 0.5043439865112305,
      "learning_rate": 4.315e-05,
      "log_odds_chosen": 1.2644999027252197,
      "log_odds_ratio": -0.43298041820526123,
      "logits/chosen": 0.1321154087781906,
      "logits/rejected": -0.09128782153129578,
      "logps/chosen": -1.0918232202529907,
      "logps/rejected": -2.1638400554656982,
      "loss": 2.9592,
      "nll_loss": 2.9159512519836426,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10918232053518295,
      "rewards/margins": 0.10720168799161911,
      "rewards/rejected": -0.21638402342796326,
      "step": 274
    },
    {
      "epoch": 0.17107309486780714,
      "grad_norm": 0.4372323751449585,
      "learning_rate": 4.3125000000000005e-05,
      "log_odds_chosen": 0.33947059512138367,
      "log_odds_ratio": -0.6284760236740112,
      "logits/chosen": 0.08530203998088837,
      "logits/rejected": -0.054735083132982254,
      "logps/chosen": -1.3148372173309326,
      "logps/rejected": -1.509586215019226,
      "loss": 3.0704,
      "nll_loss": 3.007549285888672,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13148371875286102,
      "rewards/margins": 0.019474901258945465,
      "rewards/rejected": -0.15095862746238708,
      "step": 275
    },
    {
      "epoch": 0.17169517884914465,
      "grad_norm": 0.3891263008117676,
      "learning_rate": 4.3100000000000004e-05,
      "log_odds_chosen": 0.5235820412635803,
      "log_odds_ratio": -0.6117573976516724,
      "logits/chosen": 0.34458497166633606,
      "logits/rejected": 0.05262750759720802,
      "logps/chosen": -1.4382394552230835,
      "logps/rejected": -1.8694814443588257,
      "loss": 3.8207,
      "nll_loss": 3.75954270362854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14382395148277283,
      "rewards/margins": 0.04312421381473541,
      "rewards/rejected": -0.18694815039634705,
      "step": 276
    },
    {
      "epoch": 0.17231726283048213,
      "grad_norm": 0.4415547251701355,
      "learning_rate": 4.3075000000000003e-05,
      "log_odds_chosen": 0.05041904002428055,
      "log_odds_ratio": -0.6868739128112793,
      "logits/chosen": 0.006680905818939209,
      "logits/rejected": -0.09860651195049286,
      "logps/chosen": -1.2028601169586182,
      "logps/rejected": -1.2418127059936523,
      "loss": 2.6857,
      "nll_loss": 2.6169979572296143,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1202860176563263,
      "rewards/margins": 0.0038952501490712166,
      "rewards/rejected": -0.12418127059936523,
      "step": 277
    },
    {
      "epoch": 0.1729393468118196,
      "grad_norm": 0.44455549120903015,
      "learning_rate": 4.305e-05,
      "log_odds_chosen": 1.729311466217041,
      "log_odds_ratio": -0.3406631648540497,
      "logits/chosen": 0.18643704056739807,
      "logits/rejected": 0.041028942912817,
      "logps/chosen": -0.9607067108154297,
      "logps/rejected": -2.246669292449951,
      "loss": 3.3204,
      "nll_loss": 3.286289930343628,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09607066959142685,
      "rewards/margins": 0.1285962462425232,
      "rewards/rejected": -0.22466690838336945,
      "step": 278
    },
    {
      "epoch": 0.17356143079315708,
      "grad_norm": 0.5485425591468811,
      "learning_rate": 4.3025e-05,
      "log_odds_chosen": 1.234410285949707,
      "log_odds_ratio": -0.316910058259964,
      "logits/chosen": 0.17609338462352753,
      "logits/rejected": -0.03866102546453476,
      "logps/chosen": -1.0069231986999512,
      "logps/rejected": -1.8689861297607422,
      "loss": 3.0301,
      "nll_loss": 2.998425006866455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10069232434034348,
      "rewards/margins": 0.08620627969503403,
      "rewards/rejected": -0.1868986040353775,
      "step": 279
    },
    {
      "epoch": 0.17418351477449456,
      "grad_norm": 0.5843217372894287,
      "learning_rate": 4.3e-05,
      "log_odds_chosen": 0.3664306402206421,
      "log_odds_ratio": -0.5743635892868042,
      "logits/chosen": 0.08559656143188477,
      "logits/rejected": -0.036249175667762756,
      "logps/chosen": -1.137782335281372,
      "logps/rejected": -1.3918867111206055,
      "loss": 2.5858,
      "nll_loss": 2.528327226638794,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11377823352813721,
      "rewards/margins": 0.02541043609380722,
      "rewards/rejected": -0.13918867707252502,
      "step": 280
    },
    {
      "epoch": 0.17480559875583204,
      "grad_norm": 0.4670501947402954,
      "learning_rate": 4.2975e-05,
      "log_odds_chosen": 0.6214780211448669,
      "log_odds_ratio": -0.6075859069824219,
      "logits/chosen": 0.2213689535856247,
      "logits/rejected": 0.08010444790124893,
      "logps/chosen": -1.2412688732147217,
      "logps/rejected": -1.5694103240966797,
      "loss": 3.1539,
      "nll_loss": 3.0931215286254883,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12412689626216888,
      "rewards/margins": 0.032814137637615204,
      "rewards/rejected": -0.1569410264492035,
      "step": 281
    },
    {
      "epoch": 0.17542768273716952,
      "grad_norm": 0.38018524646759033,
      "learning_rate": 4.295e-05,
      "log_odds_chosen": 0.8761900663375854,
      "log_odds_ratio": -0.5851420164108276,
      "logits/chosen": 0.04957732930779457,
      "logits/rejected": 0.10427582263946533,
      "logps/chosen": -1.2226829528808594,
      "logps/rejected": -2.00304913520813,
      "loss": 2.5916,
      "nll_loss": 2.5330915451049805,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12226830422878265,
      "rewards/margins": 0.0780366063117981,
      "rewards/rejected": -0.20030491054058075,
      "step": 282
    },
    {
      "epoch": 0.176049766718507,
      "grad_norm": 0.4785327613353729,
      "learning_rate": 4.2925000000000007e-05,
      "log_odds_chosen": 0.7665344476699829,
      "log_odds_ratio": -0.5781853795051575,
      "logits/chosen": 0.4440539479255676,
      "logits/rejected": 0.12010644376277924,
      "logps/chosen": -1.2878354787826538,
      "logps/rejected": -1.7937930822372437,
      "loss": 3.7711,
      "nll_loss": 3.713322401046753,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12878355383872986,
      "rewards/margins": 0.050595760345458984,
      "rewards/rejected": -0.17937931418418884,
      "step": 283
    },
    {
      "epoch": 0.17667185069984448,
      "grad_norm": 0.4155225157737732,
      "learning_rate": 4.29e-05,
      "log_odds_chosen": 1.0096347332000732,
      "log_odds_ratio": -0.5321923494338989,
      "logits/chosen": 0.3086691200733185,
      "logits/rejected": 0.2555179297924042,
      "logps/chosen": -1.3326821327209473,
      "logps/rejected": -2.250950813293457,
      "loss": 3.1272,
      "nll_loss": 3.0739986896514893,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13326820731163025,
      "rewards/margins": 0.09182686358690262,
      "rewards/rejected": -0.22509507834911346,
      "step": 284
    },
    {
      "epoch": 0.17729393468118196,
      "grad_norm": 0.4170939028263092,
      "learning_rate": 4.2875000000000005e-05,
      "log_odds_chosen": 0.6225306391716003,
      "log_odds_ratio": -0.5137684345245361,
      "logits/chosen": 0.4927368760108948,
      "logits/rejected": 0.17728674411773682,
      "logps/chosen": -1.3216047286987305,
      "logps/rejected": -1.7400200366973877,
      "loss": 3.8377,
      "nll_loss": 3.786278247833252,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.132160484790802,
      "rewards/margins": 0.0418415293097496,
      "rewards/rejected": -0.1740020215511322,
      "step": 285
    },
    {
      "epoch": 0.17791601866251944,
      "grad_norm": 0.5519422888755798,
      "learning_rate": 4.285e-05,
      "log_odds_chosen": 1.4737498760223389,
      "log_odds_ratio": -0.38779395818710327,
      "logits/chosen": 0.3228255808353424,
      "logits/rejected": 0.10631482303142548,
      "logps/chosen": -1.236034631729126,
      "logps/rejected": -2.550682544708252,
      "loss": 2.8773,
      "nll_loss": 2.8385581970214844,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1236034631729126,
      "rewards/margins": 0.13146479427814484,
      "rewards/rejected": -0.25506827235221863,
      "step": 286
    },
    {
      "epoch": 0.17853810264385692,
      "grad_norm": 0.3894568979740143,
      "learning_rate": 4.2825000000000004e-05,
      "log_odds_chosen": 1.6931242942810059,
      "log_odds_ratio": -0.1958678513765335,
      "logits/chosen": 0.2403871715068817,
      "logits/rejected": 0.07207652926445007,
      "logps/chosen": -1.2004201412200928,
      "logps/rejected": -2.5714733600616455,
      "loss": 3.0662,
      "nll_loss": 3.0466134548187256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12004202604293823,
      "rewards/margins": 0.1371053159236908,
      "rewards/rejected": -0.25714734196662903,
      "step": 287
    },
    {
      "epoch": 0.1791601866251944,
      "grad_norm": 0.49171674251556396,
      "learning_rate": 4.2800000000000004e-05,
      "log_odds_chosen": 0.856521487236023,
      "log_odds_ratio": -0.4331633448600769,
      "logits/chosen": 0.26269370317459106,
      "logits/rejected": 0.2361827790737152,
      "logps/chosen": -1.312029242515564,
      "logps/rejected": -1.9908480644226074,
      "loss": 3.1928,
      "nll_loss": 3.1494884490966797,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13120292127132416,
      "rewards/margins": 0.06788188219070435,
      "rewards/rejected": -0.1990848183631897,
      "step": 288
    },
    {
      "epoch": 0.17978227060653187,
      "grad_norm": 0.4975660443305969,
      "learning_rate": 4.2775e-05,
      "log_odds_chosen": 1.012892484664917,
      "log_odds_ratio": -0.3519851267337799,
      "logits/chosen": 0.21247489750385284,
      "logits/rejected": 0.05814167857170105,
      "logps/chosen": -1.152166485786438,
      "logps/rejected": -1.9508147239685059,
      "loss": 2.5502,
      "nll_loss": 2.514974355697632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11521665006875992,
      "rewards/margins": 0.07986482977867126,
      "rewards/rejected": -0.19508148729801178,
      "step": 289
    },
    {
      "epoch": 0.18040435458786935,
      "grad_norm": 0.5224384665489197,
      "learning_rate": 4.275e-05,
      "log_odds_chosen": 1.7257745265960693,
      "log_odds_ratio": -0.24555160105228424,
      "logits/chosen": 0.47818949818611145,
      "logits/rejected": 0.06968870759010315,
      "logps/chosen": -0.8608075976371765,
      "logps/rejected": -2.147359848022461,
      "loss": 3.4919,
      "nll_loss": 3.467371940612793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08608075976371765,
      "rewards/margins": 0.12865521013736725,
      "rewards/rejected": -0.2147359699010849,
      "step": 290
    },
    {
      "epoch": 0.18102643856920683,
      "grad_norm": 0.4815826416015625,
      "learning_rate": 4.2725e-05,
      "log_odds_chosen": 1.6820149421691895,
      "log_odds_ratio": -0.23076926171779633,
      "logits/chosen": 0.33108270168304443,
      "logits/rejected": 0.07832175493240356,
      "logps/chosen": -0.956098198890686,
      "logps/rejected": -2.2002487182617188,
      "loss": 2.7417,
      "nll_loss": 2.718578338623047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09560982137918472,
      "rewards/margins": 0.12441505491733551,
      "rewards/rejected": -0.22002488374710083,
      "step": 291
    },
    {
      "epoch": 0.1816485225505443,
      "grad_norm": 0.37672296166419983,
      "learning_rate": 4.27e-05,
      "log_odds_chosen": 0.5850558876991272,
      "log_odds_ratio": -0.4817024767398834,
      "logits/chosen": 0.3864336907863617,
      "logits/rejected": 0.1432633250951767,
      "logps/chosen": -1.0847342014312744,
      "logps/rejected": -1.523407220840454,
      "loss": 3.3112,
      "nll_loss": 3.2630155086517334,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10847342759370804,
      "rewards/margins": 0.04386730492115021,
      "rewards/rejected": -0.15234072506427765,
      "step": 292
    },
    {
      "epoch": 0.1822706065318818,
      "grad_norm": 0.5808604955673218,
      "learning_rate": 4.2675e-05,
      "log_odds_chosen": 0.610927939414978,
      "log_odds_ratio": -0.4852539002895355,
      "logits/chosen": 0.35360458493232727,
      "logits/rejected": 0.008299056440591812,
      "logps/chosen": -1.268179178237915,
      "logps/rejected": -1.7629632949829102,
      "loss": 2.9865,
      "nll_loss": 2.9379255771636963,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12681792676448822,
      "rewards/margins": 0.04947839677333832,
      "rewards/rejected": -0.17629633843898773,
      "step": 293
    },
    {
      "epoch": 0.1828926905132193,
      "grad_norm": 0.412265807390213,
      "learning_rate": 4.265e-05,
      "log_odds_chosen": 1.2932488918304443,
      "log_odds_ratio": -0.3016420006752014,
      "logits/chosen": 0.3499048352241516,
      "logits/rejected": 0.12447762489318848,
      "logps/chosen": -1.0451427698135376,
      "logps/rejected": -1.9540801048278809,
      "loss": 3.1025,
      "nll_loss": 3.0723202228546143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10451428592205048,
      "rewards/margins": 0.09089374542236328,
      "rewards/rejected": -0.19540801644325256,
      "step": 294
    },
    {
      "epoch": 0.18351477449455678,
      "grad_norm": 0.44288045167922974,
      "learning_rate": 4.2625000000000006e-05,
      "log_odds_chosen": 0.4354422688484192,
      "log_odds_ratio": -0.5339393615722656,
      "logits/chosen": 0.2939504086971283,
      "logits/rejected": 0.13727524876594543,
      "logps/chosen": -1.2091028690338135,
      "logps/rejected": -1.5410089492797852,
      "loss": 2.9659,
      "nll_loss": 2.9124677181243896,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12091030180454254,
      "rewards/margins": 0.03319060429930687,
      "rewards/rejected": -0.15410089492797852,
      "step": 295
    },
    {
      "epoch": 0.18413685847589426,
      "grad_norm": 0.4726203680038452,
      "learning_rate": 4.26e-05,
      "log_odds_chosen": 0.4269523620605469,
      "log_odds_ratio": -0.6357343196868896,
      "logits/chosen": 0.42432665824890137,
      "logits/rejected": 0.09816896915435791,
      "logps/chosen": -1.2562463283538818,
      "logps/rejected": -1.4864271879196167,
      "loss": 3.435,
      "nll_loss": 3.371392250061035,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1256246566772461,
      "rewards/margins": 0.023018071427941322,
      "rewards/rejected": -0.14864271879196167,
      "step": 296
    },
    {
      "epoch": 0.18475894245723173,
      "grad_norm": 0.4871671199798584,
      "learning_rate": 4.2575000000000005e-05,
      "log_odds_chosen": 0.5284110307693481,
      "log_odds_ratio": -0.6341941356658936,
      "logits/chosen": 0.3682095408439636,
      "logits/rejected": 0.31838396191596985,
      "logps/chosen": -1.3797330856323242,
      "logps/rejected": -1.8745726346969604,
      "loss": 3.0913,
      "nll_loss": 3.0278983116149902,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13797330856323242,
      "rewards/margins": 0.04948396235704422,
      "rewards/rejected": -0.18745726346969604,
      "step": 297
    },
    {
      "epoch": 0.1853810264385692,
      "grad_norm": 0.33851662278175354,
      "learning_rate": 4.2550000000000004e-05,
      "log_odds_chosen": 0.6119697093963623,
      "log_odds_ratio": -0.6034431457519531,
      "logits/chosen": 0.28094005584716797,
      "logits/rejected": 0.2045813798904419,
      "logps/chosen": -1.17061448097229,
      "logps/rejected": -1.5927424430847168,
      "loss": 3.4087,
      "nll_loss": 3.3483834266662598,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11706145107746124,
      "rewards/margins": 0.042212799191474915,
      "rewards/rejected": -0.15927425026893616,
      "step": 298
    },
    {
      "epoch": 0.1860031104199067,
      "grad_norm": 0.49449941515922546,
      "learning_rate": 4.2525000000000004e-05,
      "log_odds_chosen": 0.40445905923843384,
      "log_odds_ratio": -0.5552644729614258,
      "logits/chosen": 0.2672854959964752,
      "logits/rejected": 0.16854549944400787,
      "logps/chosen": -1.561603307723999,
      "logps/rejected": -1.825971007347107,
      "loss": 2.7683,
      "nll_loss": 2.7127437591552734,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15616033971309662,
      "rewards/margins": 0.026436757296323776,
      "rewards/rejected": -0.1825971007347107,
      "step": 299
    },
    {
      "epoch": 0.18662519440124417,
      "grad_norm": 0.48564156889915466,
      "learning_rate": 4.25e-05,
      "log_odds_chosen": 1.5461804866790771,
      "log_odds_ratio": -0.26383474469184875,
      "logits/chosen": 0.36446020007133484,
      "logits/rejected": 0.2191128432750702,
      "logps/chosen": -1.0896272659301758,
      "logps/rejected": -2.29062557220459,
      "loss": 3.5646,
      "nll_loss": 3.5382304191589355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10896272957324982,
      "rewards/margins": 0.12009983509778976,
      "rewards/rejected": -0.22906255722045898,
      "step": 300
    },
    {
      "epoch": 0.18724727838258165,
      "grad_norm": 0.39352017641067505,
      "learning_rate": 4.2475e-05,
      "log_odds_chosen": 1.5315042734146118,
      "log_odds_ratio": -0.41960352659225464,
      "logits/chosen": 0.3345024585723877,
      "logits/rejected": 0.1347406506538391,
      "logps/chosen": -1.1930088996887207,
      "logps/rejected": -2.30621337890625,
      "loss": 3.5042,
      "nll_loss": 3.4622511863708496,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11930090188980103,
      "rewards/margins": 0.11132044345140457,
      "rewards/rejected": -0.230621337890625,
      "step": 301
    },
    {
      "epoch": 0.18786936236391913,
      "grad_norm": 0.3949545621871948,
      "learning_rate": 4.245e-05,
      "log_odds_chosen": 0.7881875038146973,
      "log_odds_ratio": -0.4549405574798584,
      "logits/chosen": 0.1703367680311203,
      "logits/rejected": 0.10550281405448914,
      "logps/chosen": -1.1531308889389038,
      "logps/rejected": -1.7436622381210327,
      "loss": 3.0154,
      "nll_loss": 2.9698915481567383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1153130903840065,
      "rewards/margins": 0.05905313044786453,
      "rewards/rejected": -0.17436623573303223,
      "step": 302
    },
    {
      "epoch": 0.1884914463452566,
      "grad_norm": 0.3909085690975189,
      "learning_rate": 4.2425e-05,
      "log_odds_chosen": 1.3018293380737305,
      "log_odds_ratio": -0.33854174613952637,
      "logits/chosen": 0.32386866211891174,
      "logits/rejected": 0.33090081810951233,
      "logps/chosen": -1.2735427618026733,
      "logps/rejected": -2.3735671043395996,
      "loss": 2.9068,
      "nll_loss": 2.872992753982544,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12735427916049957,
      "rewards/margins": 0.11000245064496994,
      "rewards/rejected": -0.23735672235488892,
      "step": 303
    },
    {
      "epoch": 0.1891135303265941,
      "grad_norm": 0.678255558013916,
      "learning_rate": 4.24e-05,
      "log_odds_chosen": 0.5951331853866577,
      "log_odds_ratio": -0.5492010116577148,
      "logits/chosen": 0.18354575335979462,
      "logits/rejected": -0.09810749441385269,
      "logps/chosen": -1.291944980621338,
      "logps/rejected": -1.642785668373108,
      "loss": 3.1579,
      "nll_loss": 3.102977752685547,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1291944980621338,
      "rewards/margins": 0.0350840725004673,
      "rewards/rejected": -0.1642785668373108,
      "step": 304
    },
    {
      "epoch": 0.18973561430793157,
      "grad_norm": 0.5240991115570068,
      "learning_rate": 4.237500000000001e-05,
      "log_odds_chosen": 3.6760783195495605,
      "log_odds_ratio": -0.180177241563797,
      "logits/chosen": 0.14156953990459442,
      "logits/rejected": -0.06919633597135544,
      "logps/chosen": -0.7791962623596191,
      "logps/rejected": -3.654845714569092,
      "loss": 3.0695,
      "nll_loss": 3.051457643508911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07791963219642639,
      "rewards/margins": 0.2875649333000183,
      "rewards/rejected": -0.3654845356941223,
      "step": 305
    },
    {
      "epoch": 0.19035769828926905,
      "grad_norm": 0.4615384042263031,
      "learning_rate": 4.235e-05,
      "log_odds_chosen": 0.5181177258491516,
      "log_odds_ratio": -0.5444080829620361,
      "logits/chosen": 0.20214281976222992,
      "logits/rejected": 0.17525801062583923,
      "logps/chosen": -1.248335838317871,
      "logps/rejected": -1.6521800756454468,
      "loss": 2.82,
      "nll_loss": 2.7655587196350098,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12483358383178711,
      "rewards/margins": 0.0403844378888607,
      "rewards/rejected": -0.16521801054477692,
      "step": 306
    },
    {
      "epoch": 0.19097978227060652,
      "grad_norm": 0.4177151620388031,
      "learning_rate": 4.2325000000000006e-05,
      "log_odds_chosen": 0.533401608467102,
      "log_odds_ratio": -0.520737886428833,
      "logits/chosen": 0.1740420013666153,
      "logits/rejected": 0.11574655026197433,
      "logps/chosen": -1.2388856410980225,
      "logps/rejected": -1.618835687637329,
      "loss": 2.9597,
      "nll_loss": 2.9076099395751953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12388856709003448,
      "rewards/margins": 0.03799499571323395,
      "rewards/rejected": -0.16188356280326843,
      "step": 307
    },
    {
      "epoch": 0.191601866251944,
      "grad_norm": 0.4279910624027252,
      "learning_rate": 4.23e-05,
      "log_odds_chosen": 1.0015525817871094,
      "log_odds_ratio": -0.4986322224140167,
      "logits/chosen": 0.4348335564136505,
      "logits/rejected": 0.1550769805908203,
      "logps/chosen": -1.2172647714614868,
      "logps/rejected": -2.026916742324829,
      "loss": 3.3799,
      "nll_loss": 3.330019950866699,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12172648310661316,
      "rewards/margins": 0.08096518367528915,
      "rewards/rejected": -0.20269165933132172,
      "step": 308
    },
    {
      "epoch": 0.19222395023328148,
      "grad_norm": 0.47602641582489014,
      "learning_rate": 4.2275000000000004e-05,
      "log_odds_chosen": 1.2695807218551636,
      "log_odds_ratio": -0.34691569209098816,
      "logits/chosen": 0.2914532721042633,
      "logits/rejected": 0.17980057001113892,
      "logps/chosen": -0.9040358662605286,
      "logps/rejected": -1.8016421794891357,
      "loss": 3.2324,
      "nll_loss": 3.1976821422576904,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09040358662605286,
      "rewards/margins": 0.08976064622402191,
      "rewards/rejected": -0.18016423285007477,
      "step": 309
    },
    {
      "epoch": 0.19284603421461896,
      "grad_norm": 0.4949467182159424,
      "learning_rate": 4.2250000000000004e-05,
      "log_odds_chosen": 0.9988858103752136,
      "log_odds_ratio": -0.4942994713783264,
      "logits/chosen": 0.2518168091773987,
      "logits/rejected": 0.07765643298625946,
      "logps/chosen": -1.2327001094818115,
      "logps/rejected": -2.0293068885803223,
      "loss": 2.9551,
      "nll_loss": 2.905709743499756,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12327000498771667,
      "rewards/margins": 0.07966066896915436,
      "rewards/rejected": -0.20293068885803223,
      "step": 310
    },
    {
      "epoch": 0.19346811819595647,
      "grad_norm": 0.4487239122390747,
      "learning_rate": 4.2225e-05,
      "log_odds_chosen": 0.5391438603401184,
      "log_odds_ratio": -0.7081095576286316,
      "logits/chosen": 0.42160993814468384,
      "logits/rejected": 0.17291218042373657,
      "logps/chosen": -1.851615309715271,
      "logps/rejected": -2.3373863697052,
      "loss": 3.9019,
      "nll_loss": 3.831056833267212,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1851615458726883,
      "rewards/margins": 0.04857710748910904,
      "rewards/rejected": -0.23373863101005554,
      "step": 311
    },
    {
      "epoch": 0.19409020217729395,
      "grad_norm": 0.47805315256118774,
      "learning_rate": 4.22e-05,
      "log_odds_chosen": 1.0429258346557617,
      "log_odds_ratio": -0.42988187074661255,
      "logits/chosen": 0.12882010638713837,
      "logits/rejected": 0.03473813459277153,
      "logps/chosen": -1.2987618446350098,
      "logps/rejected": -2.2001380920410156,
      "loss": 2.8064,
      "nll_loss": 2.7633748054504395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12987619638442993,
      "rewards/margins": 0.09013761579990387,
      "rewards/rejected": -0.2200138121843338,
      "step": 312
    },
    {
      "epoch": 0.19471228615863143,
      "grad_norm": 0.633873462677002,
      "learning_rate": 4.2175e-05,
      "log_odds_chosen": 0.6489430665969849,
      "log_odds_ratio": -0.5409046411514282,
      "logits/chosen": 0.1894546002149582,
      "logits/rejected": -0.016907554119825363,
      "logps/chosen": -1.4098745584487915,
      "logps/rejected": -1.8865466117858887,
      "loss": 3.4695,
      "nll_loss": 3.4154553413391113,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14098745584487915,
      "rewards/margins": 0.047667186707258224,
      "rewards/rejected": -0.18865466117858887,
      "step": 313
    },
    {
      "epoch": 0.1953343701399689,
      "grad_norm": 0.4453391432762146,
      "learning_rate": 4.215e-05,
      "log_odds_chosen": 1.4420627355575562,
      "log_odds_ratio": -0.4442042112350464,
      "logits/chosen": 0.13987398147583008,
      "logits/rejected": 0.13250389695167542,
      "logps/chosen": -1.3193628787994385,
      "logps/rejected": -2.651116371154785,
      "loss": 2.7875,
      "nll_loss": 2.743088722229004,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13193629682064056,
      "rewards/margins": 0.133175328373909,
      "rewards/rejected": -0.26511162519454956,
      "step": 314
    },
    {
      "epoch": 0.19595645412130638,
      "grad_norm": 0.48520854115486145,
      "learning_rate": 4.2125e-05,
      "log_odds_chosen": 1.0938706398010254,
      "log_odds_ratio": -0.3769736588001251,
      "logits/chosen": 0.14695967733860016,
      "logits/rejected": 0.03137361258268356,
      "logps/chosen": -1.062088966369629,
      "logps/rejected": -1.953197717666626,
      "loss": 2.8743,
      "nll_loss": 2.8366215229034424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10620889812707901,
      "rewards/margins": 0.08911087363958359,
      "rewards/rejected": -0.1953197717666626,
      "step": 315
    },
    {
      "epoch": 0.19657853810264386,
      "grad_norm": 0.4708939492702484,
      "learning_rate": 4.21e-05,
      "log_odds_chosen": 0.7692523002624512,
      "log_odds_ratio": -0.4789087176322937,
      "logits/chosen": 0.13523553311824799,
      "logits/rejected": 0.05263520032167435,
      "logps/chosen": -1.1547871828079224,
      "logps/rejected": -1.744168996810913,
      "loss": 3.1827,
      "nll_loss": 3.134774684906006,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11547872424125671,
      "rewards/margins": 0.05893819034099579,
      "rewards/rejected": -0.1744169145822525,
      "step": 316
    },
    {
      "epoch": 0.19720062208398134,
      "grad_norm": 0.5101141929626465,
      "learning_rate": 4.2075000000000006e-05,
      "log_odds_chosen": 0.9251267910003662,
      "log_odds_ratio": -0.420250803232193,
      "logits/chosen": 0.1897871047258377,
      "logits/rejected": 0.0760931670665741,
      "logps/chosen": -1.4300227165222168,
      "logps/rejected": -2.243600845336914,
      "loss": 3.0869,
      "nll_loss": 3.044924259185791,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14300227165222168,
      "rewards/margins": 0.08135779947042465,
      "rewards/rejected": -0.22436007857322693,
      "step": 317
    },
    {
      "epoch": 0.19782270606531882,
      "grad_norm": 0.4665040373802185,
      "learning_rate": 4.205e-05,
      "log_odds_chosen": 1.6941817998886108,
      "log_odds_ratio": -0.3107852339744568,
      "logits/chosen": 0.10233496129512787,
      "logits/rejected": -0.04454576224088669,
      "logps/chosen": -1.426357388496399,
      "logps/rejected": -2.9099607467651367,
      "loss": 3.0493,
      "nll_loss": 3.018172025680542,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14263573288917542,
      "rewards/margins": 0.14836034178733826,
      "rewards/rejected": -0.29099607467651367,
      "step": 318
    },
    {
      "epoch": 0.1984447900466563,
      "grad_norm": 0.8431224822998047,
      "learning_rate": 4.2025000000000005e-05,
      "log_odds_chosen": 0.7668442130088806,
      "log_odds_ratio": -0.489085853099823,
      "logits/chosen": 0.1494210809469223,
      "logits/rejected": 0.06121387332677841,
      "logps/chosen": -1.5238502025604248,
      "logps/rejected": -2.1793744564056396,
      "loss": 3.1432,
      "nll_loss": 3.0943074226379395,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15238502621650696,
      "rewards/margins": 0.06555242836475372,
      "rewards/rejected": -0.2179374396800995,
      "step": 319
    },
    {
      "epoch": 0.19906687402799378,
      "grad_norm": 0.6626170873641968,
      "learning_rate": 4.2e-05,
      "log_odds_chosen": 1.027713656425476,
      "log_odds_ratio": -0.37676990032196045,
      "logits/chosen": 0.16855540871620178,
      "logits/rejected": -0.08387507498264313,
      "logps/chosen": -1.222766637802124,
      "logps/rejected": -2.0757288932800293,
      "loss": 2.8647,
      "nll_loss": 2.827024459838867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1222766637802124,
      "rewards/margins": 0.08529624342918396,
      "rewards/rejected": -0.20757290720939636,
      "step": 320
    },
    {
      "epoch": 0.19968895800933126,
      "grad_norm": 0.4691254794597626,
      "learning_rate": 4.1975000000000004e-05,
      "log_odds_chosen": 1.1109933853149414,
      "log_odds_ratio": -0.37901267409324646,
      "logits/chosen": 0.14058086276054382,
      "logits/rejected": -0.0027409703470766544,
      "logps/chosen": -0.9070495367050171,
      "logps/rejected": -1.6633799076080322,
      "loss": 2.8839,
      "nll_loss": 2.846041202545166,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09070496261119843,
      "rewards/margins": 0.07563304156064987,
      "rewards/rejected": -0.1663379967212677,
      "step": 321
    },
    {
      "epoch": 0.20031104199066874,
      "grad_norm": 0.5388383865356445,
      "learning_rate": 4.195e-05,
      "log_odds_chosen": 1.4941861629486084,
      "log_odds_ratio": -0.2680909335613251,
      "logits/chosen": 0.11168316006660461,
      "logits/rejected": -0.132093608379364,
      "logps/chosen": -0.9982537031173706,
      "logps/rejected": -1.9839081764221191,
      "loss": 2.9087,
      "nll_loss": 2.881884813308716,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09982537478208542,
      "rewards/margins": 0.09856545180082321,
      "rewards/rejected": -0.19839082658290863,
      "step": 322
    },
    {
      "epoch": 0.20093312597200622,
      "grad_norm": 0.39389094710350037,
      "learning_rate": 4.1925e-05,
      "log_odds_chosen": 1.2304775714874268,
      "log_odds_ratio": -0.4389082193374634,
      "logits/chosen": 0.12545819580554962,
      "logits/rejected": 0.040004927664995193,
      "logps/chosen": -1.345543384552002,
      "logps/rejected": -2.4424848556518555,
      "loss": 3.1398,
      "nll_loss": 3.095932722091675,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13455434143543243,
      "rewards/margins": 0.10969413816928864,
      "rewards/rejected": -0.24424846470355988,
      "step": 323
    },
    {
      "epoch": 0.2015552099533437,
      "grad_norm": 0.45025521516799927,
      "learning_rate": 4.19e-05,
      "log_odds_chosen": 0.9204633831977844,
      "log_odds_ratio": -0.5545144081115723,
      "logits/chosen": 0.04150621220469475,
      "logits/rejected": -0.08123153448104858,
      "logps/chosen": -1.1536731719970703,
      "logps/rejected": -1.8100297451019287,
      "loss": 3.0471,
      "nll_loss": 2.9916718006134033,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11536730825901031,
      "rewards/margins": 0.06563566625118256,
      "rewards/rejected": -0.18100297451019287,
      "step": 324
    },
    {
      "epoch": 0.20217729393468117,
      "grad_norm": 0.4303993880748749,
      "learning_rate": 4.1875e-05,
      "log_odds_chosen": 1.6544878482818604,
      "log_odds_ratio": -0.19887760281562805,
      "logits/chosen": 0.08420052379369736,
      "logits/rejected": 0.0031083375215530396,
      "logps/chosen": -1.3029048442840576,
      "logps/rejected": -2.700894594192505,
      "loss": 3.1539,
      "nll_loss": 3.1340394020080566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13029049336910248,
      "rewards/margins": 0.13979899883270264,
      "rewards/rejected": -0.2700894773006439,
      "step": 325
    },
    {
      "epoch": 0.20279937791601865,
      "grad_norm": 1.0563331842422485,
      "learning_rate": 4.185e-05,
      "log_odds_chosen": 0.21350765228271484,
      "log_odds_ratio": -0.6822980642318726,
      "logits/chosen": -0.024682089686393738,
      "logits/rejected": -0.08352095633745193,
      "logps/chosen": -1.4727632999420166,
      "logps/rejected": -1.5891326665878296,
      "loss": 2.8543,
      "nll_loss": 2.7860758304595947,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1472763568162918,
      "rewards/margins": 0.011636918410658836,
      "rewards/rejected": -0.1589132696390152,
      "step": 326
    },
    {
      "epoch": 0.20342146189735613,
      "grad_norm": 0.48655053973197937,
      "learning_rate": 4.1825e-05,
      "log_odds_chosen": 0.36451324820518494,
      "log_odds_ratio": -0.6466756463050842,
      "logits/chosen": 0.02400357648730278,
      "logits/rejected": -0.04008547589182854,
      "logps/chosen": -1.255650520324707,
      "logps/rejected": -1.5385801792144775,
      "loss": 3.1882,
      "nll_loss": 3.1234936714172363,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1255650520324707,
      "rewards/margins": 0.02829296514391899,
      "rewards/rejected": -0.15385802090168,
      "step": 327
    },
    {
      "epoch": 0.2040435458786936,
      "grad_norm": 0.5478668212890625,
      "learning_rate": 4.18e-05,
      "log_odds_chosen": 1.6087690591812134,
      "log_odds_ratio": -0.31839194893836975,
      "logits/chosen": 0.11044462025165558,
      "logits/rejected": -0.06441525369882584,
      "logps/chosen": -1.167734146118164,
      "logps/rejected": -2.54228138923645,
      "loss": 2.9786,
      "nll_loss": 2.9467685222625732,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11677341163158417,
      "rewards/margins": 0.13745473325252533,
      "rewards/rejected": -0.2542281448841095,
      "step": 328
    },
    {
      "epoch": 0.20466562986003112,
      "grad_norm": 0.3749350309371948,
      "learning_rate": 4.1775000000000006e-05,
      "log_odds_chosen": 0.4586121439933777,
      "log_odds_ratio": -0.5654648542404175,
      "logits/chosen": 0.09604926407337189,
      "logits/rejected": -0.018427755683660507,
      "logps/chosen": -1.2352399826049805,
      "logps/rejected": -1.6107468605041504,
      "loss": 3.0382,
      "nll_loss": 2.981649875640869,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1235240027308464,
      "rewards/margins": 0.03755068778991699,
      "rewards/rejected": -0.161074697971344,
      "step": 329
    },
    {
      "epoch": 0.2052877138413686,
      "grad_norm": 0.4368545413017273,
      "learning_rate": 4.175e-05,
      "log_odds_chosen": 0.8839046955108643,
      "log_odds_ratio": -0.5126104354858398,
      "logits/chosen": -0.007048317231237888,
      "logits/rejected": -0.09785065799951553,
      "logps/chosen": -1.0619440078735352,
      "logps/rejected": -1.7408676147460938,
      "loss": 3.0543,
      "nll_loss": 3.0030674934387207,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.106194406747818,
      "rewards/margins": 0.06789234280586243,
      "rewards/rejected": -0.17408674955368042,
      "step": 330
    },
    {
      "epoch": 0.20590979782270608,
      "grad_norm": 0.5278705954551697,
      "learning_rate": 4.1725000000000005e-05,
      "log_odds_chosen": 0.30167797207832336,
      "log_odds_ratio": -0.6502044796943665,
      "logits/chosen": 0.032889269292354584,
      "logits/rejected": 0.03355567902326584,
      "logps/chosen": -1.4090936183929443,
      "logps/rejected": -1.6204454898834229,
      "loss": 3.1308,
      "nll_loss": 3.0657973289489746,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1409093588590622,
      "rewards/margins": 0.021135203540325165,
      "rewards/rejected": -0.16204456984996796,
      "step": 331
    },
    {
      "epoch": 0.20653188180404355,
      "grad_norm": 0.4414202868938446,
      "learning_rate": 4.17e-05,
      "log_odds_chosen": 1.4241623878479004,
      "log_odds_ratio": -0.3517926335334778,
      "logits/chosen": 0.11898308247327805,
      "logits/rejected": -0.013245618902146816,
      "logps/chosen": -0.9463114738464355,
      "logps/rejected": -2.077383041381836,
      "loss": 2.8483,
      "nll_loss": 2.8131093978881836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0946311503648758,
      "rewards/margins": 0.11310717463493347,
      "rewards/rejected": -0.20773832499980927,
      "step": 332
    },
    {
      "epoch": 0.20715396578538103,
      "grad_norm": 0.38851961493492126,
      "learning_rate": 4.1675e-05,
      "log_odds_chosen": 0.28314775228500366,
      "log_odds_ratio": -0.578311562538147,
      "logits/chosen": -0.008570694364607334,
      "logits/rejected": -0.05588866025209427,
      "logps/chosen": -1.1838418245315552,
      "logps/rejected": -1.3598947525024414,
      "loss": 2.8154,
      "nll_loss": 2.757525682449341,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11838418245315552,
      "rewards/margins": 0.017605291679501534,
      "rewards/rejected": -0.1359894871711731,
      "step": 333
    },
    {
      "epoch": 0.2077760497667185,
      "grad_norm": 0.6326988935470581,
      "learning_rate": 4.165e-05,
      "log_odds_chosen": 0.140260249376297,
      "log_odds_ratio": -0.677656888961792,
      "logits/chosen": 0.025349613279104233,
      "logits/rejected": -0.08781476318836212,
      "logps/chosen": -1.341933250427246,
      "logps/rejected": -1.4497263431549072,
      "loss": 2.7444,
      "nll_loss": 2.676629066467285,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1341933161020279,
      "rewards/margins": 0.010779321193695068,
      "rewards/rejected": -0.14497263729572296,
      "step": 334
    },
    {
      "epoch": 0.208398133748056,
      "grad_norm": 0.4192201793193817,
      "learning_rate": 4.1625e-05,
      "log_odds_chosen": 1.2017258405685425,
      "log_odds_ratio": -0.373602032661438,
      "logits/chosen": 0.16877499222755432,
      "logits/rejected": 0.06103990972042084,
      "logps/chosen": -1.1729239225387573,
      "logps/rejected": -1.9996774196624756,
      "loss": 3.3254,
      "nll_loss": 3.2880568504333496,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1172923892736435,
      "rewards/margins": 0.08267535269260406,
      "rewards/rejected": -0.19996774196624756,
      "step": 335
    },
    {
      "epoch": 0.20902021772939347,
      "grad_norm": 0.5239099264144897,
      "learning_rate": 4.16e-05,
      "log_odds_chosen": 0.15256771445274353,
      "log_odds_ratio": -0.6872279047966003,
      "logits/chosen": 0.0012313630431890488,
      "logits/rejected": -0.11143770813941956,
      "logps/chosen": -1.2717194557189941,
      "logps/rejected": -1.4060418605804443,
      "loss": 2.7665,
      "nll_loss": 2.6978044509887695,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12717196345329285,
      "rewards/margins": 0.013432244770228863,
      "rewards/rejected": -0.1406041979789734,
      "step": 336
    },
    {
      "epoch": 0.20964230171073095,
      "grad_norm": 0.46690982580184937,
      "learning_rate": 4.1575e-05,
      "log_odds_chosen": 1.1506333351135254,
      "log_odds_ratio": -0.396980345249176,
      "logits/chosen": 0.14569717645645142,
      "logits/rejected": -0.02970746159553528,
      "logps/chosen": -0.9054533243179321,
      "logps/rejected": -1.7300324440002441,
      "loss": 3.1932,
      "nll_loss": 3.153456926345825,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09054533392190933,
      "rewards/margins": 0.08245791494846344,
      "rewards/rejected": -0.17300325632095337,
      "step": 337
    },
    {
      "epoch": 0.21026438569206843,
      "grad_norm": 0.4816306233406067,
      "learning_rate": 4.155e-05,
      "log_odds_chosen": 1.2588977813720703,
      "log_odds_ratio": -0.38654831051826477,
      "logits/chosen": 0.1284165233373642,
      "logits/rejected": 0.061425551772117615,
      "logps/chosen": -1.5059263706207275,
      "logps/rejected": -2.580245018005371,
      "loss": 2.7022,
      "nll_loss": 2.6635427474975586,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1505926251411438,
      "rewards/margins": 0.10743187367916107,
      "rewards/rejected": -0.25802451372146606,
      "step": 338
    },
    {
      "epoch": 0.2108864696734059,
      "grad_norm": 0.5407478213310242,
      "learning_rate": 4.1525e-05,
      "log_odds_chosen": 1.0435866117477417,
      "log_odds_ratio": -0.4741755723953247,
      "logits/chosen": 0.07317960262298584,
      "logits/rejected": 0.008852528408169746,
      "logps/chosen": -1.398842215538025,
      "logps/rejected": -2.2561936378479004,
      "loss": 3.1174,
      "nll_loss": 3.0700323581695557,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13988423347473145,
      "rewards/margins": 0.08573515713214874,
      "rewards/rejected": -0.225619375705719,
      "step": 339
    },
    {
      "epoch": 0.2115085536547434,
      "grad_norm": 0.5428896546363831,
      "learning_rate": 4.15e-05,
      "log_odds_chosen": 1.3254553079605103,
      "log_odds_ratio": -0.3870584964752197,
      "logits/chosen": 0.12351932376623154,
      "logits/rejected": -0.04987223446369171,
      "logps/chosen": -1.1178048849105835,
      "logps/rejected": -2.1846683025360107,
      "loss": 2.6345,
      "nll_loss": 2.5957448482513428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11178049445152283,
      "rewards/margins": 0.10668633878231049,
      "rewards/rejected": -0.2184668481349945,
      "step": 340
    },
    {
      "epoch": 0.21213063763608087,
      "grad_norm": 0.49065813422203064,
      "learning_rate": 4.1475000000000005e-05,
      "log_odds_chosen": 0.6055381298065186,
      "log_odds_ratio": -0.5067149996757507,
      "logits/chosen": 0.11017728596925735,
      "logits/rejected": -0.005071556195616722,
      "logps/chosen": -1.3153109550476074,
      "logps/rejected": -1.7827832698822021,
      "loss": 3.0023,
      "nll_loss": 2.9516162872314453,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13153107464313507,
      "rewards/margins": 0.04674723744392395,
      "rewards/rejected": -0.17827832698822021,
      "step": 341
    },
    {
      "epoch": 0.21275272161741834,
      "grad_norm": 0.38548681139945984,
      "learning_rate": 4.145e-05,
      "log_odds_chosen": 1.6406880617141724,
      "log_odds_ratio": -0.32489141821861267,
      "logits/chosen": 0.20307300984859467,
      "logits/rejected": 0.008104506880044937,
      "logps/chosen": -0.8666763305664062,
      "logps/rejected": -2.039579391479492,
      "loss": 3.6414,
      "nll_loss": 3.608938455581665,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08666763454675674,
      "rewards/margins": 0.11729030311107635,
      "rewards/rejected": -0.2039579451084137,
      "step": 342
    },
    {
      "epoch": 0.21337480559875582,
      "grad_norm": 0.44505774974823,
      "learning_rate": 4.1425000000000004e-05,
      "log_odds_chosen": 0.9692625999450684,
      "log_odds_ratio": -0.4369995594024658,
      "logits/chosen": 0.17034012079238892,
      "logits/rejected": -0.028462085872888565,
      "logps/chosen": -1.098499059677124,
      "logps/rejected": -1.6745946407318115,
      "loss": 3.3937,
      "nll_loss": 3.34997296333313,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10984990000724792,
      "rewards/margins": 0.05760956183075905,
      "rewards/rejected": -0.16745945811271667,
      "step": 343
    },
    {
      "epoch": 0.2139968895800933,
      "grad_norm": 0.4816812574863434,
      "learning_rate": 4.14e-05,
      "log_odds_chosen": 1.5928500890731812,
      "log_odds_ratio": -0.23111592233181,
      "logits/chosen": 0.188461571931839,
      "logits/rejected": 0.022169630974531174,
      "logps/chosen": -0.9164315462112427,
      "logps/rejected": -2.1156816482543945,
      "loss": 2.7098,
      "nll_loss": 2.6867213249206543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09164315462112427,
      "rewards/margins": 0.11992499977350235,
      "rewards/rejected": -0.21156814694404602,
      "step": 344
    },
    {
      "epoch": 0.21461897356143078,
      "grad_norm": 0.39990314841270447,
      "learning_rate": 4.1375e-05,
      "log_odds_chosen": 0.49777552485466003,
      "log_odds_ratio": -0.6073991656303406,
      "logits/chosen": 0.18223583698272705,
      "logits/rejected": 0.04722006618976593,
      "logps/chosen": -1.0849920511245728,
      "logps/rejected": -1.4609473943710327,
      "loss": 3.2362,
      "nll_loss": 3.1754815578460693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10849921405315399,
      "rewards/margins": 0.03759553283452988,
      "rewards/rejected": -0.14609473943710327,
      "step": 345
    },
    {
      "epoch": 0.21524105754276826,
      "grad_norm": 0.5857303738594055,
      "learning_rate": 4.135e-05,
      "log_odds_chosen": 0.844973087310791,
      "log_odds_ratio": -0.46381595730781555,
      "logits/chosen": 0.2590794265270233,
      "logits/rejected": 0.1343073546886444,
      "logps/chosen": -1.3666094541549683,
      "logps/rejected": -2.0936036109924316,
      "loss": 3.0643,
      "nll_loss": 3.0179357528686523,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13666093349456787,
      "rewards/margins": 0.0726994052529335,
      "rewards/rejected": -0.20936036109924316,
      "step": 346
    },
    {
      "epoch": 0.21586314152410577,
      "grad_norm": 0.567475438117981,
      "learning_rate": 4.1325e-05,
      "log_odds_chosen": 0.8636181354522705,
      "log_odds_ratio": -0.40255892276763916,
      "logits/chosen": 0.20596131682395935,
      "logits/rejected": 0.061537884175777435,
      "logps/chosen": -1.3861594200134277,
      "logps/rejected": -2.1171822547912598,
      "loss": 2.8103,
      "nll_loss": 2.770002603530884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1386159509420395,
      "rewards/margins": 0.07310228049755096,
      "rewards/rejected": -0.21171823143959045,
      "step": 347
    },
    {
      "epoch": 0.21648522550544325,
      "grad_norm": 0.5971509218215942,
      "learning_rate": 4.13e-05,
      "log_odds_chosen": 2.9550795555114746,
      "log_odds_ratio": -0.1911197304725647,
      "logits/chosen": 0.29377758502960205,
      "logits/rejected": 0.28102779388427734,
      "logps/chosen": -1.3482720851898193,
      "logps/rejected": -4.058900833129883,
      "loss": 3.0978,
      "nll_loss": 3.0786807537078857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13482721149921417,
      "rewards/margins": 0.2710628807544708,
      "rewards/rejected": -0.4058900773525238,
      "step": 348
    },
    {
      "epoch": 0.21710730948678073,
      "grad_norm": 0.44494226574897766,
      "learning_rate": 4.1275e-05,
      "log_odds_chosen": 1.0941739082336426,
      "log_odds_ratio": -0.4475710391998291,
      "logits/chosen": 0.15722906589508057,
      "logits/rejected": 0.02086580917239189,
      "logps/chosen": -1.191224455833435,
      "logps/rejected": -2.0817713737487793,
      "loss": 2.9021,
      "nll_loss": 2.8573296070098877,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1191224530339241,
      "rewards/margins": 0.08905468881130219,
      "rewards/rejected": -0.20817714929580688,
      "step": 349
    },
    {
      "epoch": 0.2177293934681182,
      "grad_norm": 0.5429697632789612,
      "learning_rate": 4.125e-05,
      "log_odds_chosen": 1.2548682689666748,
      "log_odds_ratio": -0.37429577112197876,
      "logits/chosen": 0.28128620982170105,
      "logits/rejected": 0.08453608304262161,
      "logps/chosen": -1.363343596458435,
      "logps/rejected": -2.3076062202453613,
      "loss": 3.444,
      "nll_loss": 3.4065957069396973,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1363343596458435,
      "rewards/margins": 0.09442625939846039,
      "rewards/rejected": -0.2307606041431427,
      "step": 350
    },
    {
      "epoch": 0.21835147744945568,
      "grad_norm": 0.41189780831336975,
      "learning_rate": 4.1225e-05,
      "log_odds_chosen": 1.1283422708511353,
      "log_odds_ratio": -0.41111886501312256,
      "logits/chosen": 0.27242082357406616,
      "logits/rejected": 0.169078066945076,
      "logps/chosen": -1.1591994762420654,
      "logps/rejected": -2.119546413421631,
      "loss": 3.0923,
      "nll_loss": 3.0512022972106934,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11591994762420654,
      "rewards/margins": 0.0960346907377243,
      "rewards/rejected": -0.21195463836193085,
      "step": 351
    },
    {
      "epoch": 0.21897356143079316,
      "grad_norm": 0.4804132878780365,
      "learning_rate": 4.12e-05,
      "log_odds_chosen": 2.3194806575775146,
      "log_odds_ratio": -0.2600080370903015,
      "logits/chosen": 0.2595370411872864,
      "logits/rejected": 0.08692187070846558,
      "logps/chosen": -0.8231872320175171,
      "logps/rejected": -2.4552807807922363,
      "loss": 3.3194,
      "nll_loss": 3.293379306793213,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08231872320175171,
      "rewards/margins": 0.16320934891700745,
      "rewards/rejected": -0.24552807211875916,
      "step": 352
    },
    {
      "epoch": 0.21959564541213064,
      "grad_norm": 0.5321614146232605,
      "learning_rate": 4.1175000000000005e-05,
      "log_odds_chosen": 0.7400184869766235,
      "log_odds_ratio": -0.46396252512931824,
      "logits/chosen": 0.18774618208408356,
      "logits/rejected": 0.041869793087244034,
      "logps/chosen": -1.439223051071167,
      "logps/rejected": -2.082319736480713,
      "loss": 2.9231,
      "nll_loss": 2.8767247200012207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14392229914665222,
      "rewards/margins": 0.06430967152118683,
      "rewards/rejected": -0.20823195576667786,
      "step": 353
    },
    {
      "epoch": 0.22021772939346812,
      "grad_norm": 0.48990121483802795,
      "learning_rate": 4.115e-05,
      "log_odds_chosen": 1.467127799987793,
      "log_odds_ratio": -0.3306187391281128,
      "logits/chosen": 0.21457459032535553,
      "logits/rejected": -0.0008951930794864893,
      "logps/chosen": -1.305351734161377,
      "logps/rejected": -2.5240726470947266,
      "loss": 3.1787,
      "nll_loss": 3.1456456184387207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13053518533706665,
      "rewards/margins": 0.12187208235263824,
      "rewards/rejected": -0.2524072527885437,
      "step": 354
    },
    {
      "epoch": 0.2208398133748056,
      "grad_norm": 0.4076468348503113,
      "learning_rate": 4.1125000000000004e-05,
      "log_odds_chosen": 3.02394962310791,
      "log_odds_ratio": -0.1947459578514099,
      "logits/chosen": 0.15360787510871887,
      "logits/rejected": 0.13919305801391602,
      "logps/chosen": -0.9943020939826965,
      "logps/rejected": -3.5736258029937744,
      "loss": 2.964,
      "nll_loss": 2.944533348083496,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09943021088838577,
      "rewards/margins": 0.2579323649406433,
      "rewards/rejected": -0.3573625683784485,
      "step": 355
    },
    {
      "epoch": 0.22146189735614308,
      "grad_norm": 0.4909219443798065,
      "learning_rate": 4.11e-05,
      "log_odds_chosen": 1.00634765625,
      "log_odds_ratio": -0.43545883893966675,
      "logits/chosen": 0.14945323765277863,
      "logits/rejected": 0.019120551645755768,
      "logps/chosen": -1.1865514516830444,
      "logps/rejected": -2.035024642944336,
      "loss": 3.0395,
      "nll_loss": 2.995992660522461,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11865514516830444,
      "rewards/margins": 0.08484730869531631,
      "rewards/rejected": -0.20350246131420135,
      "step": 356
    },
    {
      "epoch": 0.22208398133748056,
      "grad_norm": 0.7641182541847229,
      "learning_rate": 4.1075e-05,
      "log_odds_chosen": 1.090247392654419,
      "log_odds_ratio": -0.37240341305732727,
      "logits/chosen": 0.05090628191828728,
      "logits/rejected": -0.03716772794723511,
      "logps/chosen": -1.2965291738510132,
      "logps/rejected": -2.2305068969726562,
      "loss": 2.3492,
      "nll_loss": 2.3119568824768066,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12965291738510132,
      "rewards/margins": 0.09339778125286102,
      "rewards/rejected": -0.22305069863796234,
      "step": 357
    },
    {
      "epoch": 0.22270606531881804,
      "grad_norm": 0.5538558959960938,
      "learning_rate": 4.105e-05,
      "log_odds_chosen": 0.3712920546531677,
      "log_odds_ratio": -0.6206623315811157,
      "logits/chosen": 0.120402030646801,
      "logits/rejected": -0.043820858001708984,
      "logps/chosen": -1.399167537689209,
      "logps/rejected": -1.6653398275375366,
      "loss": 2.9107,
      "nll_loss": 2.848662853240967,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13991674780845642,
      "rewards/margins": 0.02661724016070366,
      "rewards/rejected": -0.16653397679328918,
      "step": 358
    },
    {
      "epoch": 0.22332814930015552,
      "grad_norm": 0.4002559185028076,
      "learning_rate": 4.1025e-05,
      "log_odds_chosen": 0.884466290473938,
      "log_odds_ratio": -0.485196590423584,
      "logits/chosen": 0.16781805455684662,
      "logits/rejected": 0.09334772825241089,
      "logps/chosen": -1.0677001476287842,
      "logps/rejected": -1.5183968544006348,
      "loss": 3.1326,
      "nll_loss": 3.084113597869873,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10677002370357513,
      "rewards/margins": 0.04506966099143028,
      "rewards/rejected": -0.15183967351913452,
      "step": 359
    },
    {
      "epoch": 0.223950233281493,
      "grad_norm": 0.45029738545417786,
      "learning_rate": 4.1e-05,
      "log_odds_chosen": 0.6543417572975159,
      "log_odds_ratio": -0.5618710517883301,
      "logits/chosen": 0.2441825568675995,
      "logits/rejected": 0.06579945981502533,
      "logps/chosen": -1.5026423931121826,
      "logps/rejected": -2.0812129974365234,
      "loss": 3.486,
      "nll_loss": 3.4297986030578613,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15026423335075378,
      "rewards/margins": 0.05785707011818886,
      "rewards/rejected": -0.20812132954597473,
      "step": 360
    },
    {
      "epoch": 0.22457231726283047,
      "grad_norm": 0.40222129225730896,
      "learning_rate": 4.0975e-05,
      "log_odds_chosen": 0.6278929710388184,
      "log_odds_ratio": -0.5374911427497864,
      "logits/chosen": 0.1057760939002037,
      "logits/rejected": 0.1274702399969101,
      "logps/chosen": -1.3516254425048828,
      "logps/rejected": -1.9070372581481934,
      "loss": 3.0704,
      "nll_loss": 3.0166256427764893,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13516254723072052,
      "rewards/margins": 0.05554117262363434,
      "rewards/rejected": -0.19070371985435486,
      "step": 361
    },
    {
      "epoch": 0.22519440124416795,
      "grad_norm": 0.40902429819107056,
      "learning_rate": 4.095e-05,
      "log_odds_chosen": 1.3791449069976807,
      "log_odds_ratio": -0.3504829406738281,
      "logits/chosen": 0.09938657283782959,
      "logits/rejected": -0.04903809353709221,
      "logps/chosen": -0.8722810745239258,
      "logps/rejected": -1.6328226327896118,
      "loss": 3.063,
      "nll_loss": 3.027944326400757,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08722810447216034,
      "rewards/margins": 0.0760541558265686,
      "rewards/rejected": -0.16328227519989014,
      "step": 362
    },
    {
      "epoch": 0.22581648522550543,
      "grad_norm": 0.5102895498275757,
      "learning_rate": 4.0925000000000005e-05,
      "log_odds_chosen": 1.6143276691436768,
      "log_odds_ratio": -0.24948588013648987,
      "logits/chosen": 0.1433638036251068,
      "logits/rejected": 0.04843227565288544,
      "logps/chosen": -1.1994624137878418,
      "logps/rejected": -2.5450100898742676,
      "loss": 2.8281,
      "nll_loss": 2.803130626678467,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11994623392820358,
      "rewards/margins": 0.13455477356910706,
      "rewards/rejected": -0.2545010447502136,
      "step": 363
    },
    {
      "epoch": 0.2264385692068429,
      "grad_norm": 0.44875141978263855,
      "learning_rate": 4.09e-05,
      "log_odds_chosen": 0.5202021598815918,
      "log_odds_ratio": -0.5619540214538574,
      "logits/chosen": 0.09863647073507309,
      "logits/rejected": -0.04804396256804466,
      "logps/chosen": -1.4303956031799316,
      "logps/rejected": -1.8165171146392822,
      "loss": 2.7145,
      "nll_loss": 2.658271312713623,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14303956925868988,
      "rewards/margins": 0.038612157106399536,
      "rewards/rejected": -0.1816517412662506,
      "step": 364
    },
    {
      "epoch": 0.22706065318818042,
      "grad_norm": 0.4456920325756073,
      "learning_rate": 4.0875000000000004e-05,
      "log_odds_chosen": 1.450744390487671,
      "log_odds_ratio": -0.3320462107658386,
      "logits/chosen": 0.09102889895439148,
      "logits/rejected": -0.05849744379520416,
      "logps/chosen": -1.1014940738677979,
      "logps/rejected": -2.2115225791931152,
      "loss": 3.3094,
      "nll_loss": 3.2761690616607666,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11014941334724426,
      "rewards/margins": 0.11100286990404129,
      "rewards/rejected": -0.22115227580070496,
      "step": 365
    },
    {
      "epoch": 0.2276827371695179,
      "grad_norm": 0.45694804191589355,
      "learning_rate": 4.085e-05,
      "log_odds_chosen": 0.8244272470474243,
      "log_odds_ratio": -0.4660112261772156,
      "logits/chosen": 0.12547065317630768,
      "logits/rejected": -0.11931107938289642,
      "logps/chosen": -1.0817716121673584,
      "logps/rejected": -1.7403573989868164,
      "loss": 2.971,
      "nll_loss": 2.9244022369384766,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10817715525627136,
      "rewards/margins": 0.06585858762264252,
      "rewards/rejected": -0.17403574287891388,
      "step": 366
    },
    {
      "epoch": 0.22830482115085537,
      "grad_norm": 0.5407382249832153,
      "learning_rate": 4.0825e-05,
      "log_odds_chosen": 2.503481864929199,
      "log_odds_ratio": -0.13278703391551971,
      "logits/chosen": 0.10355640947818756,
      "logits/rejected": -0.04844285547733307,
      "logps/chosen": -0.958575963973999,
      "logps/rejected": -2.8335940837860107,
      "loss": 2.8018,
      "nll_loss": 2.788564682006836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09585760533809662,
      "rewards/margins": 0.18750178813934326,
      "rewards/rejected": -0.2833594083786011,
      "step": 367
    },
    {
      "epoch": 0.22892690513219285,
      "grad_norm": 0.6709921956062317,
      "learning_rate": 4.08e-05,
      "log_odds_chosen": 0.13700546324253082,
      "log_odds_ratio": -0.6914812326431274,
      "logits/chosen": 0.29757586121559143,
      "logits/rejected": 0.058456920087337494,
      "logps/chosen": -1.8119229078292847,
      "logps/rejected": -1.8780629634857178,
      "loss": 3.6408,
      "nll_loss": 3.5716493129730225,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1811922788619995,
      "rewards/margins": 0.006614033132791519,
      "rewards/rejected": -0.18780632317066193,
      "step": 368
    },
    {
      "epoch": 0.22954898911353033,
      "grad_norm": 0.5146610736846924,
      "learning_rate": 4.0775e-05,
      "log_odds_chosen": 2.349278688430786,
      "log_odds_ratio": -0.31061261892318726,
      "logits/chosen": 0.2917703092098236,
      "logits/rejected": 0.0938117578625679,
      "logps/chosen": -1.0622313022613525,
      "logps/rejected": -3.1076037883758545,
      "loss": 3.223,
      "nll_loss": 3.1919517517089844,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10622313618659973,
      "rewards/margins": 0.20453724265098572,
      "rewards/rejected": -0.31076037883758545,
      "step": 369
    },
    {
      "epoch": 0.2301710730948678,
      "grad_norm": 0.32432612776756287,
      "learning_rate": 4.075e-05,
      "log_odds_chosen": 1.531700849533081,
      "log_odds_ratio": -0.26523175835609436,
      "logits/chosen": 0.28544509410858154,
      "logits/rejected": 0.16868844628334045,
      "logps/chosen": -1.1210910081863403,
      "logps/rejected": -2.238771915435791,
      "loss": 3.6429,
      "nll_loss": 3.61639666557312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11210910975933075,
      "rewards/margins": 0.11176810413599014,
      "rewards/rejected": -0.2238771915435791,
      "step": 370
    },
    {
      "epoch": 0.2307931570762053,
      "grad_norm": 0.4330305755138397,
      "learning_rate": 4.0725e-05,
      "log_odds_chosen": 1.4429383277893066,
      "log_odds_ratio": -0.29110294580459595,
      "logits/chosen": 0.26996076107025146,
      "logits/rejected": 0.07536883652210236,
      "logps/chosen": -1.1473499536514282,
      "logps/rejected": -2.227917432785034,
      "loss": 3.2222,
      "nll_loss": 3.1930618286132812,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11473499983549118,
      "rewards/margins": 0.10805673897266388,
      "rewards/rejected": -0.22279173135757446,
      "step": 371
    },
    {
      "epoch": 0.23141524105754277,
      "grad_norm": 0.5261003375053406,
      "learning_rate": 4.07e-05,
      "log_odds_chosen": 2.001164197921753,
      "log_odds_ratio": -0.22053340077400208,
      "logits/chosen": 0.22258536517620087,
      "logits/rejected": -0.05359187722206116,
      "logps/chosen": -0.984519362449646,
      "logps/rejected": -2.612748146057129,
      "loss": 3.3361,
      "nll_loss": 3.314016342163086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09845194220542908,
      "rewards/margins": 0.1628229022026062,
      "rewards/rejected": -0.2612748146057129,
      "step": 372
    },
    {
      "epoch": 0.23203732503888025,
      "grad_norm": 0.5155344009399414,
      "learning_rate": 4.0675e-05,
      "log_odds_chosen": 0.6205844879150391,
      "log_odds_ratio": -0.536081075668335,
      "logits/chosen": 0.16748444736003876,
      "logits/rejected": 0.03588568791747093,
      "logps/chosen": -0.976515531539917,
      "logps/rejected": -1.502734899520874,
      "loss": 3.0293,
      "nll_loss": 2.9756860733032227,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09765155613422394,
      "rewards/margins": 0.052621930837631226,
      "rewards/rejected": -0.15027348697185516,
      "step": 373
    },
    {
      "epoch": 0.23265940902021773,
      "grad_norm": 0.46208807826042175,
      "learning_rate": 4.065e-05,
      "log_odds_chosen": 0.9866193532943726,
      "log_odds_ratio": -0.41447627544403076,
      "logits/chosen": 0.19046258926391602,
      "logits/rejected": 0.1209864392876625,
      "logps/chosen": -0.9953656196594238,
      "logps/rejected": -1.6837455034255981,
      "loss": 3.1588,
      "nll_loss": 3.1173107624053955,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09953656792640686,
      "rewards/margins": 0.0688379779458046,
      "rewards/rejected": -0.16837453842163086,
      "step": 374
    },
    {
      "epoch": 0.2332814930015552,
      "grad_norm": 0.5262720584869385,
      "learning_rate": 4.0625000000000005e-05,
      "log_odds_chosen": 1.1617014408111572,
      "log_odds_ratio": -0.3742392063140869,
      "logits/chosen": 0.2684021592140198,
      "logits/rejected": 0.0713948905467987,
      "logps/chosen": -1.4048610925674438,
      "logps/rejected": -2.383072853088379,
      "loss": 3.3322,
      "nll_loss": 3.29473876953125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14048609137535095,
      "rewards/margins": 0.0978211984038353,
      "rewards/rejected": -0.23830729722976685,
      "step": 375
    },
    {
      "epoch": 0.23390357698289269,
      "grad_norm": 0.4657245874404907,
      "learning_rate": 4.0600000000000004e-05,
      "log_odds_chosen": 1.0480836629867554,
      "log_odds_ratio": -0.3715401887893677,
      "logits/chosen": 0.17920681834220886,
      "logits/rejected": -0.03246612474322319,
      "logps/chosen": -1.4259849786758423,
      "logps/rejected": -2.313983201980591,
      "loss": 3.0117,
      "nll_loss": 2.9745419025421143,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14259850978851318,
      "rewards/margins": 0.08879982680082321,
      "rewards/rejected": -0.2313983291387558,
      "step": 376
    },
    {
      "epoch": 0.23452566096423016,
      "grad_norm": 0.4098886549472809,
      "learning_rate": 4.0575000000000004e-05,
      "log_odds_chosen": 0.9914601445198059,
      "log_odds_ratio": -0.43344154953956604,
      "logits/chosen": 0.24701879918575287,
      "logits/rejected": 0.019207902252674103,
      "logps/chosen": -1.3967047929763794,
      "logps/rejected": -2.260988235473633,
      "loss": 3.6364,
      "nll_loss": 3.5931003093719482,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1396704763174057,
      "rewards/margins": 0.08642837405204773,
      "rewards/rejected": -0.22609883546829224,
      "step": 377
    },
    {
      "epoch": 0.23514774494556764,
      "grad_norm": 0.47420018911361694,
      "learning_rate": 4.055e-05,
      "log_odds_chosen": 2.0201539993286133,
      "log_odds_ratio": -0.2169860154390335,
      "logits/chosen": 0.25134286284446716,
      "logits/rejected": 0.0921395868062973,
      "logps/chosen": -1.110857605934143,
      "logps/rejected": -2.820281744003296,
      "loss": 2.9052,
      "nll_loss": 2.8835182189941406,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11108576506376266,
      "rewards/margins": 0.17094242572784424,
      "rewards/rejected": -0.2820281982421875,
      "step": 378
    },
    {
      "epoch": 0.23576982892690512,
      "grad_norm": 0.6942557096481323,
      "learning_rate": 4.0525e-05,
      "log_odds_chosen": 1.0329240560531616,
      "log_odds_ratio": -0.7378867268562317,
      "logits/chosen": 0.11842384189367294,
      "logits/rejected": 0.05716584250330925,
      "logps/chosen": -1.7523798942565918,
      "logps/rejected": -2.5439324378967285,
      "loss": 2.6703,
      "nll_loss": 2.596536159515381,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1752379834651947,
      "rewards/margins": 0.07915525138378143,
      "rewards/rejected": -0.25439321994781494,
      "step": 379
    },
    {
      "epoch": 0.2363919129082426,
      "grad_norm": 0.49551472067832947,
      "learning_rate": 4.05e-05,
      "log_odds_chosen": 1.2499074935913086,
      "log_odds_ratio": -0.4396931529045105,
      "logits/chosen": 0.3040961027145386,
      "logits/rejected": 0.13865968585014343,
      "logps/chosen": -1.3086928129196167,
      "logps/rejected": -2.284111499786377,
      "loss": 3.2365,
      "nll_loss": 3.192500114440918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1308692842721939,
      "rewards/margins": 0.09754188358783722,
      "rewards/rejected": -0.22841116786003113,
      "step": 380
    },
    {
      "epoch": 0.23701399688958008,
      "grad_norm": 0.4651765823364258,
      "learning_rate": 4.0475e-05,
      "log_odds_chosen": 0.9740829467773438,
      "log_odds_ratio": -0.4101206362247467,
      "logits/chosen": 0.18327367305755615,
      "logits/rejected": -0.007276562973856926,
      "logps/chosen": -1.0384430885314941,
      "logps/rejected": -1.7929588556289673,
      "loss": 2.9892,
      "nll_loss": 2.9481401443481445,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10384431481361389,
      "rewards/margins": 0.0754515752196312,
      "rewards/rejected": -0.17929589748382568,
      "step": 381
    },
    {
      "epoch": 0.2376360808709176,
      "grad_norm": 0.4907761216163635,
      "learning_rate": 4.045000000000001e-05,
      "log_odds_chosen": 0.9015824794769287,
      "log_odds_ratio": -0.4331977963447571,
      "logits/chosen": 0.11043369770050049,
      "logits/rejected": 0.11448623239994049,
      "logps/chosen": -1.1956536769866943,
      "logps/rejected": -1.9301912784576416,
      "loss": 3.053,
      "nll_loss": 3.009702682495117,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11956537514925003,
      "rewards/margins": 0.07345376163721085,
      "rewards/rejected": -0.19301912188529968,
      "step": 382
    },
    {
      "epoch": 0.23825816485225507,
      "grad_norm": 0.45944109559059143,
      "learning_rate": 4.0425e-05,
      "log_odds_chosen": 0.9134513735771179,
      "log_odds_ratio": -0.4373936057090759,
      "logits/chosen": 0.05043463408946991,
      "logits/rejected": 0.10286575555801392,
      "logps/chosen": -1.2276078462600708,
      "logps/rejected": -1.9750827550888062,
      "loss": 2.8257,
      "nll_loss": 2.781928777694702,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12276078015565872,
      "rewards/margins": 0.0747474879026413,
      "rewards/rejected": -0.19750827550888062,
      "step": 383
    },
    {
      "epoch": 0.23888024883359255,
      "grad_norm": 0.5201630592346191,
      "learning_rate": 4.0400000000000006e-05,
      "log_odds_chosen": 2.114027738571167,
      "log_odds_ratio": -0.15464414656162262,
      "logits/chosen": 0.2518041431903839,
      "logits/rejected": 0.06087159365415573,
      "logps/chosen": -0.7840594053268433,
      "logps/rejected": -2.2629692554473877,
      "loss": 3.3323,
      "nll_loss": 3.316868782043457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0784059390425682,
      "rewards/margins": 0.14789099991321564,
      "rewards/rejected": -0.22629694640636444,
      "step": 384
    },
    {
      "epoch": 0.23950233281493002,
      "grad_norm": 0.44370707869529724,
      "learning_rate": 4.0375e-05,
      "log_odds_chosen": 1.3410742282867432,
      "log_odds_ratio": -0.322849839925766,
      "logits/chosen": -0.04521143063902855,
      "logits/rejected": -0.05069145932793617,
      "logps/chosen": -1.05382239818573,
      "logps/rejected": -2.046505928039551,
      "loss": 2.6729,
      "nll_loss": 2.6405997276306152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10538224130868912,
      "rewards/margins": 0.0992683619260788,
      "rewards/rejected": -0.2046506106853485,
      "step": 385
    },
    {
      "epoch": 0.2401244167962675,
      "grad_norm": 0.40675365924835205,
      "learning_rate": 4.0350000000000005e-05,
      "log_odds_chosen": 1.3835175037384033,
      "log_odds_ratio": -0.26051661372184753,
      "logits/chosen": 0.23836039006710052,
      "logits/rejected": 0.06631730496883392,
      "logps/chosen": -1.0143346786499023,
      "logps/rejected": -2.0367674827575684,
      "loss": 3.5034,
      "nll_loss": 3.4773647785186768,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10143347829580307,
      "rewards/margins": 0.10224328190088272,
      "rewards/rejected": -0.2036767452955246,
      "step": 386
    },
    {
      "epoch": 0.24074650077760498,
      "grad_norm": 0.46325162053108215,
      "learning_rate": 4.0325000000000004e-05,
      "log_odds_chosen": 0.7983840107917786,
      "log_odds_ratio": -0.5282416939735413,
      "logits/chosen": 0.2139953374862671,
      "logits/rejected": 0.19283059239387512,
      "logps/chosen": -1.2492709159851074,
      "logps/rejected": -1.9619114398956299,
      "loss": 2.8089,
      "nll_loss": 2.756054639816284,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1249270886182785,
      "rewards/margins": 0.07126405835151672,
      "rewards/rejected": -0.19619114696979523,
      "step": 387
    },
    {
      "epoch": 0.24136858475894246,
      "grad_norm": 0.5103418231010437,
      "learning_rate": 4.0300000000000004e-05,
      "log_odds_chosen": 1.5594842433929443,
      "log_odds_ratio": -0.2448807656764984,
      "logits/chosen": 0.17046277225017548,
      "logits/rejected": 0.07869112491607666,
      "logps/chosen": -1.0376362800598145,
      "logps/rejected": -2.278496503829956,
      "loss": 2.7251,
      "nll_loss": 2.7006213665008545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1037636250257492,
      "rewards/margins": 0.12408602982759476,
      "rewards/rejected": -0.22784966230392456,
      "step": 388
    },
    {
      "epoch": 0.24199066874027994,
      "grad_norm": 0.3780629634857178,
      "learning_rate": 4.0275e-05,
      "log_odds_chosen": 1.3771535158157349,
      "log_odds_ratio": -0.3319317102432251,
      "logits/chosen": 0.32596805691719055,
      "logits/rejected": 0.1879710555076599,
      "logps/chosen": -1.2158101797103882,
      "logps/rejected": -2.255094289779663,
      "loss": 3.3506,
      "nll_loss": 3.3174211978912354,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12158101797103882,
      "rewards/margins": 0.10392840951681137,
      "rewards/rejected": -0.22550944983959198,
      "step": 389
    },
    {
      "epoch": 0.24261275272161742,
      "grad_norm": 0.4106887876987457,
      "learning_rate": 4.025e-05,
      "log_odds_chosen": 0.6944050788879395,
      "log_odds_ratio": -0.46452128887176514,
      "logits/chosen": 0.10568028688430786,
      "logits/rejected": 0.0720917209982872,
      "logps/chosen": -1.2601232528686523,
      "logps/rejected": -1.7703832387924194,
      "loss": 2.8362,
      "nll_loss": 2.789783239364624,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12601232528686523,
      "rewards/margins": 0.051026009023189545,
      "rewards/rejected": -0.17703832685947418,
      "step": 390
    },
    {
      "epoch": 0.2432348367029549,
      "grad_norm": 0.39343398809432983,
      "learning_rate": 4.0225e-05,
      "log_odds_chosen": 0.9674738645553589,
      "log_odds_ratio": -0.5284614562988281,
      "logits/chosen": 0.10175783932209015,
      "logits/rejected": 0.03822772949934006,
      "logps/chosen": -1.2551448345184326,
      "logps/rejected": -2.0882956981658936,
      "loss": 3.0856,
      "nll_loss": 3.0327351093292236,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12551449239253998,
      "rewards/margins": 0.08331510424613953,
      "rewards/rejected": -0.2088295817375183,
      "step": 391
    },
    {
      "epoch": 0.24385692068429238,
      "grad_norm": 0.4384991228580475,
      "learning_rate": 4.02e-05,
      "log_odds_chosen": 0.7777218818664551,
      "log_odds_ratio": -0.4195319414138794,
      "logits/chosen": 0.3055810332298279,
      "logits/rejected": 0.05448678508400917,
      "logps/chosen": -1.110863447189331,
      "logps/rejected": -1.6843622922897339,
      "loss": 3.2888,
      "nll_loss": 3.2468209266662598,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11108633875846863,
      "rewards/margins": 0.05734989792108536,
      "rewards/rejected": -0.1684362292289734,
      "step": 392
    },
    {
      "epoch": 0.24447900466562986,
      "grad_norm": 0.5314688682556152,
      "learning_rate": 4.0175e-05,
      "log_odds_chosen": 0.9773632884025574,
      "log_odds_ratio": -0.4269845485687256,
      "logits/chosen": 0.17569248378276825,
      "logits/rejected": 0.1796281337738037,
      "logps/chosen": -1.25931978225708,
      "logps/rejected": -2.1036195755004883,
      "loss": 2.7412,
      "nll_loss": 2.6984646320343018,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.125931978225708,
      "rewards/margins": 0.08442999422550201,
      "rewards/rejected": -0.21036197245121002,
      "step": 393
    },
    {
      "epoch": 0.24510108864696734,
      "grad_norm": 0.5598838329315186,
      "learning_rate": 4.015000000000001e-05,
      "log_odds_chosen": 1.459074854850769,
      "log_odds_ratio": -0.30608847737312317,
      "logits/chosen": 0.1628699004650116,
      "logits/rejected": 0.10045505315065384,
      "logps/chosen": -1.0321791172027588,
      "logps/rejected": -2.095522880554199,
      "loss": 2.6775,
      "nll_loss": 2.646920680999756,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10321791470050812,
      "rewards/margins": 0.10633436590433121,
      "rewards/rejected": -0.20955227315425873,
      "step": 394
    },
    {
      "epoch": 0.24572317262830481,
      "grad_norm": 0.31368446350097656,
      "learning_rate": 4.0125e-05,
      "log_odds_chosen": 1.034085750579834,
      "log_odds_ratio": -0.4138377904891968,
      "logits/chosen": 0.2146773636341095,
      "logits/rejected": 0.06570446491241455,
      "logps/chosen": -0.9849491715431213,
      "logps/rejected": -1.6467113494873047,
      "loss": 3.8255,
      "nll_loss": 3.784114360809326,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09849491715431213,
      "rewards/margins": 0.06617622822523117,
      "rewards/rejected": -0.1646711528301239,
      "step": 395
    },
    {
      "epoch": 0.2463452566096423,
      "grad_norm": 0.5984493494033813,
      "learning_rate": 4.0100000000000006e-05,
      "log_odds_chosen": 1.0793639421463013,
      "log_odds_ratio": -0.47389787435531616,
      "logits/chosen": 0.25743645429611206,
      "logits/rejected": 0.023621711879968643,
      "logps/chosen": -1.2558321952819824,
      "logps/rejected": -2.103111505508423,
      "loss": 3.2582,
      "nll_loss": 3.2107701301574707,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.125583216547966,
      "rewards/margins": 0.084727942943573,
      "rewards/rejected": -0.210311159491539,
      "step": 396
    },
    {
      "epoch": 0.24696734059097977,
      "grad_norm": 0.37724167108535767,
      "learning_rate": 4.0075e-05,
      "log_odds_chosen": 0.8990364670753479,
      "log_odds_ratio": -0.4033519923686981,
      "logits/chosen": 0.26665544509887695,
      "logits/rejected": 0.15528859198093414,
      "logps/chosen": -1.1553454399108887,
      "logps/rejected": -1.854149580001831,
      "loss": 3.484,
      "nll_loss": 3.443643093109131,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11553453654050827,
      "rewards/margins": 0.0698804259300232,
      "rewards/rejected": -0.18541496992111206,
      "step": 397
    },
    {
      "epoch": 0.24758942457231725,
      "grad_norm": 0.8459545373916626,
      "learning_rate": 4.0050000000000004e-05,
      "log_odds_chosen": 1.516906976699829,
      "log_odds_ratio": -0.2759634852409363,
      "logits/chosen": 0.23380643129348755,
      "logits/rejected": 0.14687879383563995,
      "logps/chosen": -1.2292195558547974,
      "logps/rejected": -2.45884108543396,
      "loss": 2.9524,
      "nll_loss": 2.924772024154663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12292195856571198,
      "rewards/margins": 0.12296216189861298,
      "rewards/rejected": -0.24588412046432495,
      "step": 398
    },
    {
      "epoch": 0.24821150855365473,
      "grad_norm": 0.7587859630584717,
      "learning_rate": 4.0025000000000004e-05,
      "log_odds_chosen": 1.5884180068969727,
      "log_odds_ratio": -0.3095462918281555,
      "logits/chosen": 0.18802863359451294,
      "logits/rejected": -0.01595812290906906,
      "logps/chosen": -1.3920193910598755,
      "logps/rejected": -2.773435115814209,
      "loss": 3.247,
      "nll_loss": 3.2160043716430664,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13920193910598755,
      "rewards/margins": 0.13814157247543335,
      "rewards/rejected": -0.2773435115814209,
      "step": 399
    },
    {
      "epoch": 0.24883359253499224,
      "grad_norm": 0.7157498598098755,
      "learning_rate": 4e-05,
      "log_odds_chosen": 0.774174690246582,
      "log_odds_ratio": -0.6406145691871643,
      "logits/chosen": 0.3137771189212799,
      "logits/rejected": 0.028129838407039642,
      "logps/chosen": -1.4273900985717773,
      "logps/rejected": -2.0852622985839844,
      "loss": 4.025,
      "nll_loss": 3.9609427452087402,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14273901283740997,
      "rewards/margins": 0.06578721851110458,
      "rewards/rejected": -0.20852622389793396,
      "step": 400
    },
    {
      "epoch": 0.24945567651632972,
      "grad_norm": 0.6268599033355713,
      "learning_rate": 3.9975e-05,
      "log_odds_chosen": 1.2294279336929321,
      "log_odds_ratio": -0.4779622554779053,
      "logits/chosen": 0.16509678959846497,
      "logits/rejected": 0.07056587189435959,
      "logps/chosen": -1.2676770687103271,
      "logps/rejected": -2.3259618282318115,
      "loss": 3.3218,
      "nll_loss": 3.274020195007324,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12676770985126495,
      "rewards/margins": 0.10582847893238068,
      "rewards/rejected": -0.23259618878364563,
      "step": 401
    },
    {
      "epoch": 0.25007776049766717,
      "grad_norm": 0.5052048563957214,
      "learning_rate": 3.995e-05,
      "log_odds_chosen": 1.3162705898284912,
      "log_odds_ratio": -0.32979390025138855,
      "logits/chosen": 0.14668475091457367,
      "logits/rejected": 0.01875954680144787,
      "logps/chosen": -1.279245138168335,
      "logps/rejected": -2.367394208908081,
      "loss": 3.0466,
      "nll_loss": 3.013622522354126,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12792451679706573,
      "rewards/margins": 0.10881491005420685,
      "rewards/rejected": -0.23673942685127258,
      "step": 402
    },
    {
      "epoch": 0.2506998444790047,
      "grad_norm": 0.586941659450531,
      "learning_rate": 3.9925e-05,
      "log_odds_chosen": 0.97333163022995,
      "log_odds_ratio": -0.4262157678604126,
      "logits/chosen": 0.199526846408844,
      "logits/rejected": 0.12170709669589996,
      "logps/chosen": -1.0942950248718262,
      "logps/rejected": -1.8334949016571045,
      "loss": 2.9178,
      "nll_loss": 2.8751866817474365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1094295084476471,
      "rewards/margins": 0.07391998916864395,
      "rewards/rejected": -0.18334949016571045,
      "step": 403
    },
    {
      "epoch": 0.2513219284603421,
      "grad_norm": 0.7138959765434265,
      "learning_rate": 3.99e-05,
      "log_odds_chosen": 1.5978549718856812,
      "log_odds_ratio": -0.6236208081245422,
      "logits/chosen": 0.08655344694852829,
      "logits/rejected": 0.06523916870355606,
      "logps/chosen": -1.0683348178863525,
      "logps/rejected": -2.3083322048187256,
      "loss": 3.0347,
      "nll_loss": 2.972341537475586,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10683348774909973,
      "rewards/margins": 0.12399972975254059,
      "rewards/rejected": -0.2308332324028015,
      "step": 404
    },
    {
      "epoch": 0.25194401244167963,
      "grad_norm": 0.4803338050842285,
      "learning_rate": 3.9875e-05,
      "log_odds_chosen": 0.7191067337989807,
      "log_odds_ratio": -0.4708095192909241,
      "logits/chosen": 0.049791183322668076,
      "logits/rejected": -0.03897171467542648,
      "logps/chosen": -1.1012663841247559,
      "logps/rejected": -1.5942356586456299,
      "loss": 3.0085,
      "nll_loss": 2.961468458175659,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11012664437294006,
      "rewards/margins": 0.049296922981739044,
      "rewards/rejected": -0.1594235748052597,
      "step": 405
    },
    {
      "epoch": 0.2525660964230171,
      "grad_norm": 0.5744175314903259,
      "learning_rate": 3.9850000000000006e-05,
      "log_odds_chosen": 0.09449946880340576,
      "log_odds_ratio": -0.6956384778022766,
      "logits/chosen": 0.15730737149715424,
      "logits/rejected": -0.03350464627146721,
      "logps/chosen": -1.461653709411621,
      "logps/rejected": -1.5220292806625366,
      "loss": 3.3611,
      "nll_loss": 3.2915573120117188,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1461653709411621,
      "rewards/margins": 0.006037542596459389,
      "rewards/rejected": -0.15220291912555695,
      "step": 406
    },
    {
      "epoch": 0.2531881804043546,
      "grad_norm": 0.4293562173843384,
      "learning_rate": 3.9825e-05,
      "log_odds_chosen": 0.5901978611946106,
      "log_odds_ratio": -0.5310933589935303,
      "logits/chosen": 0.02693868800997734,
      "logits/rejected": -0.0009972341358661652,
      "logps/chosen": -1.046227216720581,
      "logps/rejected": -1.3914040327072144,
      "loss": 2.7241,
      "nll_loss": 2.670952320098877,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1046227216720581,
      "rewards/margins": 0.03451768681406975,
      "rewards/rejected": -0.13914041221141815,
      "step": 407
    },
    {
      "epoch": 0.25381026438569204,
      "grad_norm": 1.1754698753356934,
      "learning_rate": 3.9800000000000005e-05,
      "log_odds_chosen": 1.0236449241638184,
      "log_odds_ratio": -0.46178126335144043,
      "logits/chosen": 0.1028258204460144,
      "logits/rejected": -0.047515541315078735,
      "logps/chosen": -1.7946947813034058,
      "logps/rejected": -2.7080202102661133,
      "loss": 2.8661,
      "nll_loss": 2.8199520111083984,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17946948111057281,
      "rewards/margins": 0.09133254736661911,
      "rewards/rejected": -0.27080202102661133,
      "step": 408
    },
    {
      "epoch": 0.25443234836702955,
      "grad_norm": 0.4363967180252075,
      "learning_rate": 3.9775e-05,
      "log_odds_chosen": 1.6175098419189453,
      "log_odds_ratio": -0.37422215938568115,
      "logits/chosen": 0.22522154450416565,
      "logits/rejected": 0.0489632673561573,
      "logps/chosen": -0.9020571112632751,
      "logps/rejected": -1.936967134475708,
      "loss": 3.776,
      "nll_loss": 3.7385733127593994,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09020572155714035,
      "rewards/margins": 0.10349100083112717,
      "rewards/rejected": -0.1936967372894287,
      "step": 409
    },
    {
      "epoch": 0.25505443234836706,
      "grad_norm": 0.594385027885437,
      "learning_rate": 3.9750000000000004e-05,
      "log_odds_chosen": 1.2434051036834717,
      "log_odds_ratio": -0.32507485151290894,
      "logits/chosen": 0.14571481943130493,
      "logits/rejected": 0.02298334613442421,
      "logps/chosen": -1.0427602529525757,
      "logps/rejected": -1.886343240737915,
      "loss": 3.0004,
      "nll_loss": 2.9678754806518555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10427601635456085,
      "rewards/margins": 0.08435830473899841,
      "rewards/rejected": -0.18863432109355927,
      "step": 410
    },
    {
      "epoch": 0.2556765163297045,
      "grad_norm": 0.41299790143966675,
      "learning_rate": 3.9725e-05,
      "log_odds_chosen": 0.741004228591919,
      "log_odds_ratio": -0.4465721845626831,
      "logits/chosen": 0.18152594566345215,
      "logits/rejected": 0.13067328929901123,
      "logps/chosen": -1.281395435333252,
      "logps/rejected": -1.903831958770752,
      "loss": 3.0622,
      "nll_loss": 3.0175766944885254,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12813954055309296,
      "rewards/margins": 0.06224365532398224,
      "rewards/rejected": -0.1903831958770752,
      "step": 411
    },
    {
      "epoch": 0.256298600311042,
      "grad_norm": 0.6275978088378906,
      "learning_rate": 3.97e-05,
      "log_odds_chosen": 0.0313977487385273,
      "log_odds_ratio": -0.7415766716003418,
      "logits/chosen": -0.03772881627082825,
      "logits/rejected": 0.12028350681066513,
      "logps/chosen": -1.4508265256881714,
      "logps/rejected": -1.4406688213348389,
      "loss": 2.3823,
      "nll_loss": 2.30818772315979,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14508265256881714,
      "rewards/margins": -0.001015770249068737,
      "rewards/rejected": -0.14406687021255493,
      "step": 412
    },
    {
      "epoch": 0.25692068429237946,
      "grad_norm": 0.4781981408596039,
      "learning_rate": 3.9675e-05,
      "log_odds_chosen": 1.5340921878814697,
      "log_odds_ratio": -0.28959184885025024,
      "logits/chosen": 0.34700697660446167,
      "logits/rejected": 0.2925771176815033,
      "logps/chosen": -1.3058507442474365,
      "logps/rejected": -2.6288981437683105,
      "loss": 3.491,
      "nll_loss": 3.4620461463928223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13058508932590485,
      "rewards/margins": 0.13230472803115845,
      "rewards/rejected": -0.2628898024559021,
      "step": 413
    },
    {
      "epoch": 0.25754276827371697,
      "grad_norm": 0.494865357875824,
      "learning_rate": 3.965e-05,
      "log_odds_chosen": 1.5314979553222656,
      "log_odds_ratio": -0.25367340445518494,
      "logits/chosen": 0.18154457211494446,
      "logits/rejected": 0.1370812952518463,
      "logps/chosen": -0.9703949689865112,
      "logps/rejected": -1.9872558116912842,
      "loss": 3.0317,
      "nll_loss": 3.006338119506836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09703950583934784,
      "rewards/margins": 0.10168607532978058,
      "rewards/rejected": -0.19872558116912842,
      "step": 414
    },
    {
      "epoch": 0.2581648522550544,
      "grad_norm": 0.45149925351142883,
      "learning_rate": 3.9625e-05,
      "log_odds_chosen": 0.49176663160324097,
      "log_odds_ratio": -0.6137273907661438,
      "logits/chosen": 0.19695249199867249,
      "logits/rejected": 0.18039798736572266,
      "logps/chosen": -1.330711841583252,
      "logps/rejected": -1.8058357238769531,
      "loss": 3.122,
      "nll_loss": 3.0606541633605957,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1330711841583252,
      "rewards/margins": 0.047512397170066833,
      "rewards/rejected": -0.18058358132839203,
      "step": 415
    },
    {
      "epoch": 0.25878693623639193,
      "grad_norm": 0.4677426517009735,
      "learning_rate": 3.960000000000001e-05,
      "log_odds_chosen": 1.4863536357879639,
      "log_odds_ratio": -0.3277415931224823,
      "logits/chosen": 0.3289963901042938,
      "logits/rejected": 0.295125275850296,
      "logps/chosen": -1.3954858779907227,
      "logps/rejected": -2.662785530090332,
      "loss": 3.3443,
      "nll_loss": 3.3114874362945557,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13954858481884003,
      "rewards/margins": 0.12672998011112213,
      "rewards/rejected": -0.26627856492996216,
      "step": 416
    },
    {
      "epoch": 0.2594090202177294,
      "grad_norm": 0.5604446530342102,
      "learning_rate": 3.9575e-05,
      "log_odds_chosen": 1.654098629951477,
      "log_odds_ratio": -0.34370043873786926,
      "logits/chosen": 0.44623494148254395,
      "logits/rejected": 0.26756906509399414,
      "logps/chosen": -1.5715718269348145,
      "logps/rejected": -3.0832409858703613,
      "loss": 3.74,
      "nll_loss": 3.7056450843811035,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15715718269348145,
      "rewards/margins": 0.1511669158935547,
      "rewards/rejected": -0.30832409858703613,
      "step": 417
    },
    {
      "epoch": 0.2600311041990669,
      "grad_norm": 0.5016684532165527,
      "learning_rate": 3.9550000000000006e-05,
      "log_odds_chosen": 1.0671627521514893,
      "log_odds_ratio": -0.34129106998443604,
      "logits/chosen": 0.09102344512939453,
      "logits/rejected": 0.1018139123916626,
      "logps/chosen": -1.1463091373443604,
      "logps/rejected": -1.9536468982696533,
      "loss": 2.8157,
      "nll_loss": 2.781564474105835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11463090777397156,
      "rewards/margins": 0.0807337760925293,
      "rewards/rejected": -0.19536468386650085,
      "step": 418
    },
    {
      "epoch": 0.26065318818040434,
      "grad_norm": 0.9377952218055725,
      "learning_rate": 3.9525e-05,
      "log_odds_chosen": 2.2686891555786133,
      "log_odds_ratio": -0.14280670881271362,
      "logits/chosen": 0.29692742228507996,
      "logits/rejected": 0.15223044157028198,
      "logps/chosen": -1.185152292251587,
      "logps/rejected": -3.158766269683838,
      "loss": 2.8944,
      "nll_loss": 2.880141258239746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11851523816585541,
      "rewards/margins": 0.19736140966415405,
      "rewards/rejected": -0.31587663292884827,
      "step": 419
    },
    {
      "epoch": 0.26127527216174184,
      "grad_norm": 0.5251947045326233,
      "learning_rate": 3.9500000000000005e-05,
      "log_odds_chosen": 1.5507677793502808,
      "log_odds_ratio": -0.3057706654071808,
      "logits/chosen": 0.32404136657714844,
      "logits/rejected": 0.18444237112998962,
      "logps/chosen": -1.1873574256896973,
      "logps/rejected": -2.446171522140503,
      "loss": 3.1758,
      "nll_loss": 3.1452338695526123,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11873574554920197,
      "rewards/margins": 0.1258814036846161,
      "rewards/rejected": -0.24461714923381805,
      "step": 420
    },
    {
      "epoch": 0.2618973561430793,
      "grad_norm": 0.4308774471282959,
      "learning_rate": 3.9475000000000004e-05,
      "log_odds_chosen": 0.3947753608226776,
      "log_odds_ratio": -0.5612890124320984,
      "logits/chosen": 0.13869617879390717,
      "logits/rejected": 0.044950664043426514,
      "logps/chosen": -1.2930006980895996,
      "logps/rejected": -1.6258165836334229,
      "loss": 2.9868,
      "nll_loss": 2.9306952953338623,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.129300057888031,
      "rewards/margins": 0.0332816019654274,
      "rewards/rejected": -0.162581667304039,
      "step": 421
    },
    {
      "epoch": 0.2625194401244168,
      "grad_norm": 0.5830161571502686,
      "learning_rate": 3.9450000000000003e-05,
      "log_odds_chosen": 1.1204309463500977,
      "log_odds_ratio": -0.3383479714393616,
      "logits/chosen": 0.07506772875785828,
      "logits/rejected": -0.03575318679213524,
      "logps/chosen": -1.3158361911773682,
      "logps/rejected": -2.236865997314453,
      "loss": 2.7736,
      "nll_loss": 2.7397449016571045,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13158363103866577,
      "rewards/margins": 0.09210299700498581,
      "rewards/rejected": -0.22368663549423218,
      "step": 422
    },
    {
      "epoch": 0.26314152410575425,
      "grad_norm": 0.44899922609329224,
      "learning_rate": 3.9425e-05,
      "log_odds_chosen": 0.8703987002372742,
      "log_odds_ratio": -0.5454012155532837,
      "logits/chosen": 0.2875048816204071,
      "logits/rejected": 0.11582295596599579,
      "logps/chosen": -1.363781213760376,
      "logps/rejected": -2.0890636444091797,
      "loss": 3.2192,
      "nll_loss": 3.1646461486816406,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13637810945510864,
      "rewards/margins": 0.07252823561429977,
      "rewards/rejected": -0.208906352519989,
      "step": 423
    },
    {
      "epoch": 0.26376360808709176,
      "grad_norm": 0.5044350028038025,
      "learning_rate": 3.94e-05,
      "log_odds_chosen": 1.7807879447937012,
      "log_odds_ratio": -0.2641463577747345,
      "logits/chosen": 0.25100329518318176,
      "logits/rejected": 0.0806911289691925,
      "logps/chosen": -1.1451863050460815,
      "logps/rejected": -2.675082206726074,
      "loss": 2.879,
      "nll_loss": 2.852630138397217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11451863497495651,
      "rewards/margins": 0.15298959612846375,
      "rewards/rejected": -0.26750820875167847,
      "step": 424
    },
    {
      "epoch": 0.2643856920684292,
      "grad_norm": 0.5463127493858337,
      "learning_rate": 3.9375e-05,
      "log_odds_chosen": 1.7012643814086914,
      "log_odds_ratio": -0.3870460093021393,
      "logits/chosen": 0.1893111914396286,
      "logits/rejected": 0.12804879248142242,
      "logps/chosen": -1.3288249969482422,
      "logps/rejected": -2.868129253387451,
      "loss": 3.3723,
      "nll_loss": 3.3335795402526855,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1328825205564499,
      "rewards/margins": 0.1539304256439209,
      "rewards/rejected": -0.286812961101532,
      "step": 425
    },
    {
      "epoch": 0.2650077760497667,
      "grad_norm": 0.8619997501373291,
      "learning_rate": 3.935e-05,
      "log_odds_chosen": 0.8874719738960266,
      "log_odds_ratio": -0.7657593488693237,
      "logits/chosen": 0.08044306933879852,
      "logits/rejected": -0.008549835532903671,
      "logps/chosen": -1.7457143068313599,
      "logps/rejected": -2.4340968132019043,
      "loss": 2.9367,
      "nll_loss": 2.8601346015930176,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1745714396238327,
      "rewards/margins": 0.06883823126554489,
      "rewards/rejected": -0.2434096783399582,
      "step": 426
    },
    {
      "epoch": 0.2656298600311042,
      "grad_norm": 0.5804911851882935,
      "learning_rate": 3.9325e-05,
      "log_odds_chosen": 1.0181046724319458,
      "log_odds_ratio": -0.5276904106140137,
      "logits/chosen": 0.20519691705703735,
      "logits/rejected": 0.11686623841524124,
      "logps/chosen": -1.3472039699554443,
      "logps/rejected": -2.2560369968414307,
      "loss": 3.1032,
      "nll_loss": 3.050477981567383,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13472039997577667,
      "rewards/margins": 0.0908832848072052,
      "rewards/rejected": -0.22560366988182068,
      "step": 427
    },
    {
      "epoch": 0.2662519440124417,
      "grad_norm": 0.4489701986312866,
      "learning_rate": 3.9300000000000007e-05,
      "log_odds_chosen": 1.4261120557785034,
      "log_odds_ratio": -0.3344850540161133,
      "logits/chosen": 0.06815585494041443,
      "logits/rejected": 0.005228975787758827,
      "logps/chosen": -1.0918066501617432,
      "logps/rejected": -2.263209342956543,
      "loss": 2.801,
      "nll_loss": 2.767580509185791,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10918065905570984,
      "rewards/margins": 0.1171402856707573,
      "rewards/rejected": -0.22632096707820892,
      "step": 428
    },
    {
      "epoch": 0.2668740279937792,
      "grad_norm": 0.6455628871917725,
      "learning_rate": 3.9275e-05,
      "log_odds_chosen": 1.4339535236358643,
      "log_odds_ratio": -0.32922857999801636,
      "logits/chosen": 0.24753251671791077,
      "logits/rejected": 0.17592373490333557,
      "logps/chosen": -1.4441932439804077,
      "logps/rejected": -2.687839984893799,
      "loss": 3.094,
      "nll_loss": 3.0610883235931396,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14441931247711182,
      "rewards/margins": 0.1243646889925003,
      "rewards/rejected": -0.2687840163707733,
      "step": 429
    },
    {
      "epoch": 0.26749611197511663,
      "grad_norm": 0.29282695055007935,
      "learning_rate": 3.9250000000000005e-05,
      "log_odds_chosen": 1.340848445892334,
      "log_odds_ratio": -0.34622523188591003,
      "logits/chosen": 0.22931736707687378,
      "logits/rejected": 0.14595317840576172,
      "logps/chosen": -1.1169357299804688,
      "logps/rejected": -2.1826369762420654,
      "loss": 3.4491,
      "nll_loss": 3.4144582748413086,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11169356852769852,
      "rewards/margins": 0.10657013207674026,
      "rewards/rejected": -0.21826371550559998,
      "step": 430
    },
    {
      "epoch": 0.26811819595645414,
      "grad_norm": 0.3759579062461853,
      "learning_rate": 3.9225e-05,
      "log_odds_chosen": 0.9186004996299744,
      "log_odds_ratio": -0.4763484001159668,
      "logits/chosen": 0.2613193690776825,
      "logits/rejected": 0.11609724164009094,
      "logps/chosen": -0.910784125328064,
      "logps/rejected": -1.6030433177947998,
      "loss": 3.5052,
      "nll_loss": 3.4575998783111572,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09107840061187744,
      "rewards/margins": 0.06922592967748642,
      "rewards/rejected": -0.16030432283878326,
      "step": 431
    },
    {
      "epoch": 0.2687402799377916,
      "grad_norm": 0.4951987564563751,
      "learning_rate": 3.9200000000000004e-05,
      "log_odds_chosen": 1.3083531856536865,
      "log_odds_ratio": -0.3793916702270508,
      "logits/chosen": 0.3077886700630188,
      "logits/rejected": 0.05748777836561203,
      "logps/chosen": -1.2742183208465576,
      "logps/rejected": -2.2997703552246094,
      "loss": 3.7485,
      "nll_loss": 3.710557460784912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12742182612419128,
      "rewards/margins": 0.10255520790815353,
      "rewards/rejected": -0.22997704148292542,
      "step": 432
    },
    {
      "epoch": 0.2693623639191291,
      "grad_norm": 0.4444817006587982,
      "learning_rate": 3.9175000000000004e-05,
      "log_odds_chosen": 1.2291901111602783,
      "log_odds_ratio": -0.35217738151550293,
      "logits/chosen": 0.2221524715423584,
      "logits/rejected": 0.05530994012951851,
      "logps/chosen": -0.9905591011047363,
      "logps/rejected": -1.9143832921981812,
      "loss": 3.3073,
      "nll_loss": 3.272057056427002,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09905591607093811,
      "rewards/margins": 0.09238240122795105,
      "rewards/rejected": -0.19143833220005035,
      "step": 433
    },
    {
      "epoch": 0.26998444790046655,
      "grad_norm": 0.6299095749855042,
      "learning_rate": 3.915e-05,
      "log_odds_chosen": 1.615001916885376,
      "log_odds_ratio": -0.3864189386367798,
      "logits/chosen": 0.24696266651153564,
      "logits/rejected": 0.18707673251628876,
      "logps/chosen": -1.9024136066436768,
      "logps/rejected": -3.3632092475891113,
      "loss": 2.8607,
      "nll_loss": 2.822024345397949,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19024136662483215,
      "rewards/margins": 0.14607959985733032,
      "rewards/rejected": -0.3363209664821625,
      "step": 434
    },
    {
      "epoch": 0.27060653188180406,
      "grad_norm": 0.5415927767753601,
      "learning_rate": 3.9125e-05,
      "log_odds_chosen": 4.024477481842041,
      "log_odds_ratio": -0.14453338086605072,
      "logits/chosen": 0.2853761315345764,
      "logits/rejected": 0.15076112747192383,
      "logps/chosen": -1.2033590078353882,
      "logps/rejected": -4.912008285522461,
      "loss": 2.7905,
      "nll_loss": 2.776012897491455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1203359067440033,
      "rewards/margins": 0.3708649277687073,
      "rewards/rejected": -0.4912008047103882,
      "step": 435
    },
    {
      "epoch": 0.2712286158631415,
      "grad_norm": 0.6048150658607483,
      "learning_rate": 3.91e-05,
      "log_odds_chosen": 0.31674981117248535,
      "log_odds_ratio": -0.6299794912338257,
      "logits/chosen": 0.13437308371067047,
      "logits/rejected": 0.002337227575480938,
      "logps/chosen": -1.596144199371338,
      "logps/rejected": -1.849827527999878,
      "loss": 2.6519,
      "nll_loss": 2.588890790939331,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1596144139766693,
      "rewards/margins": 0.025368336588144302,
      "rewards/rejected": -0.1849827617406845,
      "step": 436
    },
    {
      "epoch": 0.271850699844479,
      "grad_norm": 0.44196486473083496,
      "learning_rate": 3.9075e-05,
      "log_odds_chosen": 1.744307041168213,
      "log_odds_ratio": -0.26184773445129395,
      "logits/chosen": 0.14910796284675598,
      "logits/rejected": 0.08620636910200119,
      "logps/chosen": -1.1879832744598389,
      "logps/rejected": -2.592353343963623,
      "loss": 2.8721,
      "nll_loss": 2.845905303955078,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11879833042621613,
      "rewards/margins": 0.14043700695037842,
      "rewards/rejected": -0.25923532247543335,
      "step": 437
    },
    {
      "epoch": 0.27247278382581647,
      "grad_norm": 0.5670360326766968,
      "learning_rate": 3.905e-05,
      "log_odds_chosen": 2.0719993114471436,
      "log_odds_ratio": -0.2230648547410965,
      "logits/chosen": 0.24322016537189484,
      "logits/rejected": 0.2413720339536667,
      "logps/chosen": -1.1990163326263428,
      "logps/rejected": -2.9432950019836426,
      "loss": 2.7375,
      "nll_loss": 2.715202569961548,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1199016347527504,
      "rewards/margins": 0.17442789673805237,
      "rewards/rejected": -0.29432952404022217,
      "step": 438
    },
    {
      "epoch": 0.273094867807154,
      "grad_norm": 0.4377356767654419,
      "learning_rate": 3.9025e-05,
      "log_odds_chosen": 2.4671876430511475,
      "log_odds_ratio": -0.4254373013973236,
      "logits/chosen": 0.3846701383590698,
      "logits/rejected": 0.2242540866136551,
      "logps/chosen": -0.9989205598831177,
      "logps/rejected": -3.226830244064331,
      "loss": 3.7535,
      "nll_loss": 3.7109501361846924,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09989205747842789,
      "rewards/margins": 0.22279097139835358,
      "rewards/rejected": -0.3226830065250397,
      "step": 439
    },
    {
      "epoch": 0.2737169517884914,
      "grad_norm": 0.6218655109405518,
      "learning_rate": 3.9000000000000006e-05,
      "log_odds_chosen": 1.7963087558746338,
      "log_odds_ratio": -0.24013981223106384,
      "logits/chosen": 0.22267626225948334,
      "logits/rejected": 0.15220728516578674,
      "logps/chosen": -1.4764683246612549,
      "logps/rejected": -3.08296275138855,
      "loss": 2.6897,
      "nll_loss": 2.665696620941162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14764682948589325,
      "rewards/margins": 0.16064943373203278,
      "rewards/rejected": -0.308296263217926,
      "step": 440
    },
    {
      "epoch": 0.27433903576982893,
      "grad_norm": 0.4783235788345337,
      "learning_rate": 3.8975e-05,
      "log_odds_chosen": 3.722914934158325,
      "log_odds_ratio": -0.1403815746307373,
      "logits/chosen": 0.3612845838069916,
      "logits/rejected": 0.3681058883666992,
      "logps/chosen": -1.2698994874954224,
      "logps/rejected": -4.6545090675354,
      "loss": 3.0846,
      "nll_loss": 3.0705621242523193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1269899606704712,
      "rewards/margins": 0.3384609520435333,
      "rewards/rejected": -0.46545088291168213,
      "step": 441
    },
    {
      "epoch": 0.2749611197511664,
      "grad_norm": 0.6110908389091492,
      "learning_rate": 3.8950000000000005e-05,
      "log_odds_chosen": 1.8417094945907593,
      "log_odds_ratio": -0.23535192012786865,
      "logits/chosen": 0.3509153127670288,
      "logits/rejected": 0.10706407576799393,
      "logps/chosen": -1.146148920059204,
      "logps/rejected": -2.6451005935668945,
      "loss": 3.4359,
      "nll_loss": 3.4123191833496094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11461488902568817,
      "rewards/margins": 0.14989519119262695,
      "rewards/rejected": -0.2645100951194763,
      "step": 442
    },
    {
      "epoch": 0.2755832037325039,
      "grad_norm": 0.378417044878006,
      "learning_rate": 3.8925e-05,
      "log_odds_chosen": 1.9329280853271484,
      "log_odds_ratio": -0.31626445055007935,
      "logits/chosen": 0.2477678805589676,
      "logits/rejected": 0.19056656956672668,
      "logps/chosen": -0.9245951175689697,
      "logps/rejected": -2.271580219268799,
      "loss": 3.5382,
      "nll_loss": 3.5066051483154297,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09245951473712921,
      "rewards/margins": 0.1346985101699829,
      "rewards/rejected": -0.22715803980827332,
      "step": 443
    },
    {
      "epoch": 0.27620528771384134,
      "grad_norm": 0.45432165265083313,
      "learning_rate": 3.8900000000000004e-05,
      "log_odds_chosen": 2.492096185684204,
      "log_odds_ratio": -0.20271611213684082,
      "logits/chosen": 0.1072557121515274,
      "logits/rejected": 0.1309928297996521,
      "logps/chosen": -1.2175402641296387,
      "logps/rejected": -3.431077003479004,
      "loss": 2.5011,
      "nll_loss": 2.480823040008545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12175402045249939,
      "rewards/margins": 0.2213537096977234,
      "rewards/rejected": -0.3431077003479004,
      "step": 444
    },
    {
      "epoch": 0.27682737169517885,
      "grad_norm": 0.4896884262561798,
      "learning_rate": 3.8875e-05,
      "log_odds_chosen": 2.018673896789551,
      "log_odds_ratio": -0.23892150819301605,
      "logits/chosen": 0.18642401695251465,
      "logits/rejected": 0.1272982805967331,
      "logps/chosen": -1.0007119178771973,
      "logps/rejected": -2.652472496032715,
      "loss": 2.8004,
      "nll_loss": 2.776474714279175,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10007119923830032,
      "rewards/margins": 0.16517606377601624,
      "rewards/rejected": -0.26524725556373596,
      "step": 445
    },
    {
      "epoch": 0.27744945567651635,
      "grad_norm": 0.4476563036441803,
      "learning_rate": 3.885e-05,
      "log_odds_chosen": 1.5900170803070068,
      "log_odds_ratio": -0.3700055480003357,
      "logits/chosen": 0.41444650292396545,
      "logits/rejected": 0.26796096563339233,
      "logps/chosen": -1.4082850217819214,
      "logps/rejected": -2.710878372192383,
      "loss": 3.8047,
      "nll_loss": 3.7677254676818848,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14082849025726318,
      "rewards/margins": 0.13025934994220734,
      "rewards/rejected": -0.2710878551006317,
      "step": 446
    },
    {
      "epoch": 0.2780715396578538,
      "grad_norm": 0.44968464970588684,
      "learning_rate": 3.8825e-05,
      "log_odds_chosen": 1.3255045413970947,
      "log_odds_ratio": -0.2901913523674011,
      "logits/chosen": 0.2749924957752228,
      "logits/rejected": 0.2749479413032532,
      "logps/chosen": -1.219002604484558,
      "logps/rejected": -2.211946964263916,
      "loss": 3.2752,
      "nll_loss": 3.246131420135498,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12190026044845581,
      "rewards/margins": 0.09929443150758743,
      "rewards/rejected": -0.22119468450546265,
      "step": 447
    },
    {
      "epoch": 0.2786936236391913,
      "grad_norm": 0.6323361992835999,
      "learning_rate": 3.88e-05,
      "log_odds_chosen": 2.376352071762085,
      "log_odds_ratio": -0.16851823031902313,
      "logits/chosen": 0.2564919590950012,
      "logits/rejected": 0.3483697772026062,
      "logps/chosen": -1.1505968570709229,
      "logps/rejected": -3.2237629890441895,
      "loss": 2.4031,
      "nll_loss": 2.386216640472412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11505968123674393,
      "rewards/margins": 0.20731663703918457,
      "rewards/rejected": -0.3223763108253479,
      "step": 448
    },
    {
      "epoch": 0.27931570762052876,
      "grad_norm": 0.4380817115306854,
      "learning_rate": 3.8775e-05,
      "log_odds_chosen": 1.486014723777771,
      "log_odds_ratio": -0.3191292881965637,
      "logits/chosen": 0.35001346468925476,
      "logits/rejected": 0.26604193449020386,
      "logps/chosen": -1.0675163269042969,
      "logps/rejected": -2.318084478378296,
      "loss": 3.3673,
      "nll_loss": 3.3354063034057617,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10675162822008133,
      "rewards/margins": 0.12505681812763214,
      "rewards/rejected": -0.23180843889713287,
      "step": 449
    },
    {
      "epoch": 0.27993779160186627,
      "grad_norm": 0.5374330282211304,
      "learning_rate": 3.875e-05,
      "log_odds_chosen": 2.0906572341918945,
      "log_odds_ratio": -0.2168201506137848,
      "logits/chosen": 0.2877808213233948,
      "logits/rejected": 0.3449041545391083,
      "logps/chosen": -1.3126355409622192,
      "logps/rejected": -2.9990274906158447,
      "loss": 3.0555,
      "nll_loss": 3.033806085586548,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13126355409622192,
      "rewards/margins": 0.1686391979455948,
      "rewards/rejected": -0.2999027371406555,
      "step": 450
    },
    {
      "epoch": 0.2805598755832037,
      "grad_norm": 0.6015781760215759,
      "learning_rate": 3.8725e-05,
      "log_odds_chosen": 2.5022025108337402,
      "log_odds_ratio": -0.19380271434783936,
      "logits/chosen": 0.3818150460720062,
      "logits/rejected": 0.35132819414138794,
      "logps/chosen": -1.2525804042816162,
      "logps/rejected": -3.4683022499084473,
      "loss": 3.4237,
      "nll_loss": 3.4043030738830566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12525805830955505,
      "rewards/margins": 0.22157219052314758,
      "rewards/rejected": -0.34683024883270264,
      "step": 451
    },
    {
      "epoch": 0.28118195956454123,
      "grad_norm": 0.5427994132041931,
      "learning_rate": 3.8700000000000006e-05,
      "log_odds_chosen": 1.5231120586395264,
      "log_odds_ratio": -0.28168755769729614,
      "logits/chosen": 0.3691140115261078,
      "logits/rejected": 0.37976768612861633,
      "logps/chosen": -1.2954881191253662,
      "logps/rejected": -2.606727361679077,
      "loss": 3.1849,
      "nll_loss": 3.1567115783691406,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1295488178730011,
      "rewards/margins": 0.13112393021583557,
      "rewards/rejected": -0.26067274808883667,
      "step": 452
    },
    {
      "epoch": 0.2818040435458787,
      "grad_norm": 0.6339102983474731,
      "learning_rate": 3.8675e-05,
      "log_odds_chosen": 4.337133407592773,
      "log_odds_ratio": -0.21208305656909943,
      "logits/chosen": 0.25384026765823364,
      "logits/rejected": 0.20061862468719482,
      "logps/chosen": -1.4610943794250488,
      "logps/rejected": -5.5489044189453125,
      "loss": 3.273,
      "nll_loss": 3.251758098602295,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1461094468832016,
      "rewards/margins": 0.4087810218334198,
      "rewards/rejected": -0.5548904538154602,
      "step": 453
    },
    {
      "epoch": 0.2824261275272162,
      "grad_norm": 1.4807077646255493,
      "learning_rate": 3.8650000000000004e-05,
      "log_odds_chosen": 2.719972848892212,
      "log_odds_ratio": -0.1599472612142563,
      "logits/chosen": 0.38763532042503357,
      "logits/rejected": 0.3248913884162903,
      "logps/chosen": -1.1536898612976074,
      "logps/rejected": -3.492626905441284,
      "loss": 3.2056,
      "nll_loss": 3.1896491050720215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11536898463964462,
      "rewards/margins": 0.2338937222957611,
      "rewards/rejected": -0.34926271438598633,
      "step": 454
    },
    {
      "epoch": 0.28304821150855364,
      "grad_norm": 0.4285638928413391,
      "learning_rate": 3.8625e-05,
      "log_odds_chosen": 3.288182020187378,
      "log_odds_ratio": -0.05790887400507927,
      "logits/chosen": 0.2987384498119354,
      "logits/rejected": 0.29059848189353943,
      "logps/chosen": -1.2740700244903564,
      "logps/rejected": -4.2349653244018555,
      "loss": 3.027,
      "nll_loss": 3.0212230682373047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1274069994688034,
      "rewards/margins": 0.2960895299911499,
      "rewards/rejected": -0.4234965443611145,
      "step": 455
    },
    {
      "epoch": 0.28367029548989114,
      "grad_norm": 0.8652162551879883,
      "learning_rate": 3.86e-05,
      "log_odds_chosen": 1.5928850173950195,
      "log_odds_ratio": -0.311477929353714,
      "logits/chosen": 0.17478427290916443,
      "logits/rejected": 0.1352236568927765,
      "logps/chosen": -1.60139799118042,
      "logps/rejected": -2.999675750732422,
      "loss": 3.2718,
      "nll_loss": 3.2406058311462402,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.160139799118042,
      "rewards/margins": 0.13982778787612915,
      "rewards/rejected": -0.29996758699417114,
      "step": 456
    },
    {
      "epoch": 0.2842923794712286,
      "grad_norm": 0.3627634644508362,
      "learning_rate": 3.8575e-05,
      "log_odds_chosen": 2.181387186050415,
      "log_odds_ratio": -0.3236231803894043,
      "logits/chosen": 0.05983370169997215,
      "logits/rejected": 0.02825239673256874,
      "logps/chosen": -1.1143271923065186,
      "logps/rejected": -2.988661766052246,
      "loss": 3.0779,
      "nll_loss": 3.0455780029296875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11143273115158081,
      "rewards/margins": 0.18743345141410828,
      "rewards/rejected": -0.2988661825656891,
      "step": 457
    },
    {
      "epoch": 0.2849144634525661,
      "grad_norm": 0.5619986057281494,
      "learning_rate": 3.855e-05,
      "log_odds_chosen": 1.8027219772338867,
      "log_odds_ratio": -0.46999043226242065,
      "logits/chosen": -0.01704075187444687,
      "logits/rejected": 0.08329185098409653,
      "logps/chosen": -1.0725547075271606,
      "logps/rejected": -2.68211030960083,
      "loss": 2.6529,
      "nll_loss": 2.6059412956237793,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1072554737329483,
      "rewards/margins": 0.16095557808876038,
      "rewards/rejected": -0.2682110369205475,
      "step": 458
    },
    {
      "epoch": 0.28553654743390355,
      "grad_norm": 0.4395221471786499,
      "learning_rate": 3.8525e-05,
      "log_odds_chosen": 3.0232295989990234,
      "log_odds_ratio": -0.25974196195602417,
      "logits/chosen": 0.24080729484558105,
      "logits/rejected": 0.15837368369102478,
      "logps/chosen": -0.9511310458183289,
      "logps/rejected": -3.5407259464263916,
      "loss": 3.3904,
      "nll_loss": 3.364377021789551,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.095113106071949,
      "rewards/margins": 0.25895950198173523,
      "rewards/rejected": -0.35407260060310364,
      "step": 459
    },
    {
      "epoch": 0.28615863141524106,
      "grad_norm": 0.39195650815963745,
      "learning_rate": 3.85e-05,
      "log_odds_chosen": 3.548543930053711,
      "log_odds_ratio": -0.2414306104183197,
      "logits/chosen": 0.20818278193473816,
      "logits/rejected": 0.2002563178539276,
      "logps/chosen": -0.9960837364196777,
      "logps/rejected": -4.0006537437438965,
      "loss": 3.0262,
      "nll_loss": 3.002067804336548,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09960837662220001,
      "rewards/margins": 0.3004570007324219,
      "rewards/rejected": -0.40006542205810547,
      "step": 460
    },
    {
      "epoch": 0.2867807153965785,
      "grad_norm": 0.6457217931747437,
      "learning_rate": 3.8475e-05,
      "log_odds_chosen": 2.5402870178222656,
      "log_odds_ratio": -0.3850615918636322,
      "logits/chosen": 0.11771635711193085,
      "logits/rejected": 0.03514918312430382,
      "logps/chosen": -1.3628007173538208,
      "logps/rejected": -3.764535427093506,
      "loss": 2.7094,
      "nll_loss": 2.670870304107666,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13628007471561432,
      "rewards/margins": 0.24017345905303955,
      "rewards/rejected": -0.3764535188674927,
      "step": 461
    },
    {
      "epoch": 0.287402799377916,
      "grad_norm": 0.5981135368347168,
      "learning_rate": 3.845e-05,
      "log_odds_chosen": 1.2403737306594849,
      "log_odds_ratio": -0.6255621314048767,
      "logits/chosen": 0.06767088919878006,
      "logits/rejected": 0.015279887244105339,
      "logps/chosen": -1.2029376029968262,
      "logps/rejected": -2.2572832107543945,
      "loss": 2.8872,
      "nll_loss": 2.8246262073516846,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12029377371072769,
      "rewards/margins": 0.10543454438447952,
      "rewards/rejected": -0.22572831809520721,
      "step": 462
    },
    {
      "epoch": 0.2880248833592535,
      "grad_norm": 0.6233921647071838,
      "learning_rate": 3.8425e-05,
      "log_odds_chosen": 2.528999090194702,
      "log_odds_ratio": -0.2973198890686035,
      "logits/chosen": 0.1612185388803482,
      "logits/rejected": 0.18806105852127075,
      "logps/chosen": -1.4305648803710938,
      "logps/rejected": -3.712616443634033,
      "loss": 2.868,
      "nll_loss": 2.838242292404175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1430564969778061,
      "rewards/margins": 0.22820515930652618,
      "rewards/rejected": -0.3712616562843323,
      "step": 463
    },
    {
      "epoch": 0.288646967340591,
      "grad_norm": 0.43462324142456055,
      "learning_rate": 3.8400000000000005e-05,
      "log_odds_chosen": 3.214184522628784,
      "log_odds_ratio": -0.06043536216020584,
      "logits/chosen": 0.21719320118427277,
      "logits/rejected": 0.25715163350105286,
      "logps/chosen": -1.121093511581421,
      "logps/rejected": -3.9434640407562256,
      "loss": 2.9904,
      "nll_loss": 2.9843649864196777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11210934817790985,
      "rewards/margins": 0.28223708271980286,
      "rewards/rejected": -0.3943464159965515,
      "step": 464
    },
    {
      "epoch": 0.2892690513219285,
      "grad_norm": 0.5641807913780212,
      "learning_rate": 3.8375e-05,
      "log_odds_chosen": 4.3896636962890625,
      "log_odds_ratio": -0.18577708303928375,
      "logits/chosen": 0.0453701950609684,
      "logits/rejected": 0.07587449252605438,
      "logps/chosen": -1.2915798425674438,
      "logps/rejected": -5.313374996185303,
      "loss": 2.8604,
      "nll_loss": 2.841811418533325,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12915797531604767,
      "rewards/margins": 0.4021795392036438,
      "rewards/rejected": -0.5313374996185303,
      "step": 465
    },
    {
      "epoch": 0.28989113530326593,
      "grad_norm": 0.7072799801826477,
      "learning_rate": 3.8350000000000004e-05,
      "log_odds_chosen": 3.1329102516174316,
      "log_odds_ratio": -0.38895708322525024,
      "logits/chosen": 0.36617377400398254,
      "logits/rejected": 0.32684844732284546,
      "logps/chosen": -1.486217975616455,
      "logps/rejected": -4.195459842681885,
      "loss": 3.3864,
      "nll_loss": 3.347512722015381,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1486217975616455,
      "rewards/margins": 0.2709242105484009,
      "rewards/rejected": -0.4195460081100464,
      "step": 466
    },
    {
      "epoch": 0.29051321928460344,
      "grad_norm": 0.5267555713653564,
      "learning_rate": 3.8324999999999996e-05,
      "log_odds_chosen": 1.2240618467330933,
      "log_odds_ratio": -0.39744335412979126,
      "logits/chosen": 0.22255158424377441,
      "logits/rejected": 0.18081636726856232,
      "logps/chosen": -1.1293349266052246,
      "logps/rejected": -2.1319031715393066,
      "loss": 2.8698,
      "nll_loss": 2.830047130584717,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11293349415063858,
      "rewards/margins": 0.10025681555271149,
      "rewards/rejected": -0.21319028735160828,
      "step": 467
    },
    {
      "epoch": 0.2911353032659409,
      "grad_norm": 0.6344406604766846,
      "learning_rate": 3.83e-05,
      "log_odds_chosen": 6.454854488372803,
      "log_odds_ratio": -0.14626577496528625,
      "logits/chosen": 0.26416561007499695,
      "logits/rejected": 0.411790668964386,
      "logps/chosen": -1.2627651691436768,
      "logps/rejected": -7.43460750579834,
      "loss": 3.1443,
      "nll_loss": 3.129704475402832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12627652287483215,
      "rewards/margins": 0.6171842813491821,
      "rewards/rejected": -0.7434607148170471,
      "step": 468
    },
    {
      "epoch": 0.2917573872472784,
      "grad_norm": 2.3335068225860596,
      "learning_rate": 3.8275e-05,
      "log_odds_chosen": 5.498930931091309,
      "log_odds_ratio": -0.5096065998077393,
      "logits/chosen": 0.3187922239303589,
      "logits/rejected": 0.4804098308086395,
      "logps/chosen": -1.7013590335845947,
      "logps/rejected": -6.9031877517700195,
      "loss": 3.3374,
      "nll_loss": 3.2864151000976562,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.17013590037822723,
      "rewards/margins": 0.5201829075813293,
      "rewards/rejected": -0.690318763256073,
      "step": 469
    },
    {
      "epoch": 0.29237947122861585,
      "grad_norm": 0.3446737825870514,
      "learning_rate": 3.825e-05,
      "log_odds_chosen": 4.659626007080078,
      "log_odds_ratio": -0.05383168160915375,
      "logits/chosen": 0.3359685242176056,
      "logits/rejected": 0.3225572407245636,
      "logps/chosen": -1.2879467010498047,
      "logps/rejected": -5.530665397644043,
      "loss": 4.0427,
      "nll_loss": 4.037296295166016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12879468500614166,
      "rewards/margins": 0.4242718815803528,
      "rewards/rejected": -0.5530665516853333,
      "step": 470
    },
    {
      "epoch": 0.29300155520995336,
      "grad_norm": 0.6117813587188721,
      "learning_rate": 3.8225e-05,
      "log_odds_chosen": 3.6450319290161133,
      "log_odds_ratio": -0.11542224884033203,
      "logits/chosen": 0.22536617517471313,
      "logits/rejected": 0.26635074615478516,
      "logps/chosen": -0.9902899265289307,
      "logps/rejected": -4.163326740264893,
      "loss": 3.1959,
      "nll_loss": 3.184352397918701,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09902900457382202,
      "rewards/margins": 0.31730368733406067,
      "rewards/rejected": -0.4163326919078827,
      "step": 471
    },
    {
      "epoch": 0.2936236391912908,
      "grad_norm": 0.6647369861602783,
      "learning_rate": 3.82e-05,
      "log_odds_chosen": 4.759244441986084,
      "log_odds_ratio": -0.22806090116500854,
      "logits/chosen": 0.2276982069015503,
      "logits/rejected": 0.2902960777282715,
      "logps/chosen": -1.0806303024291992,
      "logps/rejected": -5.489457607269287,
      "loss": 2.7768,
      "nll_loss": 2.754019260406494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10806304216384888,
      "rewards/margins": 0.44088274240493774,
      "rewards/rejected": -0.5489457845687866,
      "step": 472
    },
    {
      "epoch": 0.2942457231726283,
      "grad_norm": 0.4787210524082184,
      "learning_rate": 3.8175e-05,
      "log_odds_chosen": 3.1084511280059814,
      "log_odds_ratio": -0.15226207673549652,
      "logits/chosen": 0.3156070113182068,
      "logits/rejected": 0.38743242621421814,
      "logps/chosen": -1.3634812831878662,
      "logps/rejected": -4.172502040863037,
      "loss": 3.6274,
      "nll_loss": 3.612142324447632,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13634812831878662,
      "rewards/margins": 0.28090211749076843,
      "rewards/rejected": -0.41725024580955505,
      "step": 473
    },
    {
      "epoch": 0.29486780715396577,
      "grad_norm": 0.4799170196056366,
      "learning_rate": 3.8150000000000006e-05,
      "log_odds_chosen": 5.302570343017578,
      "log_odds_ratio": -0.17532306909561157,
      "logits/chosen": 0.2545361816883087,
      "logits/rejected": 0.4782401919364929,
      "logps/chosen": -1.1034696102142334,
      "logps/rejected": -6.034841537475586,
      "loss": 2.6736,
      "nll_loss": 2.6560819149017334,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1103469654917717,
      "rewards/margins": 0.4931372106075287,
      "rewards/rejected": -0.6034841537475586,
      "step": 474
    },
    {
      "epoch": 0.2954898911353033,
      "grad_norm": 0.43535855412483215,
      "learning_rate": 3.8125e-05,
      "log_odds_chosen": 7.762436389923096,
      "log_odds_ratio": -0.13623476028442383,
      "logits/chosen": 0.1623404324054718,
      "logits/rejected": 0.2579556405544281,
      "logps/chosen": -0.7613686323165894,
      "logps/rejected": -7.789710521697998,
      "loss": 3.0684,
      "nll_loss": 3.054818630218506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07613686472177505,
      "rewards/margins": 0.7028341889381409,
      "rewards/rejected": -0.7789710760116577,
      "step": 475
    },
    {
      "epoch": 0.2961119751166407,
      "grad_norm": 1.3790374994277954,
      "learning_rate": 3.8100000000000005e-05,
      "log_odds_chosen": 3.3232266902923584,
      "log_odds_ratio": -0.1542171686887741,
      "logits/chosen": 0.13157705962657928,
      "logits/rejected": 0.14837421476840973,
      "logps/chosen": -1.329514741897583,
      "logps/rejected": -4.3654656410217285,
      "loss": 2.7209,
      "nll_loss": 2.7054383754730225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13295146822929382,
      "rewards/margins": 0.3035951256752014,
      "rewards/rejected": -0.43654656410217285,
      "step": 476
    },
    {
      "epoch": 0.29673405909797823,
      "grad_norm": 0.4154740869998932,
      "learning_rate": 3.8075e-05,
      "log_odds_chosen": 3.3031208515167236,
      "log_odds_ratio": -0.19959819316864014,
      "logits/chosen": 0.29767587780952454,
      "logits/rejected": 0.3195898234844208,
      "logps/chosen": -1.2283899784088135,
      "logps/rejected": -4.255227088928223,
      "loss": 3.426,
      "nll_loss": 3.4059970378875732,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12283900380134583,
      "rewards/margins": 0.3026837110519409,
      "rewards/rejected": -0.42552274465560913,
      "step": 477
    },
    {
      "epoch": 0.2973561430793157,
      "grad_norm": 0.440824955701828,
      "learning_rate": 3.805e-05,
      "log_odds_chosen": 5.133951187133789,
      "log_odds_ratio": -0.1343551129102707,
      "logits/chosen": -0.004576465114951134,
      "logits/rejected": 0.09814856946468353,
      "logps/chosen": -1.0036342144012451,
      "logps/rejected": -5.681285381317139,
      "loss": 2.9374,
      "nll_loss": 2.923962116241455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10036341845989227,
      "rewards/margins": 0.46776506304740906,
      "rewards/rejected": -0.5681285262107849,
      "step": 478
    },
    {
      "epoch": 0.2979782270606532,
      "grad_norm": 0.4802106022834778,
      "learning_rate": 3.8025e-05,
      "log_odds_chosen": 4.080648422241211,
      "log_odds_ratio": -0.15446895360946655,
      "logits/chosen": 0.006258752197027206,
      "logits/rejected": 0.04351950064301491,
      "logps/chosen": -1.1790379285812378,
      "logps/rejected": -4.813777923583984,
      "loss": 2.8196,
      "nll_loss": 2.804121732711792,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11790378391742706,
      "rewards/margins": 0.3634740114212036,
      "rewards/rejected": -0.48137781023979187,
      "step": 479
    },
    {
      "epoch": 0.2986003110419907,
      "grad_norm": 0.4847407341003418,
      "learning_rate": 3.8e-05,
      "log_odds_chosen": 4.479074478149414,
      "log_odds_ratio": -0.12381869554519653,
      "logits/chosen": 0.2093774825334549,
      "logits/rejected": 0.29991966485977173,
      "logps/chosen": -1.0310206413269043,
      "logps/rejected": -5.081138610839844,
      "loss": 2.6951,
      "nll_loss": 2.682741165161133,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10310205817222595,
      "rewards/margins": 0.4050118029117584,
      "rewards/rejected": -0.5081138610839844,
      "step": 480
    },
    {
      "epoch": 0.29922239502332815,
      "grad_norm": 0.9063189029693604,
      "learning_rate": 3.7975e-05,
      "log_odds_chosen": 2.8616793155670166,
      "log_odds_ratio": -0.12458339333534241,
      "logits/chosen": 0.1728355586528778,
      "logits/rejected": 0.1352435052394867,
      "logps/chosen": -1.2799311876296997,
      "logps/rejected": -3.802501678466797,
      "loss": 3.3188,
      "nll_loss": 3.306375026702881,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1279931366443634,
      "rewards/margins": 0.25225701928138733,
      "rewards/rejected": -0.38025015592575073,
      "step": 481
    },
    {
      "epoch": 0.29984447900466565,
      "grad_norm": 0.42489972710609436,
      "learning_rate": 3.795e-05,
      "log_odds_chosen": 5.341192245483398,
      "log_odds_ratio": -0.26994985342025757,
      "logits/chosen": 0.02017594873905182,
      "logits/rejected": 0.0014428049325942993,
      "logps/chosen": -1.1768755912780762,
      "logps/rejected": -6.195101261138916,
      "loss": 2.8912,
      "nll_loss": 2.8642163276672363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11768755316734314,
      "rewards/margins": 0.5018225312232971,
      "rewards/rejected": -0.6195101737976074,
      "step": 482
    },
    {
      "epoch": 0.3004665629860031,
      "grad_norm": 0.4654333293437958,
      "learning_rate": 3.7925e-05,
      "log_odds_chosen": 2.1485209465026855,
      "log_odds_ratio": -0.2876686751842499,
      "logits/chosen": 0.02476568892598152,
      "logits/rejected": 0.06350227445363998,
      "logps/chosen": -1.2585699558258057,
      "logps/rejected": -3.1734275817871094,
      "loss": 3.0375,
      "nll_loss": 3.008713960647583,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12585701048374176,
      "rewards/margins": 0.19148576259613037,
      "rewards/rejected": -0.31734275817871094,
      "step": 483
    },
    {
      "epoch": 0.3010886469673406,
      "grad_norm": 1.332303524017334,
      "learning_rate": 3.79e-05,
      "log_odds_chosen": 1.5843894481658936,
      "log_odds_ratio": -0.5189324617385864,
      "logits/chosen": 0.19418621063232422,
      "logits/rejected": 0.06159728765487671,
      "logps/chosen": -1.9245703220367432,
      "logps/rejected": -3.3311855792999268,
      "loss": 3.4679,
      "nll_loss": 3.4160549640655518,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19245702028274536,
      "rewards/margins": 0.14066153764724731,
      "rewards/rejected": -0.3331185579299927,
      "step": 484
    },
    {
      "epoch": 0.30171073094867806,
      "grad_norm": 0.6246785521507263,
      "learning_rate": 3.7875e-05,
      "log_odds_chosen": 4.188319206237793,
      "log_odds_ratio": -0.4303605556488037,
      "logits/chosen": 0.15692827105522156,
      "logits/rejected": 0.29972484707832336,
      "logps/chosen": -1.1631245613098145,
      "logps/rejected": -5.161314964294434,
      "loss": 2.4339,
      "nll_loss": 2.390859365463257,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11631244421005249,
      "rewards/margins": 0.3998190760612488,
      "rewards/rejected": -0.5161314606666565,
      "step": 485
    },
    {
      "epoch": 0.30233281493001557,
      "grad_norm": 0.4516681134700775,
      "learning_rate": 3.7850000000000005e-05,
      "log_odds_chosen": 3.0724825859069824,
      "log_odds_ratio": -0.13238348066806793,
      "logits/chosen": 0.14594241976737976,
      "logits/rejected": 0.19076189398765564,
      "logps/chosen": -0.9410706758499146,
      "logps/rejected": -3.3255326747894287,
      "loss": 2.9302,
      "nll_loss": 2.9170100688934326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09410707652568817,
      "rewards/margins": 0.2384462058544159,
      "rewards/rejected": -0.33255326747894287,
      "step": 486
    },
    {
      "epoch": 0.302954898911353,
      "grad_norm": 0.44087883830070496,
      "learning_rate": 3.7825e-05,
      "log_odds_chosen": 3.9114091396331787,
      "log_odds_ratio": -0.2950913608074188,
      "logits/chosen": 0.15809951722621918,
      "logits/rejected": 0.14425241947174072,
      "logps/chosen": -1.040985345840454,
      "logps/rejected": -4.549111366271973,
      "loss": 3.16,
      "nll_loss": 3.1304802894592285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10409855097532272,
      "rewards/margins": 0.3508126437664032,
      "rewards/rejected": -0.45491117238998413,
      "step": 487
    },
    {
      "epoch": 0.30357698289269053,
      "grad_norm": 0.556199312210083,
      "learning_rate": 3.7800000000000004e-05,
      "log_odds_chosen": 5.235114097595215,
      "log_odds_ratio": -0.041219230741262436,
      "logits/chosen": 0.14572152495384216,
      "logits/rejected": 0.3106115758419037,
      "logps/chosen": -1.1664040088653564,
      "logps/rejected": -5.896844863891602,
      "loss": 3.1604,
      "nll_loss": 3.1562514305114746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11664040386676788,
      "rewards/margins": 0.47304409742355347,
      "rewards/rejected": -0.5896844863891602,
      "step": 488
    },
    {
      "epoch": 0.304199066874028,
      "grad_norm": 0.5189743041992188,
      "learning_rate": 3.7775e-05,
      "log_odds_chosen": 7.963743209838867,
      "log_odds_ratio": -0.1266322284936905,
      "logits/chosen": 0.3704788088798523,
      "logits/rejected": 0.6580029726028442,
      "logps/chosen": -1.3174169063568115,
      "logps/rejected": -9.020464897155762,
      "loss": 3.4173,
      "nll_loss": 3.4046013355255127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1317417025566101,
      "rewards/margins": 0.7703048586845398,
      "rewards/rejected": -0.9020465016365051,
      "step": 489
    },
    {
      "epoch": 0.3048211508553655,
      "grad_norm": 0.4847100079059601,
      "learning_rate": 3.775e-05,
      "log_odds_chosen": 9.21757984161377,
      "log_odds_ratio": -0.10597797483205795,
      "logits/chosen": 0.16972588002681732,
      "logits/rejected": 0.3820986747741699,
      "logps/chosen": -1.2503827810287476,
      "logps/rejected": -10.164284706115723,
      "loss": 3.2205,
      "nll_loss": 3.2099387645721436,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.125038281083107,
      "rewards/margins": 0.8913902044296265,
      "rewards/rejected": -1.0164284706115723,
      "step": 490
    },
    {
      "epoch": 0.30544323483670294,
      "grad_norm": 0.8318912386894226,
      "learning_rate": 3.7725e-05,
      "log_odds_chosen": 9.271547317504883,
      "log_odds_ratio": -0.021590309217572212,
      "logits/chosen": 0.3189174234867096,
      "logits/rejected": 0.49795106053352356,
      "logps/chosen": -1.382540225982666,
      "logps/rejected": -10.197100639343262,
      "loss": 3.9656,
      "nll_loss": 3.963407039642334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13825401663780212,
      "rewards/margins": 0.8814560174942017,
      "rewards/rejected": -1.0197100639343262,
      "step": 491
    },
    {
      "epoch": 0.30606531881804044,
      "grad_norm": 0.40964096784591675,
      "learning_rate": 3.77e-05,
      "log_odds_chosen": 5.522929668426514,
      "log_odds_ratio": -0.1387336701154709,
      "logits/chosen": 0.29802757501602173,
      "logits/rejected": 0.4043997526168823,
      "logps/chosen": -1.073734164237976,
      "logps/rejected": -6.166069507598877,
      "loss": 3.7311,
      "nll_loss": 3.7172298431396484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1073734238743782,
      "rewards/margins": 0.5092335343360901,
      "rewards/rejected": -0.6166069507598877,
      "step": 492
    },
    {
      "epoch": 0.3066874027993779,
      "grad_norm": 0.5874708294868469,
      "learning_rate": 3.7675e-05,
      "log_odds_chosen": 12.224292755126953,
      "log_odds_ratio": -0.021359411999583244,
      "logits/chosen": 0.136999249458313,
      "logits/rejected": 0.7200852632522583,
      "logps/chosen": -0.9732953906059265,
      "logps/rejected": -12.676000595092773,
      "loss": 2.3038,
      "nll_loss": 2.3016741275787354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09732954204082489,
      "rewards/margins": 1.1702704429626465,
      "rewards/rejected": -1.2676000595092773,
      "step": 493
    },
    {
      "epoch": 0.3073094867807154,
      "grad_norm": 0.5743467807769775,
      "learning_rate": 3.765e-05,
      "log_odds_chosen": 12.042713165283203,
      "log_odds_ratio": -0.070807546377182,
      "logits/chosen": 0.2361012101173401,
      "logits/rejected": 0.562972903251648,
      "logps/chosen": -1.5356069803237915,
      "logps/rejected": -13.233451843261719,
      "loss": 3.359,
      "nll_loss": 3.3519067764282227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15356069803237915,
      "rewards/margins": 1.1697845458984375,
      "rewards/rejected": -1.3233451843261719,
      "step": 494
    },
    {
      "epoch": 0.30793157076205285,
      "grad_norm": 1.004876732826233,
      "learning_rate": 3.7625e-05,
      "log_odds_chosen": 6.287815570831299,
      "log_odds_ratio": -0.2341613471508026,
      "logits/chosen": 0.1172030121088028,
      "logits/rejected": 0.23774453997612,
      "logps/chosen": -1.0990171432495117,
      "logps/rejected": -6.949299335479736,
      "loss": 3.1907,
      "nll_loss": 3.167259454727173,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10990171879529953,
      "rewards/margins": 0.5850282311439514,
      "rewards/rejected": -0.6949299573898315,
      "step": 495
    },
    {
      "epoch": 0.30855365474339036,
      "grad_norm": 3.0034093856811523,
      "learning_rate": 3.76e-05,
      "log_odds_chosen": 7.123260498046875,
      "log_odds_ratio": -0.10044410824775696,
      "logits/chosen": 0.23248106241226196,
      "logits/rejected": 0.4872136116027832,
      "logps/chosen": -1.2667349576950073,
      "logps/rejected": -8.054340362548828,
      "loss": 3.0054,
      "nll_loss": 2.9953811168670654,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12667348980903625,
      "rewards/margins": 0.6787605285644531,
      "rewards/rejected": -0.8054340481758118,
      "step": 496
    },
    {
      "epoch": 0.3091757387247278,
      "grad_norm": 1.1735531091690063,
      "learning_rate": 3.7575e-05,
      "log_odds_chosen": 8.58930492401123,
      "log_odds_ratio": -0.04813535511493683,
      "logits/chosen": 0.26603323221206665,
      "logits/rejected": 0.5334312915802002,
      "logps/chosen": -1.2953003644943237,
      "logps/rejected": -9.486214637756348,
      "loss": 3.1711,
      "nll_loss": 3.166259288787842,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12953002750873566,
      "rewards/margins": 0.8190913796424866,
      "rewards/rejected": -0.9486214518547058,
      "step": 497
    },
    {
      "epoch": 0.3097978227060653,
      "grad_norm": 0.6103359460830688,
      "learning_rate": 3.7550000000000005e-05,
      "log_odds_chosen": 2.1337530612945557,
      "log_odds_ratio": -0.2864910066127777,
      "logits/chosen": 0.15103283524513245,
      "logits/rejected": 0.25624915957450867,
      "logps/chosen": -1.439260482788086,
      "logps/rejected": -3.371507406234741,
      "loss": 3.1723,
      "nll_loss": 3.143685817718506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14392603933811188,
      "rewards/margins": 0.1932247132062912,
      "rewards/rejected": -0.33715078234672546,
      "step": 498
    },
    {
      "epoch": 0.3104199066874028,
      "grad_norm": 0.43494969606399536,
      "learning_rate": 3.7525e-05,
      "log_odds_chosen": 3.525885820388794,
      "log_odds_ratio": -0.3266465663909912,
      "logits/chosen": 0.008623511530458927,
      "logits/rejected": 0.14156433939933777,
      "logps/chosen": -1.0080041885375977,
      "logps/rejected": -4.299755096435547,
      "loss": 2.4517,
      "nll_loss": 2.4190571308135986,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10080042481422424,
      "rewards/margins": 0.32917505502700806,
      "rewards/rejected": -0.4299754798412323,
      "step": 499
    },
    {
      "epoch": 0.3110419906687403,
      "grad_norm": 0.4379410147666931,
      "learning_rate": 3.7500000000000003e-05,
      "log_odds_chosen": 1.4496530294418335,
      "log_odds_ratio": -0.3601369857788086,
      "logits/chosen": 0.027029283344745636,
      "logits/rejected": 0.10909566283226013,
      "logps/chosen": -1.3700931072235107,
      "logps/rejected": -2.6318204402923584,
      "loss": 2.8374,
      "nll_loss": 2.801370620727539,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13700930774211884,
      "rewards/margins": 0.1261727213859558,
      "rewards/rejected": -0.26318204402923584,
      "step": 500
    },
    {
      "epoch": 0.3116640746500778,
      "grad_norm": 0.631767988204956,
      "learning_rate": 3.7475e-05,
      "log_odds_chosen": 2.5015196800231934,
      "log_odds_ratio": -0.40100595355033875,
      "logits/chosen": -0.09599044919013977,
      "logits/rejected": -0.06666186451911926,
      "logps/chosen": -0.8881614804267883,
      "logps/rejected": -3.0457301139831543,
      "loss": 2.8761,
      "nll_loss": 2.8360183238983154,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08881615102291107,
      "rewards/margins": 0.2157568335533142,
      "rewards/rejected": -0.3045729994773865,
      "step": 501
    },
    {
      "epoch": 0.31228615863141523,
      "grad_norm": 0.5571648478507996,
      "learning_rate": 3.745e-05,
      "log_odds_chosen": 3.5235862731933594,
      "log_odds_ratio": -0.2553158402442932,
      "logits/chosen": 0.08271875232458115,
      "logits/rejected": 0.159298837184906,
      "logps/chosen": -1.5579495429992676,
      "logps/rejected": -4.873930931091309,
      "loss": 3.0358,
      "nll_loss": 3.010316848754883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15579496324062347,
      "rewards/margins": 0.331598162651062,
      "rewards/rejected": -0.4873931109905243,
      "step": 502
    },
    {
      "epoch": 0.31290824261275274,
      "grad_norm": 0.9667811393737793,
      "learning_rate": 3.7425e-05,
      "log_odds_chosen": 1.5735365152359009,
      "log_odds_ratio": -0.4731370806694031,
      "logits/chosen": 0.03942291438579559,
      "logits/rejected": -0.06971655786037445,
      "logps/chosen": -1.4615811109542847,
      "logps/rejected": -2.880722761154175,
      "loss": 3.1011,
      "nll_loss": 3.0537822246551514,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1461580991744995,
      "rewards/margins": 0.14191415905952454,
      "rewards/rejected": -0.28807225823402405,
      "step": 503
    },
    {
      "epoch": 0.3135303265940902,
      "grad_norm": 0.496913880109787,
      "learning_rate": 3.74e-05,
      "log_odds_chosen": 2.018817901611328,
      "log_odds_ratio": -0.37110626697540283,
      "logits/chosen": -0.06371671706438065,
      "logits/rejected": -0.035864636301994324,
      "logps/chosen": -0.9149460792541504,
      "logps/rejected": -2.4723567962646484,
      "loss": 2.7536,
      "nll_loss": 2.7165191173553467,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0914946049451828,
      "rewards/margins": 0.15574106574058533,
      "rewards/rejected": -0.24723568558692932,
      "step": 504
    },
    {
      "epoch": 0.3141524105754277,
      "grad_norm": 0.45206576585769653,
      "learning_rate": 3.737500000000001e-05,
      "log_odds_chosen": 2.5706026554107666,
      "log_odds_ratio": -0.44339919090270996,
      "logits/chosen": 0.18639887869358063,
      "logits/rejected": 0.09812915325164795,
      "logps/chosen": -1.0650825500488281,
      "logps/rejected": -3.2684202194213867,
      "loss": 3.6104,
      "nll_loss": 3.5660290718078613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10650825500488281,
      "rewards/margins": 0.2203337550163269,
      "rewards/rejected": -0.3268420100212097,
      "step": 505
    },
    {
      "epoch": 0.31477449455676515,
      "grad_norm": 0.46192917227745056,
      "learning_rate": 3.735e-05,
      "log_odds_chosen": 1.1359951496124268,
      "log_odds_ratio": -0.45393139123916626,
      "logits/chosen": 0.20836736261844635,
      "logits/rejected": 0.045999638736248016,
      "logps/chosen": -1.4677722454071045,
      "logps/rejected": -2.4714083671569824,
      "loss": 3.5338,
      "nll_loss": 3.4884276390075684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1467772126197815,
      "rewards/margins": 0.10036361217498779,
      "rewards/rejected": -0.24714083969593048,
      "step": 506
    },
    {
      "epoch": 0.31539657853810266,
      "grad_norm": 0.8858140110969543,
      "learning_rate": 3.7325000000000006e-05,
      "log_odds_chosen": 0.8142578601837158,
      "log_odds_ratio": -0.41193997859954834,
      "logits/chosen": 0.23445630073547363,
      "logits/rejected": 0.15942780673503876,
      "logps/chosen": -1.313204050064087,
      "logps/rejected": -1.9680757522583008,
      "loss": 3.3672,
      "nll_loss": 3.326014995574951,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13132041692733765,
      "rewards/margins": 0.06548717617988586,
      "rewards/rejected": -0.19680756330490112,
      "step": 507
    },
    {
      "epoch": 0.3160186625194401,
      "grad_norm": 0.513054370880127,
      "learning_rate": 3.73e-05,
      "log_odds_chosen": 1.9759085178375244,
      "log_odds_ratio": -0.40382054448127747,
      "logits/chosen": 0.1355867236852646,
      "logits/rejected": -0.025263607501983643,
      "logps/chosen": -1.2560956478118896,
      "logps/rejected": -3.0063164234161377,
      "loss": 3.0785,
      "nll_loss": 3.038083553314209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12560957670211792,
      "rewards/margins": 0.17502208054065704,
      "rewards/rejected": -0.30063164234161377,
      "step": 508
    },
    {
      "epoch": 0.3166407465007776,
      "grad_norm": 0.29589080810546875,
      "learning_rate": 3.7275000000000005e-05,
      "log_odds_chosen": 3.7314577102661133,
      "log_odds_ratio": -0.2652418315410614,
      "logits/chosen": 0.1787126213312149,
      "logits/rejected": 0.24298204481601715,
      "logps/chosen": -1.1753450632095337,
      "logps/rejected": -4.593437194824219,
      "loss": 3.1982,
      "nll_loss": 3.1716480255126953,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11753450334072113,
      "rewards/margins": 0.3418092131614685,
      "rewards/rejected": -0.45934373140335083,
      "step": 509
    },
    {
      "epoch": 0.31726283048211507,
      "grad_norm": 0.37716519832611084,
      "learning_rate": 3.7250000000000004e-05,
      "log_odds_chosen": 6.13275146484375,
      "log_odds_ratio": -0.0573071613907814,
      "logits/chosen": 0.1888727992773056,
      "logits/rejected": 0.2561497092247009,
      "logps/chosen": -1.123389720916748,
      "logps/rejected": -6.81281852722168,
      "loss": 3.15,
      "nll_loss": 3.1442646980285645,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11233897507190704,
      "rewards/margins": 0.5689429044723511,
      "rewards/rejected": -0.6812818050384521,
      "step": 510
    },
    {
      "epoch": 0.3178849144634526,
      "grad_norm": 0.48151102662086487,
      "learning_rate": 3.7225000000000004e-05,
      "log_odds_chosen": 2.6327872276306152,
      "log_odds_ratio": -0.2002970576286316,
      "logits/chosen": 0.3196089267730713,
      "logits/rejected": 0.27417632937431335,
      "logps/chosen": -1.2908976078033447,
      "logps/rejected": -3.636533260345459,
      "loss": 3.504,
      "nll_loss": 3.4839515686035156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12908975780010223,
      "rewards/margins": 0.23456355929374695,
      "rewards/rejected": -0.3636533319950104,
      "step": 511
    },
    {
      "epoch": 0.31850699844479,
      "grad_norm": 0.43662747740745544,
      "learning_rate": 3.72e-05,
      "log_odds_chosen": 4.564843654632568,
      "log_odds_ratio": -0.06708259880542755,
      "logits/chosen": 0.24633480608463287,
      "logits/rejected": 0.48307889699935913,
      "logps/chosen": -0.9144176840782166,
      "logps/rejected": -4.8161234855651855,
      "loss": 3.12,
      "nll_loss": 3.1133110523223877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09144177287817001,
      "rewards/margins": 0.39017054438591003,
      "rewards/rejected": -0.48161232471466064,
      "step": 512
    },
    {
      "epoch": 0.31912908242612753,
      "grad_norm": 0.5609356164932251,
      "learning_rate": 3.7175e-05,
      "log_odds_chosen": 5.504610538482666,
      "log_odds_ratio": -0.08810669928789139,
      "logits/chosen": 0.2721938490867615,
      "logits/rejected": 0.42912524938583374,
      "logps/chosen": -1.2787365913391113,
      "logps/rejected": -6.235076904296875,
      "loss": 3.0593,
      "nll_loss": 3.050450086593628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12787367403507233,
      "rewards/margins": 0.49563395977020264,
      "rewards/rejected": -0.6235076189041138,
      "step": 513
    },
    {
      "epoch": 0.319751166407465,
      "grad_norm": 0.3104366958141327,
      "learning_rate": 3.715e-05,
      "log_odds_chosen": 6.418957710266113,
      "log_odds_ratio": -0.17403581738471985,
      "logits/chosen": 0.2422790229320526,
      "logits/rejected": 0.45724615454673767,
      "logps/chosen": -1.1649143695831299,
      "logps/rejected": -7.193992614746094,
      "loss": 3.2466,
      "nll_loss": 3.2292304039001465,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11649143695831299,
      "rewards/margins": 0.6029078364372253,
      "rewards/rejected": -0.7193993330001831,
      "step": 514
    },
    {
      "epoch": 0.3203732503888025,
      "grad_norm": 0.4039505422115326,
      "learning_rate": 3.7125e-05,
      "log_odds_chosen": 4.793672561645508,
      "log_odds_ratio": -0.17864283919334412,
      "logits/chosen": 0.31905102729797363,
      "logits/rejected": 0.5531411170959473,
      "logps/chosen": -0.8685755133628845,
      "logps/rejected": -5.093353271484375,
      "loss": 3.4313,
      "nll_loss": 3.4134421348571777,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08685755729675293,
      "rewards/margins": 0.4224777817726135,
      "rewards/rejected": -0.5093352794647217,
      "step": 515
    },
    {
      "epoch": 0.32099533437014,
      "grad_norm": 0.6209097504615784,
      "learning_rate": 3.71e-05,
      "log_odds_chosen": 13.188308715820312,
      "log_odds_ratio": -0.03541084751486778,
      "logits/chosen": 0.2741844952106476,
      "logits/rejected": 0.9028793573379517,
      "logps/chosen": -1.2709870338439941,
      "logps/rejected": -13.850850105285645,
      "loss": 2.5904,
      "nll_loss": 2.5868520736694336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1270987093448639,
      "rewards/margins": 1.257986307144165,
      "rewards/rejected": -1.385085105895996,
      "step": 516
    },
    {
      "epoch": 0.32161741835147745,
      "grad_norm": 0.4564976990222931,
      "learning_rate": 3.707500000000001e-05,
      "log_odds_chosen": 8.428442001342773,
      "log_odds_ratio": -0.1521356999874115,
      "logits/chosen": 0.31411972641944885,
      "logits/rejected": 0.6657370328903198,
      "logps/chosen": -1.1624163389205933,
      "logps/rejected": -9.256325721740723,
      "loss": 3.2951,
      "nll_loss": 3.279919385910034,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11624162644147873,
      "rewards/margins": 0.8093909025192261,
      "rewards/rejected": -0.9256325960159302,
      "step": 517
    },
    {
      "epoch": 0.32223950233281495,
      "grad_norm": 0.4547824263572693,
      "learning_rate": 3.705e-05,
      "log_odds_chosen": 8.413576126098633,
      "log_odds_ratio": -0.18886855244636536,
      "logits/chosen": 0.28079456090927124,
      "logits/rejected": 0.5538430213928223,
      "logps/chosen": -1.1201964616775513,
      "logps/rejected": -9.210822105407715,
      "loss": 3.1612,
      "nll_loss": 3.1422665119171143,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11201964318752289,
      "rewards/margins": 0.8090626001358032,
      "rewards/rejected": -0.9210821986198425,
      "step": 518
    },
    {
      "epoch": 0.3228615863141524,
      "grad_norm": 0.47383055090904236,
      "learning_rate": 3.7025000000000005e-05,
      "log_odds_chosen": 7.101673126220703,
      "log_odds_ratio": -0.2151382565498352,
      "logits/chosen": 0.19054706394672394,
      "logits/rejected": 0.4452645480632782,
      "logps/chosen": -0.9745645523071289,
      "logps/rejected": -7.625181674957275,
      "loss": 2.8218,
      "nll_loss": 2.8003106117248535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09745645523071289,
      "rewards/margins": 0.6650617718696594,
      "rewards/rejected": -0.7625181674957275,
      "step": 519
    },
    {
      "epoch": 0.3234836702954899,
      "grad_norm": 0.6366752982139587,
      "learning_rate": 3.7e-05,
      "log_odds_chosen": 8.329351425170898,
      "log_odds_ratio": -0.3641209304332733,
      "logits/chosen": 0.14251962304115295,
      "logits/rejected": 0.6523263454437256,
      "logps/chosen": -1.3124771118164062,
      "logps/rejected": -9.418330192565918,
      "loss": 2.3793,
      "nll_loss": 2.3428869247436523,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13124771416187286,
      "rewards/margins": 0.8105852603912354,
      "rewards/rejected": -0.9418330192565918,
      "step": 520
    },
    {
      "epoch": 0.32410575427682736,
      "grad_norm": 0.4376268684864044,
      "learning_rate": 3.6975000000000004e-05,
      "log_odds_chosen": 10.756841659545898,
      "log_odds_ratio": -0.2801904082298279,
      "logits/chosen": 0.31616631150245667,
      "logits/rejected": 0.5765305757522583,
      "logps/chosen": -1.2397606372833252,
      "logps/rejected": -11.706258773803711,
      "loss": 3.3466,
      "nll_loss": 3.318598508834839,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12397606670856476,
      "rewards/margins": 1.046649694442749,
      "rewards/rejected": -1.1706258058547974,
      "step": 521
    },
    {
      "epoch": 0.32472783825816487,
      "grad_norm": 0.4194689393043518,
      "learning_rate": 3.6950000000000004e-05,
      "log_odds_chosen": 9.908319473266602,
      "log_odds_ratio": -0.1611042022705078,
      "logits/chosen": 0.32544493675231934,
      "logits/rejected": 0.6462383270263672,
      "logps/chosen": -1.2010586261749268,
      "logps/rejected": -10.787912368774414,
      "loss": 3.4332,
      "nll_loss": 3.4171175956726074,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12010586261749268,
      "rewards/margins": 0.9586854577064514,
      "rewards/rejected": -1.0787913799285889,
      "step": 522
    },
    {
      "epoch": 0.3253499222395023,
      "grad_norm": 1.505491018295288,
      "learning_rate": 3.6925e-05,
      "log_odds_chosen": 14.340056419372559,
      "log_odds_ratio": -0.0028261165134608746,
      "logits/chosen": 0.2382410168647766,
      "logits/rejected": 0.7966728806495667,
      "logps/chosen": -1.5478522777557373,
      "logps/rejected": -15.318090438842773,
      "loss": 2.7164,
      "nll_loss": 2.716071128845215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15478524565696716,
      "rewards/margins": 1.3770238161087036,
      "rewards/rejected": -1.5318089723587036,
      "step": 523
    },
    {
      "epoch": 0.3259720062208398,
      "grad_norm": 0.4445713460445404,
      "learning_rate": 3.69e-05,
      "log_odds_chosen": 10.326703071594238,
      "log_odds_ratio": -0.0006719048251397908,
      "logits/chosen": 0.380736768245697,
      "logits/rejected": 1.0418426990509033,
      "logps/chosen": -1.3673847913742065,
      "logps/rejected": -11.366878509521484,
      "loss": 3.4216,
      "nll_loss": 3.4214975833892822,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13673847913742065,
      "rewards/margins": 0.9999493956565857,
      "rewards/rejected": -1.1366878747940063,
      "step": 524
    },
    {
      "epoch": 0.3265940902021773,
      "grad_norm": 0.7201062440872192,
      "learning_rate": 3.6875e-05,
      "log_odds_chosen": 10.034008979797363,
      "log_odds_ratio": -0.09337079524993896,
      "logits/chosen": 0.24129562079906464,
      "logits/rejected": 0.7426089644432068,
      "logps/chosen": -1.3333206176757812,
      "logps/rejected": -11.076189041137695,
      "loss": 2.9608,
      "nll_loss": 2.951493263244629,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13333207368850708,
      "rewards/margins": 0.9742867946624756,
      "rewards/rejected": -1.1076189279556274,
      "step": 525
    },
    {
      "epoch": 0.3272161741835148,
      "grad_norm": 0.3761167526245117,
      "learning_rate": 3.685e-05,
      "log_odds_chosen": 3.660090923309326,
      "log_odds_ratio": -0.5031037330627441,
      "logits/chosen": 0.13932842016220093,
      "logits/rejected": 0.2806503176689148,
      "logps/chosen": -1.4467041492462158,
      "logps/rejected": -5.056726455688477,
      "loss": 3.1734,
      "nll_loss": 3.1230688095092773,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14467042684555054,
      "rewards/margins": 0.36100226640701294,
      "rewards/rejected": -0.5056726932525635,
      "step": 526
    },
    {
      "epoch": 0.32783825816485224,
      "grad_norm": 0.48558077216148376,
      "learning_rate": 3.6825e-05,
      "log_odds_chosen": 7.3520402908325195,
      "log_odds_ratio": -0.22055096924304962,
      "logits/chosen": 0.23364008963108063,
      "logits/rejected": 0.43131324648857117,
      "logps/chosen": -0.8987447023391724,
      "logps/rejected": -7.75795316696167,
      "loss": 3.0376,
      "nll_loss": 3.0155234336853027,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08987447619438171,
      "rewards/margins": 0.6859208345413208,
      "rewards/rejected": -0.7757953405380249,
      "step": 527
    },
    {
      "epoch": 0.32846034214618974,
      "grad_norm": 0.5533658266067505,
      "learning_rate": 3.68e-05,
      "log_odds_chosen": 7.092010498046875,
      "log_odds_ratio": -0.21083347499370575,
      "logits/chosen": 0.16821296513080597,
      "logits/rejected": 0.5857659578323364,
      "logps/chosen": -1.1237003803253174,
      "logps/rejected": -7.805783748626709,
      "loss": 2.9438,
      "nll_loss": 2.9227511882781982,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11237004399299622,
      "rewards/margins": 0.6682083606719971,
      "rewards/rejected": -0.7805783748626709,
      "step": 528
    },
    {
      "epoch": 0.3290824261275272,
      "grad_norm": 0.3960796594619751,
      "learning_rate": 3.6775000000000006e-05,
      "log_odds_chosen": 5.841064453125,
      "log_odds_ratio": -0.2500973641872406,
      "logits/chosen": 0.27127107977867126,
      "logits/rejected": 0.43577903509140015,
      "logps/chosen": -1.4152770042419434,
      "logps/rejected": -7.040769100189209,
      "loss": 3.6124,
      "nll_loss": 3.587347984313965,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1415277123451233,
      "rewards/margins": 0.5625491738319397,
      "rewards/rejected": -0.704076886177063,
      "step": 529
    },
    {
      "epoch": 0.3297045101088647,
      "grad_norm": 0.5108814835548401,
      "learning_rate": 3.675e-05,
      "log_odds_chosen": 4.806069850921631,
      "log_odds_ratio": -0.23009052872657776,
      "logits/chosen": -0.039525844156742096,
      "logits/rejected": 0.47278881072998047,
      "logps/chosen": -1.1424496173858643,
      "logps/rejected": -5.656998157501221,
      "loss": 2.4652,
      "nll_loss": 2.442180633544922,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1142449676990509,
      "rewards/margins": 0.4514548182487488,
      "rewards/rejected": -0.5656998157501221,
      "step": 530
    },
    {
      "epoch": 0.33032659409020215,
      "grad_norm": 0.4421898424625397,
      "learning_rate": 3.6725000000000005e-05,
      "log_odds_chosen": 4.877275466918945,
      "log_odds_ratio": -0.16743728518486023,
      "logits/chosen": 0.05251733213663101,
      "logits/rejected": 0.2228899598121643,
      "logps/chosen": -0.9268969297409058,
      "logps/rejected": -5.308384418487549,
      "loss": 2.8371,
      "nll_loss": 2.820338010787964,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09268969297409058,
      "rewards/margins": 0.4381487965583801,
      "rewards/rejected": -0.5308384895324707,
      "step": 531
    },
    {
      "epoch": 0.33094867807153966,
      "grad_norm": 0.33455726504325867,
      "learning_rate": 3.6700000000000004e-05,
      "log_odds_chosen": 3.417280912399292,
      "log_odds_ratio": -0.38553696870803833,
      "logits/chosen": 0.15889662504196167,
      "logits/rejected": 0.3938848376274109,
      "logps/chosen": -1.285894751548767,
      "logps/rejected": -4.560215950012207,
      "loss": 3.2018,
      "nll_loss": 3.1632089614868164,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1285894811153412,
      "rewards/margins": 0.3274320960044861,
      "rewards/rejected": -0.45602160692214966,
      "step": 532
    },
    {
      "epoch": 0.33157076205287717,
      "grad_norm": 0.3787088096141815,
      "learning_rate": 3.6675000000000004e-05,
      "log_odds_chosen": 9.068405151367188,
      "log_odds_ratio": -0.08513309061527252,
      "logits/chosen": 0.15244531631469727,
      "logits/rejected": 0.667584240436554,
      "logps/chosen": -1.1898918151855469,
      "logps/rejected": -9.916119575500488,
      "loss": 2.9166,
      "nll_loss": 2.9080939292907715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11898918449878693,
      "rewards/margins": 0.8726227879524231,
      "rewards/rejected": -0.9916119575500488,
      "step": 533
    },
    {
      "epoch": 0.3321928460342146,
      "grad_norm": 0.39991286396980286,
      "learning_rate": 3.665e-05,
      "log_odds_chosen": 12.67526912689209,
      "log_odds_ratio": -0.1024663895368576,
      "logits/chosen": 0.21998846530914307,
      "logits/rejected": 0.7742512226104736,
      "logps/chosen": -1.0572272539138794,
      "logps/rejected": -13.273682594299316,
      "loss": 2.9368,
      "nll_loss": 2.9265105724334717,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10572272539138794,
      "rewards/margins": 1.2216455936431885,
      "rewards/rejected": -1.3273682594299316,
      "step": 534
    },
    {
      "epoch": 0.3328149300155521,
      "grad_norm": 0.49979427456855774,
      "learning_rate": 3.6625e-05,
      "log_odds_chosen": 1.2147586345672607,
      "log_odds_ratio": -0.6691287755966187,
      "logits/chosen": 0.12728288769721985,
      "logits/rejected": 0.22928330302238464,
      "logps/chosen": -1.2779240608215332,
      "logps/rejected": -2.464627265930176,
      "loss": 2.5579,
      "nll_loss": 2.491028308868408,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12779241800308228,
      "rewards/margins": 0.11867032200098038,
      "rewards/rejected": -0.24646273255348206,
      "step": 535
    },
    {
      "epoch": 0.3334370139968896,
      "grad_norm": 0.5618945360183716,
      "learning_rate": 3.66e-05,
      "log_odds_chosen": 5.239035129547119,
      "log_odds_ratio": -0.4188866913318634,
      "logits/chosen": 0.32197320461273193,
      "logits/rejected": 0.5903685092926025,
      "logps/chosen": -1.3105459213256836,
      "logps/rejected": -6.369636535644531,
      "loss": 3.088,
      "nll_loss": 3.0460805892944336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1310545951128006,
      "rewards/margins": 0.5059090852737427,
      "rewards/rejected": -0.6369636654853821,
      "step": 536
    },
    {
      "epoch": 0.3340590979782271,
      "grad_norm": 0.5546212792396545,
      "learning_rate": 3.6575e-05,
      "log_odds_chosen": 8.182401657104492,
      "log_odds_ratio": -0.22162377834320068,
      "logits/chosen": 0.09892675280570984,
      "logits/rejected": 0.21920329332351685,
      "logps/chosen": -1.0027060508728027,
      "logps/rejected": -8.776458740234375,
      "loss": 2.8454,
      "nll_loss": 2.823225975036621,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1002705991268158,
      "rewards/margins": 0.777375340461731,
      "rewards/rejected": -0.8776458501815796,
      "step": 537
    },
    {
      "epoch": 0.33468118195956453,
      "grad_norm": 0.47566530108451843,
      "learning_rate": 3.655e-05,
      "log_odds_chosen": 5.837797164916992,
      "log_odds_ratio": -0.18034599721431732,
      "logits/chosen": 0.30575647950172424,
      "logits/rejected": 0.5681968331336975,
      "logps/chosen": -1.1159521341323853,
      "logps/rejected": -6.579317092895508,
      "loss": 3.0867,
      "nll_loss": 3.068636178970337,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11159522086381912,
      "rewards/margins": 0.5463365316390991,
      "rewards/rejected": -0.6579316854476929,
      "step": 538
    },
    {
      "epoch": 0.33530326594090204,
      "grad_norm": 0.5338764786720276,
      "learning_rate": 3.652500000000001e-05,
      "log_odds_chosen": 9.230720520019531,
      "log_odds_ratio": -0.18835598230361938,
      "logits/chosen": 0.4432418942451477,
      "logits/rejected": 1.0067334175109863,
      "logps/chosen": -0.791349470615387,
      "logps/rejected": -9.255400657653809,
      "loss": 3.4137,
      "nll_loss": 3.3948874473571777,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07913494855165482,
      "rewards/margins": 0.8464051485061646,
      "rewards/rejected": -0.9255399703979492,
      "step": 539
    },
    {
      "epoch": 0.3359253499222395,
      "grad_norm": 0.4006078839302063,
      "learning_rate": 3.65e-05,
      "log_odds_chosen": 7.939189434051514,
      "log_odds_ratio": -0.17464585602283478,
      "logits/chosen": 0.3552432358264923,
      "logits/rejected": 0.7513606548309326,
      "logps/chosen": -1.0403473377227783,
      "logps/rejected": -8.616355895996094,
      "loss": 3.3614,
      "nll_loss": 3.3439626693725586,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10403473675251007,
      "rewards/margins": 0.7576008439064026,
      "rewards/rejected": -0.8616355061531067,
      "step": 540
    },
    {
      "epoch": 0.336547433903577,
      "grad_norm": 0.652712345123291,
      "learning_rate": 3.6475000000000006e-05,
      "log_odds_chosen": 7.361583709716797,
      "log_odds_ratio": -0.44913148880004883,
      "logits/chosen": 0.5314218997955322,
      "logits/rejected": 0.9203664660453796,
      "logps/chosen": -1.7339417934417725,
      "logps/rejected": -8.831292152404785,
      "loss": 3.4607,
      "nll_loss": 3.4157681465148926,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17339418828487396,
      "rewards/margins": 0.7097350358963013,
      "rewards/rejected": -0.8831292390823364,
      "step": 541
    },
    {
      "epoch": 0.33716951788491445,
      "grad_norm": 0.5391427278518677,
      "learning_rate": 3.645e-05,
      "log_odds_chosen": 5.927590370178223,
      "log_odds_ratio": -0.17163582146167755,
      "logits/chosen": 0.45557430386543274,
      "logits/rejected": 0.758292019367218,
      "logps/chosen": -1.0969210863113403,
      "logps/rejected": -6.686923980712891,
      "loss": 3.0634,
      "nll_loss": 3.0462679862976074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10969211906194687,
      "rewards/margins": 0.5590003728866577,
      "rewards/rejected": -0.6686924695968628,
      "step": 542
    },
    {
      "epoch": 0.33779160186625196,
      "grad_norm": 0.5085594058036804,
      "learning_rate": 3.6425000000000004e-05,
      "log_odds_chosen": 6.805022239685059,
      "log_odds_ratio": -0.04457619786262512,
      "logits/chosen": 0.2579970359802246,
      "logits/rejected": 0.8251427412033081,
      "logps/chosen": -1.0637521743774414,
      "logps/rejected": -7.367990016937256,
      "loss": 2.7691,
      "nll_loss": 2.7646713256835938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10637522488832474,
      "rewards/margins": 0.6304237246513367,
      "rewards/rejected": -0.7367990016937256,
      "step": 543
    },
    {
      "epoch": 0.3384136858475894,
      "grad_norm": 1.546451449394226,
      "learning_rate": 3.6400000000000004e-05,
      "log_odds_chosen": 7.0844831466674805,
      "log_odds_ratio": -0.15514059364795685,
      "logits/chosen": 0.32124999165534973,
      "logits/rejected": 0.5545608997344971,
      "logps/chosen": -1.4081883430480957,
      "logps/rejected": -8.002361297607422,
      "loss": 3.4905,
      "nll_loss": 3.475017547607422,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14081883430480957,
      "rewards/margins": 0.6594172716140747,
      "rewards/rejected": -0.8002361059188843,
      "step": 544
    },
    {
      "epoch": 0.3390357698289269,
      "grad_norm": 0.9646157622337341,
      "learning_rate": 3.6375e-05,
      "log_odds_chosen": 5.977232933044434,
      "log_odds_ratio": -0.31709274649620056,
      "logits/chosen": 0.2529332637786865,
      "logits/rejected": 0.3822195529937744,
      "logps/chosen": -1.1004133224487305,
      "logps/rejected": -6.800418376922607,
      "loss": 3.3593,
      "nll_loss": 3.3275961875915527,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11004132777452469,
      "rewards/margins": 0.5700005292892456,
      "rewards/rejected": -0.6800418496131897,
      "step": 545
    },
    {
      "epoch": 0.33965785381026437,
      "grad_norm": 0.4293280839920044,
      "learning_rate": 3.635e-05,
      "log_odds_chosen": 8.562162399291992,
      "log_odds_ratio": -0.13144966959953308,
      "logits/chosen": 0.12223690003156662,
      "logits/rejected": 0.3512270450592041,
      "logps/chosen": -0.9846524596214294,
      "logps/rejected": -9.026788711547852,
      "loss": 2.9709,
      "nll_loss": 2.957728385925293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09846524894237518,
      "rewards/margins": 0.8042137026786804,
      "rewards/rejected": -0.9026789665222168,
      "step": 546
    },
    {
      "epoch": 0.34027993779160187,
      "grad_norm": 0.5592858195304871,
      "learning_rate": 3.6325e-05,
      "log_odds_chosen": 7.228453159332275,
      "log_odds_ratio": -0.07215896248817444,
      "logits/chosen": 0.19593659043312073,
      "logits/rejected": 0.7204120755195618,
      "logps/chosen": -1.225344181060791,
      "logps/rejected": -8.123269081115723,
      "loss": 3.223,
      "nll_loss": 3.2157747745513916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12253442406654358,
      "rewards/margins": 0.6897925138473511,
      "rewards/rejected": -0.8123269081115723,
      "step": 547
    },
    {
      "epoch": 0.3409020217729393,
      "grad_norm": 0.3459852635860443,
      "learning_rate": 3.63e-05,
      "log_odds_chosen": 8.127766609191895,
      "log_odds_ratio": -0.10982684791088104,
      "logits/chosen": 0.18933087587356567,
      "logits/rejected": 0.6136522889137268,
      "logps/chosen": -1.2939542531967163,
      "logps/rejected": -9.101163864135742,
      "loss": 3.2767,
      "nll_loss": 3.265726089477539,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12939542531967163,
      "rewards/margins": 0.7807210683822632,
      "rewards/rejected": -0.91011643409729,
      "step": 548
    },
    {
      "epoch": 0.34152410575427683,
      "grad_norm": 0.5195790529251099,
      "learning_rate": 3.6275e-05,
      "log_odds_chosen": 10.92127799987793,
      "log_odds_ratio": -0.03485168516635895,
      "logits/chosen": 0.1153639703989029,
      "logits/rejected": 1.0010063648223877,
      "logps/chosen": -0.8691779375076294,
      "logps/rejected": -11.154041290283203,
      "loss": 2.4175,
      "nll_loss": 2.414062976837158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08691780269145966,
      "rewards/margins": 1.0284863710403442,
      "rewards/rejected": -1.1154041290283203,
      "step": 549
    },
    {
      "epoch": 0.3421461897356143,
      "grad_norm": 0.4225841760635376,
      "learning_rate": 3.625e-05,
      "log_odds_chosen": 11.029512405395508,
      "log_odds_ratio": -0.003762052860110998,
      "logits/chosen": 0.1677618771791458,
      "logits/rejected": 0.9653577208518982,
      "logps/chosen": -1.1759905815124512,
      "logps/rejected": -11.536764144897461,
      "loss": 3.2061,
      "nll_loss": 3.2056827545166016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11759904772043228,
      "rewards/margins": 1.0360772609710693,
      "rewards/rejected": -1.1536762714385986,
      "step": 550
    },
    {
      "epoch": 0.3427682737169518,
      "grad_norm": 0.4881388247013092,
      "learning_rate": 3.6225000000000006e-05,
      "log_odds_chosen": 10.496110916137695,
      "log_odds_ratio": -0.012914421036839485,
      "logits/chosen": 0.15544462203979492,
      "logits/rejected": 0.910711944103241,
      "logps/chosen": -1.2286415100097656,
      "logps/rejected": -11.356847763061523,
      "loss": 2.8724,
      "nll_loss": 2.871134042739868,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12286416441202164,
      "rewards/margins": 1.0128206014633179,
      "rewards/rejected": -1.1356847286224365,
      "step": 551
    },
    {
      "epoch": 0.3433903576982893,
      "grad_norm": 0.5085164308547974,
      "learning_rate": 3.62e-05,
      "log_odds_chosen": 3.2137742042541504,
      "log_odds_ratio": -0.2135535031557083,
      "logits/chosen": 0.052660077810287476,
      "logits/rejected": 0.18513916432857513,
      "logps/chosen": -1.4653301239013672,
      "logps/rejected": -4.4103498458862305,
      "loss": 3.3976,
      "nll_loss": 3.37619686126709,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14653301239013672,
      "rewards/margins": 0.294501930475235,
      "rewards/rejected": -0.4410349726676941,
      "step": 552
    },
    {
      "epoch": 0.34401244167962675,
      "grad_norm": 0.45229020714759827,
      "learning_rate": 3.6175000000000005e-05,
      "log_odds_chosen": 6.850950241088867,
      "log_odds_ratio": -0.22252152860164642,
      "logits/chosen": 0.07053791731595993,
      "logits/rejected": 0.5561230182647705,
      "logps/chosen": -1.2072479724884033,
      "logps/rejected": -7.82691764831543,
      "loss": 2.8457,
      "nll_loss": 2.8234715461730957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12072480469942093,
      "rewards/margins": 0.6619669198989868,
      "rewards/rejected": -0.7826917767524719,
      "step": 553
    },
    {
      "epoch": 0.34463452566096425,
      "grad_norm": 0.38006460666656494,
      "learning_rate": 3.615e-05,
      "log_odds_chosen": 4.494772911071777,
      "log_odds_ratio": -0.40767160058021545,
      "logits/chosen": 0.1025007963180542,
      "logits/rejected": 0.28547078371047974,
      "logps/chosen": -1.0280405282974243,
      "logps/rejected": -5.124785900115967,
      "loss": 3.0916,
      "nll_loss": 3.050813674926758,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10280404984951019,
      "rewards/margins": 0.4096745550632477,
      "rewards/rejected": -0.5124785900115967,
      "step": 554
    },
    {
      "epoch": 0.3452566096423017,
      "grad_norm": 0.41790586709976196,
      "learning_rate": 3.6125000000000004e-05,
      "log_odds_chosen": 9.663520812988281,
      "log_odds_ratio": -0.1240549236536026,
      "logits/chosen": 0.013118164613842964,
      "logits/rejected": 0.7078239917755127,
      "logps/chosen": -1.0113468170166016,
      "logps/rejected": -10.269187927246094,
      "loss": 2.3077,
      "nll_loss": 2.2952585220336914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10113468766212463,
      "rewards/margins": 0.9257840514183044,
      "rewards/rejected": -1.026918888092041,
      "step": 555
    },
    {
      "epoch": 0.3458786936236392,
      "grad_norm": 0.42868050932884216,
      "learning_rate": 3.61e-05,
      "log_odds_chosen": 11.801397323608398,
      "log_odds_ratio": -0.05558867007493973,
      "logits/chosen": 0.11525575071573257,
      "logits/rejected": 0.5333128571510315,
      "logps/chosen": -0.6986356377601624,
      "logps/rejected": -11.682586669921875,
      "loss": 2.5445,
      "nll_loss": 2.538949728012085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06986355781555176,
      "rewards/margins": 1.0983952283859253,
      "rewards/rejected": -1.1682586669921875,
      "step": 556
    },
    {
      "epoch": 0.34650077760497666,
      "grad_norm": 0.3941185772418976,
      "learning_rate": 3.6075e-05,
      "log_odds_chosen": 9.488451957702637,
      "log_odds_ratio": -0.022738970816135406,
      "logits/chosen": 0.2211320698261261,
      "logits/rejected": 0.9476622343063354,
      "logps/chosen": -1.3519847393035889,
      "logps/rejected": -10.521648406982422,
      "loss": 3.1268,
      "nll_loss": 3.1244912147521973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1351984739303589,
      "rewards/margins": 0.9169663190841675,
      "rewards/rejected": -1.0521647930145264,
      "step": 557
    },
    {
      "epoch": 0.34712286158631417,
      "grad_norm": 0.5461376309394836,
      "learning_rate": 3.605e-05,
      "log_odds_chosen": 6.737714767456055,
      "log_odds_ratio": -0.270801305770874,
      "logits/chosen": 0.20157453417778015,
      "logits/rejected": 0.566937267780304,
      "logps/chosen": -1.1833453178405762,
      "logps/rejected": -7.544872760772705,
      "loss": 3.1792,
      "nll_loss": 3.152073621749878,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11833453178405762,
      "rewards/margins": 0.6361527442932129,
      "rewards/rejected": -0.7544872760772705,
      "step": 558
    },
    {
      "epoch": 0.3477449455676516,
      "grad_norm": 0.5232445597648621,
      "learning_rate": 3.6025e-05,
      "log_odds_chosen": 14.541240692138672,
      "log_odds_ratio": -0.1170097142457962,
      "logits/chosen": 0.005186443217098713,
      "logits/rejected": 0.5835030674934387,
      "logps/chosen": -1.1031867265701294,
      "logps/rejected": -15.032424926757812,
      "loss": 2.6264,
      "nll_loss": 2.6146867275238037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11031867563724518,
      "rewards/margins": 1.3929238319396973,
      "rewards/rejected": -1.5032424926757812,
      "step": 559
    },
    {
      "epoch": 0.3483670295489891,
      "grad_norm": 0.40398097038269043,
      "learning_rate": 3.6e-05,
      "log_odds_chosen": 10.217870712280273,
      "log_odds_ratio": -0.021281030029058456,
      "logits/chosen": 0.26363763213157654,
      "logits/rejected": 0.7080330848693848,
      "logps/chosen": -1.2973829507827759,
      "logps/rejected": -11.150710105895996,
      "loss": 3.5862,
      "nll_loss": 3.58408260345459,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12973830103874207,
      "rewards/margins": 0.985332727432251,
      "rewards/rejected": -1.1150710582733154,
      "step": 560
    },
    {
      "epoch": 0.3489891135303266,
      "grad_norm": 0.3686501085758209,
      "learning_rate": 3.5975e-05,
      "log_odds_chosen": 11.192573547363281,
      "log_odds_ratio": -0.00902550108730793,
      "logits/chosen": 0.07758531719446182,
      "logits/rejected": 0.7558944225311279,
      "logps/chosen": -0.9165749549865723,
      "logps/rejected": -11.570627212524414,
      "loss": 2.5896,
      "nll_loss": 2.5887372493743896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09165749698877335,
      "rewards/margins": 1.0654051303863525,
      "rewards/rejected": -1.1570627689361572,
      "step": 561
    },
    {
      "epoch": 0.3496111975116641,
      "grad_norm": 0.3752516508102417,
      "learning_rate": 3.595e-05,
      "log_odds_chosen": 13.246245384216309,
      "log_odds_ratio": -0.1449487805366516,
      "logits/chosen": 0.2694312334060669,
      "logits/rejected": 1.2381070852279663,
      "logps/chosen": -1.137485384941101,
      "logps/rejected": -14.047223091125488,
      "loss": 3.0143,
      "nll_loss": 2.999814033508301,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11374853551387787,
      "rewards/margins": 1.2909739017486572,
      "rewards/rejected": -1.4047224521636963,
      "step": 562
    },
    {
      "epoch": 0.35023328149300154,
      "grad_norm": 0.3949683904647827,
      "learning_rate": 3.5925000000000006e-05,
      "log_odds_chosen": 8.974077224731445,
      "log_odds_ratio": -0.0277986042201519,
      "logits/chosen": 0.18479128181934357,
      "logits/rejected": 0.6321989297866821,
      "logps/chosen": -0.7476387023925781,
      "logps/rejected": -8.921815872192383,
      "loss": 3.007,
      "nll_loss": 3.004196882247925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07476387172937393,
      "rewards/margins": 0.8174177408218384,
      "rewards/rejected": -0.8921815752983093,
      "step": 563
    },
    {
      "epoch": 0.35085536547433904,
      "grad_norm": 5.831315994262695,
      "learning_rate": 3.59e-05,
      "log_odds_chosen": 9.392239570617676,
      "log_odds_ratio": -0.17418979108333588,
      "logits/chosen": 0.1219196617603302,
      "logits/rejected": 0.9300506114959717,
      "logps/chosen": -1.233995795249939,
      "logps/rejected": -10.267557144165039,
      "loss": 2.6961,
      "nll_loss": 2.6787209510803223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12339960038661957,
      "rewards/margins": 0.9033560752868652,
      "rewards/rejected": -1.026755690574646,
      "step": 564
    },
    {
      "epoch": 0.3514774494556765,
      "grad_norm": 0.4015295207500458,
      "learning_rate": 3.5875000000000005e-05,
      "log_odds_chosen": 11.647599220275879,
      "log_odds_ratio": -0.007850930094718933,
      "logits/chosen": 0.2969825267791748,
      "logits/rejected": 0.9183492660522461,
      "logps/chosen": -1.142836332321167,
      "logps/rejected": -12.2813720703125,
      "loss": 3.3011,
      "nll_loss": 3.3002769947052,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11428363621234894,
      "rewards/margins": 1.1138535737991333,
      "rewards/rejected": -1.2281371355056763,
      "step": 565
    },
    {
      "epoch": 0.352099533437014,
      "grad_norm": 0.8392999768257141,
      "learning_rate": 3.585e-05,
      "log_odds_chosen": 15.985481262207031,
      "log_odds_ratio": -0.0001611363550182432,
      "logits/chosen": 0.33824992179870605,
      "logits/rejected": 1.2024070024490356,
      "logps/chosen": -1.6567131280899048,
      "logps/rejected": -17.20331573486328,
      "loss": 3.5828,
      "nll_loss": 3.5828089714050293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16567131876945496,
      "rewards/margins": 1.5546603202819824,
      "rewards/rejected": -1.7203315496444702,
      "step": 566
    },
    {
      "epoch": 0.35272161741835145,
      "grad_norm": 0.4848373830318451,
      "learning_rate": 3.5825000000000003e-05,
      "log_odds_chosen": 13.249602317810059,
      "log_odds_ratio": -0.12133760005235672,
      "logits/chosen": 0.1779986023902893,
      "logits/rejected": 1.0257817506790161,
      "logps/chosen": -1.069126009941101,
      "logps/rejected": -13.913080215454102,
      "loss": 2.6563,
      "nll_loss": 2.644139289855957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10691259801387787,
      "rewards/margins": 1.284395456314087,
      "rewards/rejected": -1.391308069229126,
      "step": 567
    },
    {
      "epoch": 0.35334370139968896,
      "grad_norm": 0.4035547375679016,
      "learning_rate": 3.58e-05,
      "log_odds_chosen": 13.095964431762695,
      "log_odds_ratio": -0.006714683026075363,
      "logits/chosen": 0.09633934497833252,
      "logits/rejected": 0.7238105535507202,
      "logps/chosen": -0.9360955357551575,
      "logps/rejected": -13.495253562927246,
      "loss": 2.7449,
      "nll_loss": 2.744229555130005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09360955655574799,
      "rewards/margins": 1.255915880203247,
      "rewards/rejected": -1.3495254516601562,
      "step": 568
    },
    {
      "epoch": 0.35396578538102647,
      "grad_norm": 0.7661040425300598,
      "learning_rate": 3.5775e-05,
      "log_odds_chosen": 9.719877243041992,
      "log_odds_ratio": -0.23540478944778442,
      "logits/chosen": 0.3083692789077759,
      "logits/rejected": 1.01234769821167,
      "logps/chosen": -1.212859034538269,
      "logps/rejected": -10.61805534362793,
      "loss": 3.1301,
      "nll_loss": 3.1065545082092285,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12128590047359467,
      "rewards/margins": 0.9405196905136108,
      "rewards/rejected": -1.0618056058883667,
      "step": 569
    },
    {
      "epoch": 0.3545878693623639,
      "grad_norm": 0.3926217257976532,
      "learning_rate": 3.575e-05,
      "log_odds_chosen": 10.073858261108398,
      "log_odds_ratio": -0.1235310509800911,
      "logits/chosen": 0.13625189661979675,
      "logits/rejected": 0.7520464658737183,
      "logps/chosen": -1.2251389026641846,
      "logps/rejected": -10.796701431274414,
      "loss": 2.7511,
      "nll_loss": 2.738753318786621,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12251389026641846,
      "rewards/margins": 0.957156240940094,
      "rewards/rejected": -1.0796700716018677,
      "step": 570
    },
    {
      "epoch": 0.3552099533437014,
      "grad_norm": 0.8033375144004822,
      "learning_rate": 3.5725e-05,
      "log_odds_chosen": 4.0529279708862305,
      "log_odds_ratio": -0.20374912023544312,
      "logits/chosen": 0.1487824022769928,
      "logits/rejected": 0.3493782877922058,
      "logps/chosen": -1.0385345220565796,
      "logps/rejected": -4.73582649230957,
      "loss": 2.6228,
      "nll_loss": 2.6024231910705566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10385345667600632,
      "rewards/margins": 0.369729220867157,
      "rewards/rejected": -0.4735826551914215,
      "step": 571
    },
    {
      "epoch": 0.3558320373250389,
      "grad_norm": 0.38423430919647217,
      "learning_rate": 3.57e-05,
      "log_odds_chosen": 7.926821708679199,
      "log_odds_ratio": -0.0515306256711483,
      "logits/chosen": 0.3775959610939026,
      "logits/rejected": 0.7432839274406433,
      "logps/chosen": -1.2405486106872559,
      "logps/rejected": -8.742298126220703,
      "loss": 3.7806,
      "nll_loss": 3.775494337081909,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12405486404895782,
      "rewards/margins": 0.7501749992370605,
      "rewards/rejected": -0.874229907989502,
      "step": 572
    },
    {
      "epoch": 0.3564541213063764,
      "grad_norm": 0.5379979610443115,
      "learning_rate": 3.5675e-05,
      "log_odds_chosen": 4.479024410247803,
      "log_odds_ratio": -0.45168882608413696,
      "logits/chosen": 0.07607246190309525,
      "logits/rejected": 0.2333858758211136,
      "logps/chosen": -1.1368803977966309,
      "logps/rejected": -5.305628776550293,
      "loss": 2.7889,
      "nll_loss": 2.7437705993652344,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11368802934885025,
      "rewards/margins": 0.41687485575675964,
      "rewards/rejected": -0.5305629372596741,
      "step": 573
    },
    {
      "epoch": 0.35707620528771383,
      "grad_norm": 1.3011409044265747,
      "learning_rate": 3.565e-05,
      "log_odds_chosen": 3.907680034637451,
      "log_odds_ratio": -0.33975449204444885,
      "logits/chosen": 0.2571965754032135,
      "logits/rejected": 0.5251089930534363,
      "logps/chosen": -1.092237949371338,
      "logps/rejected": -4.7046799659729,
      "loss": 3.1342,
      "nll_loss": 3.1002023220062256,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10922379791736603,
      "rewards/margins": 0.36124420166015625,
      "rewards/rejected": -0.4704679846763611,
      "step": 574
    },
    {
      "epoch": 0.35769828926905134,
      "grad_norm": 0.6529536247253418,
      "learning_rate": 3.5625000000000005e-05,
      "log_odds_chosen": 7.877595901489258,
      "log_odds_ratio": -0.17618416249752045,
      "logits/chosen": 0.3382919430732727,
      "logits/rejected": 0.7670020461082458,
      "logps/chosen": -1.239711880683899,
      "logps/rejected": -8.82756233215332,
      "loss": 3.5095,
      "nll_loss": 3.4918410778045654,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12397119402885437,
      "rewards/margins": 0.7587849497795105,
      "rewards/rejected": -0.882756233215332,
      "step": 575
    },
    {
      "epoch": 0.3583203732503888,
      "grad_norm": 0.4566708505153656,
      "learning_rate": 3.56e-05,
      "log_odds_chosen": 11.294231414794922,
      "log_odds_ratio": -0.1296280026435852,
      "logits/chosen": 0.2539590299129486,
      "logits/rejected": 0.8094176054000854,
      "logps/chosen": -1.7060835361480713,
      "logps/rejected": -12.409913063049316,
      "loss": 3.248,
      "nll_loss": 3.234999656677246,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17060835659503937,
      "rewards/margins": 1.070383071899414,
      "rewards/rejected": -1.2409913539886475,
      "step": 576
    },
    {
      "epoch": 0.3589424572317263,
      "grad_norm": 0.46674543619155884,
      "learning_rate": 3.5575000000000004e-05,
      "log_odds_chosen": 3.2900829315185547,
      "log_odds_ratio": -0.31960904598236084,
      "logits/chosen": 0.08701478689908981,
      "logits/rejected": 0.1937023103237152,
      "logps/chosen": -0.9611541628837585,
      "logps/rejected": -3.7472667694091797,
      "loss": 2.775,
      "nll_loss": 2.743015766143799,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09611541777849197,
      "rewards/margins": 0.27861127257347107,
      "rewards/rejected": -0.37472671270370483,
      "step": 577
    },
    {
      "epoch": 0.35956454121306375,
      "grad_norm": 0.3776179552078247,
      "learning_rate": 3.555e-05,
      "log_odds_chosen": 3.8524093627929688,
      "log_odds_ratio": -0.1908583641052246,
      "logits/chosen": 0.1532333493232727,
      "logits/rejected": 0.4199594557285309,
      "logps/chosen": -1.2285915613174438,
      "logps/rejected": -4.827481269836426,
      "loss": 3.0596,
      "nll_loss": 3.0405075550079346,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1228591576218605,
      "rewards/margins": 0.3598890006542206,
      "rewards/rejected": -0.4827481508255005,
      "step": 578
    },
    {
      "epoch": 0.36018662519440126,
      "grad_norm": 0.5118311643600464,
      "learning_rate": 3.5525e-05,
      "log_odds_chosen": 7.782455921173096,
      "log_odds_ratio": -0.2796177864074707,
      "logits/chosen": 0.08961069583892822,
      "logits/rejected": 0.5031421780586243,
      "logps/chosen": -1.2598867416381836,
      "logps/rejected": -8.860183715820312,
      "loss": 2.4754,
      "nll_loss": 2.4474620819091797,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1259886771440506,
      "rewards/margins": 0.760029673576355,
      "rewards/rejected": -0.8860183954238892,
      "step": 579
    },
    {
      "epoch": 0.3608087091757387,
      "grad_norm": 0.40134724974632263,
      "learning_rate": 3.55e-05,
      "log_odds_chosen": 10.93133544921875,
      "log_odds_ratio": -0.1257864534854889,
      "logits/chosen": 0.13825632631778717,
      "logits/rejected": 0.43383318185806274,
      "logps/chosen": -1.1258553266525269,
      "logps/rejected": -11.62828540802002,
      "loss": 3.1864,
      "nll_loss": 3.173846960067749,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11258553713560104,
      "rewards/margins": 1.0502430200576782,
      "rewards/rejected": -1.1628284454345703,
      "step": 580
    },
    {
      "epoch": 0.3614307931570762,
      "grad_norm": 0.37659627199172974,
      "learning_rate": 3.5475e-05,
      "log_odds_chosen": 9.985986709594727,
      "log_odds_ratio": -0.02613000012934208,
      "logits/chosen": 0.2525095045566559,
      "logits/rejected": 0.599713146686554,
      "logps/chosen": -1.3548399209976196,
      "logps/rejected": -11.0252685546875,
      "loss": 3.217,
      "nll_loss": 3.214369535446167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1354840099811554,
      "rewards/margins": 0.967042863368988,
      "rewards/rejected": -1.1025269031524658,
      "step": 581
    },
    {
      "epoch": 0.36205287713841366,
      "grad_norm": 0.4002346098423004,
      "learning_rate": 3.545e-05,
      "log_odds_chosen": 7.260525226593018,
      "log_odds_ratio": -0.22730308771133423,
      "logits/chosen": 0.3146141767501831,
      "logits/rejected": 0.7007775902748108,
      "logps/chosen": -1.2889691591262817,
      "logps/rejected": -8.326926231384277,
      "loss": 3.4248,
      "nll_loss": 3.4020333290100098,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12889692187309265,
      "rewards/margins": 0.7037956714630127,
      "rewards/rejected": -0.8326926231384277,
      "step": 582
    },
    {
      "epoch": 0.36267496111975117,
      "grad_norm": 0.4641818404197693,
      "learning_rate": 3.5425e-05,
      "log_odds_chosen": 8.128804206848145,
      "log_odds_ratio": -0.26333796977996826,
      "logits/chosen": 0.2824775278568268,
      "logits/rejected": 0.928533673286438,
      "logps/chosen": -1.3320082426071167,
      "logps/rejected": -9.260583877563477,
      "loss": 2.9645,
      "nll_loss": 2.9382100105285645,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13320083916187286,
      "rewards/margins": 0.7928575873374939,
      "rewards/rejected": -0.9260584115982056,
      "step": 583
    },
    {
      "epoch": 0.3632970451010886,
      "grad_norm": 0.4908921718597412,
      "learning_rate": 3.54e-05,
      "log_odds_chosen": 7.648657321929932,
      "log_odds_ratio": -0.20667998492717743,
      "logits/chosen": 0.1102341040968895,
      "logits/rejected": 0.6346576809883118,
      "logps/chosen": -1.1072837114334106,
      "logps/rejected": -8.343591690063477,
      "loss": 2.6697,
      "nll_loss": 2.6490468978881836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11072837561368942,
      "rewards/margins": 0.7236307859420776,
      "rewards/rejected": -0.8343591094017029,
      "step": 584
    },
    {
      "epoch": 0.36391912908242613,
      "grad_norm": 0.42618241906166077,
      "learning_rate": 3.5375e-05,
      "log_odds_chosen": 6.4050774574279785,
      "log_odds_ratio": -0.31210067868232727,
      "logits/chosen": 0.232399120926857,
      "logits/rejected": 0.5041719079017639,
      "logps/chosen": -1.3073663711547852,
      "logps/rejected": -7.530215263366699,
      "loss": 3.0651,
      "nll_loss": 3.033935785293579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13073663413524628,
      "rewards/margins": 0.6222848892211914,
      "rewards/rejected": -0.7530215382575989,
      "step": 585
    },
    {
      "epoch": 0.3645412130637636,
      "grad_norm": 0.3700605034828186,
      "learning_rate": 3.535e-05,
      "log_odds_chosen": 9.741218566894531,
      "log_odds_ratio": -0.2235887050628662,
      "logits/chosen": 0.17234095931053162,
      "logits/rejected": 0.7051503658294678,
      "logps/chosen": -1.2577295303344727,
      "logps/rejected": -10.757645606994629,
      "loss": 2.9902,
      "nll_loss": 2.967867851257324,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12577295303344727,
      "rewards/margins": 0.9499915838241577,
      "rewards/rejected": -1.075764536857605,
      "step": 586
    },
    {
      "epoch": 0.3651632970451011,
      "grad_norm": 0.4975121021270752,
      "learning_rate": 3.5325000000000005e-05,
      "log_odds_chosen": 8.96474838256836,
      "log_odds_ratio": -0.33923766016960144,
      "logits/chosen": 0.3609507977962494,
      "logits/rejected": 0.8460711240768433,
      "logps/chosen": -1.3260118961334229,
      "logps/rejected": -10.090775489807129,
      "loss": 3.1647,
      "nll_loss": 3.1307742595672607,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13260118663311005,
      "rewards/margins": 0.8764763474464417,
      "rewards/rejected": -1.009077548980713,
      "step": 587
    },
    {
      "epoch": 0.3657853810264386,
      "grad_norm": 0.7693401575088501,
      "learning_rate": 3.53e-05,
      "log_odds_chosen": 3.7467236518859863,
      "log_odds_ratio": -0.6245080828666687,
      "logits/chosen": 0.09779280424118042,
      "logits/rejected": 0.26720985770225525,
      "logps/chosen": -1.4264414310455322,
      "logps/rejected": -4.837800025939941,
      "loss": 2.7459,
      "nll_loss": 2.6834800243377686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14264413714408875,
      "rewards/margins": 0.3411359190940857,
      "rewards/rejected": -0.48378005623817444,
      "step": 588
    },
    {
      "epoch": 0.36640746500777605,
      "grad_norm": 0.41673484444618225,
      "learning_rate": 3.5275000000000004e-05,
      "log_odds_chosen": 7.740867614746094,
      "log_odds_ratio": -0.17404800653457642,
      "logits/chosen": 0.23524239659309387,
      "logits/rejected": 0.43157610297203064,
      "logps/chosen": -0.9836545586585999,
      "logps/rejected": -8.333390235900879,
      "loss": 3.4373,
      "nll_loss": 3.419887065887451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09836546331644058,
      "rewards/margins": 0.7349735498428345,
      "rewards/rejected": -0.8333390951156616,
      "step": 589
    },
    {
      "epoch": 0.36702954898911355,
      "grad_norm": 0.5059419870376587,
      "learning_rate": 3.525e-05,
      "log_odds_chosen": 4.646327018737793,
      "log_odds_ratio": -0.17816919088363647,
      "logits/chosen": 0.14172425866127014,
      "logits/rejected": 0.35883334279060364,
      "logps/chosen": -1.3644582033157349,
      "logps/rejected": -5.79020881652832,
      "loss": 2.8427,
      "nll_loss": 2.8248753547668457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1364458203315735,
      "rewards/margins": 0.442575067281723,
      "rewards/rejected": -0.5790208578109741,
      "step": 590
    },
    {
      "epoch": 0.367651632970451,
      "grad_norm": 0.45872944593429565,
      "learning_rate": 3.5225e-05,
      "log_odds_chosen": 14.634015083312988,
      "log_odds_ratio": -0.07316752523183823,
      "logits/chosen": 0.20786404609680176,
      "logits/rejected": 1.1298377513885498,
      "logps/chosen": -1.0286473035812378,
      "logps/rejected": -15.15748405456543,
      "loss": 2.9729,
      "nll_loss": 2.965557098388672,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10286473482847214,
      "rewards/margins": 1.4128836393356323,
      "rewards/rejected": -1.5157485008239746,
      "step": 591
    },
    {
      "epoch": 0.3682737169517885,
      "grad_norm": 0.678117036819458,
      "learning_rate": 3.52e-05,
      "log_odds_chosen": 6.029167175292969,
      "log_odds_ratio": -0.36632686853408813,
      "logits/chosen": 0.255694717168808,
      "logits/rejected": 0.5321189761161804,
      "logps/chosen": -0.940081000328064,
      "logps/rejected": -6.712700843811035,
      "loss": 3.3547,
      "nll_loss": 3.3180642127990723,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09400809556245804,
      "rewards/margins": 0.5772619843482971,
      "rewards/rejected": -0.6712701320648193,
      "step": 592
    },
    {
      "epoch": 0.36889580093312596,
      "grad_norm": 0.415947824716568,
      "learning_rate": 3.5175e-05,
      "log_odds_chosen": 9.287468910217285,
      "log_odds_ratio": -0.33420291543006897,
      "logits/chosen": 0.1860431730747223,
      "logits/rejected": 0.6892718076705933,
      "logps/chosen": -1.1524367332458496,
      "logps/rejected": -10.236968994140625,
      "loss": 3.0043,
      "nll_loss": 2.9708549976348877,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11524367332458496,
      "rewards/margins": 0.9084532260894775,
      "rewards/rejected": -1.0236968994140625,
      "step": 593
    },
    {
      "epoch": 0.36951788491446347,
      "grad_norm": 0.6838355660438538,
      "learning_rate": 3.515e-05,
      "log_odds_chosen": 11.52259635925293,
      "log_odds_ratio": -0.16391010582447052,
      "logits/chosen": 0.14682823419570923,
      "logits/rejected": 0.7776925563812256,
      "logps/chosen": -1.0399855375289917,
      "logps/rejected": -12.213512420654297,
      "loss": 2.6591,
      "nll_loss": 2.642709493637085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10399855673313141,
      "rewards/margins": 1.1173527240753174,
      "rewards/rejected": -1.2213512659072876,
      "step": 594
    },
    {
      "epoch": 0.3701399688958009,
      "grad_norm": 0.5135987997055054,
      "learning_rate": 3.5125e-05,
      "log_odds_chosen": 10.131647109985352,
      "log_odds_ratio": -0.15441890060901642,
      "logits/chosen": 0.1124928668141365,
      "logits/rejected": 0.4724903106689453,
      "logps/chosen": -1.242720365524292,
      "logps/rejected": -11.012077331542969,
      "loss": 2.9245,
      "nll_loss": 2.9090170860290527,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12427204102277756,
      "rewards/margins": 0.9769356846809387,
      "rewards/rejected": -1.1012077331542969,
      "step": 595
    },
    {
      "epoch": 0.3707620528771384,
      "grad_norm": 40.30850601196289,
      "learning_rate": 3.51e-05,
      "log_odds_chosen": 12.990202903747559,
      "log_odds_ratio": -0.9076607823371887,
      "logits/chosen": 0.37599942088127136,
      "logits/rejected": 1.0942538976669312,
      "logps/chosen": -3.2952306270599365,
      "logps/rejected": -15.915863037109375,
      "loss": 3.9839,
      "nll_loss": 3.893106460571289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.32952308654785156,
      "rewards/margins": 1.2620632648468018,
      "rewards/rejected": -1.5915863513946533,
      "step": 596
    },
    {
      "epoch": 0.3713841368584759,
      "grad_norm": 30.15015983581543,
      "learning_rate": 3.5075000000000006e-05,
      "log_odds_chosen": 10.713224411010742,
      "log_odds_ratio": -0.17117588222026825,
      "logits/chosen": 0.28844138979911804,
      "logits/rejected": 0.7312687039375305,
      "logps/chosen": -2.2271180152893066,
      "logps/rejected": -12.698269844055176,
      "loss": 3.5309,
      "nll_loss": 3.5137851238250732,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22271178662776947,
      "rewards/margins": 1.0471150875091553,
      "rewards/rejected": -1.269826889038086,
      "step": 597
    },
    {
      "epoch": 0.3720062208398134,
      "grad_norm": 0.4343259036540985,
      "learning_rate": 3.505e-05,
      "log_odds_chosen": 16.90182876586914,
      "log_odds_ratio": -0.0005982535076327622,
      "logits/chosen": 0.18842889368534088,
      "logits/rejected": 0.9794631600379944,
      "logps/chosen": -1.1533726453781128,
      "logps/rejected": -17.613147735595703,
      "loss": 3.0408,
      "nll_loss": 3.0407166481018066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11533726751804352,
      "rewards/margins": 1.64597749710083,
      "rewards/rejected": -1.761314868927002,
      "step": 598
    },
    {
      "epoch": 0.37262830482115084,
      "grad_norm": 0.38888421654701233,
      "learning_rate": 3.5025000000000004e-05,
      "log_odds_chosen": 6.894892692565918,
      "log_odds_ratio": -0.13730782270431519,
      "logits/chosen": 0.24703556299209595,
      "logits/rejected": 0.6600065231323242,
      "logps/chosen": -0.8034927845001221,
      "logps/rejected": -7.0167083740234375,
      "loss": 3.3792,
      "nll_loss": 3.3654870986938477,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08034928143024445,
      "rewards/margins": 0.6213215589523315,
      "rewards/rejected": -0.7016708254814148,
      "step": 599
    },
    {
      "epoch": 0.37325038880248834,
      "grad_norm": 0.3728528320789337,
      "learning_rate": 3.5e-05,
      "log_odds_chosen": 10.392243385314941,
      "log_odds_ratio": -0.08781400322914124,
      "logits/chosen": 0.14524298906326294,
      "logits/rejected": 0.8344522714614868,
      "logps/chosen": -1.0598855018615723,
      "logps/rejected": -11.012116432189941,
      "loss": 2.7949,
      "nll_loss": 2.786162853240967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10598855465650558,
      "rewards/margins": 0.9952231645584106,
      "rewards/rejected": -1.101211667060852,
      "step": 600
    },
    {
      "epoch": 0.3738724727838258,
      "grad_norm": 0.4189922511577606,
      "learning_rate": 3.4975e-05,
      "log_odds_chosen": 2.0390243530273438,
      "log_odds_ratio": -0.39055636525154114,
      "logits/chosen": 0.08198841661214828,
      "logits/rejected": 0.19098129868507385,
      "logps/chosen": -1.0478918552398682,
      "logps/rejected": -2.8190433979034424,
      "loss": 2.9721,
      "nll_loss": 2.932997226715088,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10478918254375458,
      "rewards/margins": 0.17711517214775085,
      "rewards/rejected": -0.28190433979034424,
      "step": 601
    },
    {
      "epoch": 0.3744945567651633,
      "grad_norm": 0.4921775758266449,
      "learning_rate": 3.495e-05,
      "log_odds_chosen": 3.0136878490448,
      "log_odds_ratio": -0.41104233264923096,
      "logits/chosen": 0.2594601511955261,
      "logits/rejected": 0.3218687176704407,
      "logps/chosen": -1.2040386199951172,
      "logps/rejected": -4.0521650314331055,
      "loss": 3.3135,
      "nll_loss": 3.2724430561065674,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12040386348962784,
      "rewards/margins": 0.2848126292228699,
      "rewards/rejected": -0.4052164852619171,
      "step": 602
    },
    {
      "epoch": 0.37511664074650075,
      "grad_norm": 0.5495923161506653,
      "learning_rate": 3.4925e-05,
      "log_odds_chosen": 2.3447442054748535,
      "log_odds_ratio": -0.38162410259246826,
      "logits/chosen": 0.23126523196697235,
      "logits/rejected": 0.14873181283473969,
      "logps/chosen": -1.3720605373382568,
      "logps/rejected": -3.5602757930755615,
      "loss": 3.5564,
      "nll_loss": 3.5182876586914062,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1372060626745224,
      "rewards/margins": 0.21882151067256927,
      "rewards/rejected": -0.3560275733470917,
      "step": 603
    },
    {
      "epoch": 0.37573872472783826,
      "grad_norm": 0.5212722420692444,
      "learning_rate": 3.49e-05,
      "log_odds_chosen": 6.591350078582764,
      "log_odds_ratio": -0.1355379819869995,
      "logits/chosen": 0.07449732720851898,
      "logits/rejected": 0.338863730430603,
      "logps/chosen": -1.1350582838058472,
      "logps/rejected": -7.260507583618164,
      "loss": 2.7158,
      "nll_loss": 2.7022814750671387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11350581794977188,
      "rewards/margins": 0.6125448942184448,
      "rewards/rejected": -0.7260507941246033,
      "step": 604
    },
    {
      "epoch": 0.37636080870917576,
      "grad_norm": 0.47101324796676636,
      "learning_rate": 3.4875e-05,
      "log_odds_chosen": 2.375208616256714,
      "log_odds_ratio": -0.4302332103252411,
      "logits/chosen": 0.042256325483322144,
      "logits/rejected": 0.19507254660129547,
      "logps/chosen": -1.2770401239395142,
      "logps/rejected": -3.526949882507324,
      "loss": 2.6779,
      "nll_loss": 2.6349117755889893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12770400941371918,
      "rewards/margins": 0.22499097883701324,
      "rewards/rejected": -0.3526949882507324,
      "step": 605
    },
    {
      "epoch": 0.3769828926905132,
      "grad_norm": 0.534155547618866,
      "learning_rate": 3.485e-05,
      "log_odds_chosen": 4.362737655639648,
      "log_odds_ratio": -0.16578631103038788,
      "logits/chosen": -0.0013291984796524048,
      "logits/rejected": 0.4363200068473816,
      "logps/chosen": -1.247031569480896,
      "logps/rejected": -5.332918167114258,
      "loss": 2.764,
      "nll_loss": 2.7474396228790283,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12470315396785736,
      "rewards/margins": 0.4085886478424072,
      "rewards/rejected": -0.5332918167114258,
      "step": 606
    },
    {
      "epoch": 0.3776049766718507,
      "grad_norm": 0.5048543810844421,
      "learning_rate": 3.4825e-05,
      "log_odds_chosen": 3.234736919403076,
      "log_odds_ratio": -0.36818602681159973,
      "logits/chosen": 0.2557424008846283,
      "logits/rejected": 0.39144402742385864,
      "logps/chosen": -1.2747143507003784,
      "logps/rejected": -4.362112998962402,
      "loss": 3.2001,
      "nll_loss": 3.163267135620117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1274714320898056,
      "rewards/margins": 0.3087398409843445,
      "rewards/rejected": -0.4362112581729889,
      "step": 607
    },
    {
      "epoch": 0.3782270606531882,
      "grad_norm": 0.3789282441139221,
      "learning_rate": 3.48e-05,
      "log_odds_chosen": 5.484085559844971,
      "log_odds_ratio": -0.24823862314224243,
      "logits/chosen": 0.07289771735668182,
      "logits/rejected": 0.3960074782371521,
      "logps/chosen": -1.0095453262329102,
      "logps/rejected": -6.180008411407471,
      "loss": 3.062,
      "nll_loss": 3.037135124206543,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10095453262329102,
      "rewards/margins": 0.517046332359314,
      "rewards/rejected": -0.6180008053779602,
      "step": 608
    },
    {
      "epoch": 0.3788491446345257,
      "grad_norm": 0.45806336402893066,
      "learning_rate": 3.4775000000000005e-05,
      "log_odds_chosen": 5.055695533752441,
      "log_odds_ratio": -0.07001174986362457,
      "logits/chosen": 0.29797932505607605,
      "logits/rejected": 0.7238092422485352,
      "logps/chosen": -1.1003090143203735,
      "logps/rejected": -5.755534648895264,
      "loss": 3.4197,
      "nll_loss": 3.4127392768859863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11003090441226959,
      "rewards/margins": 0.4655225872993469,
      "rewards/rejected": -0.5755534768104553,
      "step": 609
    },
    {
      "epoch": 0.37947122861586313,
      "grad_norm": 0.4006802439689636,
      "learning_rate": 3.475e-05,
      "log_odds_chosen": 8.513465881347656,
      "log_odds_ratio": -0.1920199692249298,
      "logits/chosen": 0.18292070925235748,
      "logits/rejected": 0.8109362721443176,
      "logps/chosen": -1.025404691696167,
      "logps/rejected": -8.980233192443848,
      "loss": 3.2596,
      "nll_loss": 3.2403993606567383,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10254047811031342,
      "rewards/margins": 0.795482873916626,
      "rewards/rejected": -0.8980233073234558,
      "step": 610
    },
    {
      "epoch": 0.38009331259720064,
      "grad_norm": 0.49815765023231506,
      "learning_rate": 3.4725000000000004e-05,
      "log_odds_chosen": 5.950308799743652,
      "log_odds_ratio": -0.26779890060424805,
      "logits/chosen": 0.12611763179302216,
      "logits/rejected": 0.2929439842700958,
      "logps/chosen": -1.2592666149139404,
      "logps/rejected": -6.927753925323486,
      "loss": 3.173,
      "nll_loss": 3.146212100982666,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.125926673412323,
      "rewards/margins": 0.5668487548828125,
      "rewards/rejected": -0.6927754878997803,
      "step": 611
    },
    {
      "epoch": 0.3807153965785381,
      "grad_norm": 0.44551563262939453,
      "learning_rate": 3.4699999999999996e-05,
      "log_odds_chosen": 8.533232688903809,
      "log_odds_ratio": -0.24877887964248657,
      "logits/chosen": 0.3306812644004822,
      "logits/rejected": 0.6930794715881348,
      "logps/chosen": -0.9571521878242493,
      "logps/rejected": -9.140741348266602,
      "loss": 3.4917,
      "nll_loss": 3.4668092727661133,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09571521729230881,
      "rewards/margins": 0.8183589577674866,
      "rewards/rejected": -0.914074182510376,
      "step": 612
    },
    {
      "epoch": 0.3813374805598756,
      "grad_norm": 1.205088496208191,
      "learning_rate": 3.4675e-05,
      "log_odds_chosen": 11.991146087646484,
      "log_odds_ratio": -0.0009046989143826067,
      "logits/chosen": 0.16803990304470062,
      "logits/rejected": 1.0780525207519531,
      "logps/chosen": -1.2963323593139648,
      "logps/rejected": -12.865903854370117,
      "loss": 2.9082,
      "nll_loss": 2.9080753326416016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12963323295116425,
      "rewards/margins": 1.1569572687149048,
      "rewards/rejected": -1.2865904569625854,
      "step": 613
    },
    {
      "epoch": 0.38195956454121305,
      "grad_norm": 0.6109058260917664,
      "learning_rate": 3.465e-05,
      "log_odds_chosen": 10.368602752685547,
      "log_odds_ratio": -0.05110776424407959,
      "logits/chosen": 0.15692661702632904,
      "logits/rejected": 0.9058154821395874,
      "logps/chosen": -1.3313194513320923,
      "logps/rejected": -11.122682571411133,
      "loss": 3.1653,
      "nll_loss": 3.160191297531128,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1331319510936737,
      "rewards/margins": 0.9791362881660461,
      "rewards/rejected": -1.112268328666687,
      "step": 614
    },
    {
      "epoch": 0.38258164852255055,
      "grad_norm": 0.3697699308395386,
      "learning_rate": 3.4625e-05,
      "log_odds_chosen": 6.597249984741211,
      "log_odds_ratio": -0.18403266370296478,
      "logits/chosen": 0.2341020703315735,
      "logits/rejected": 0.8733110427856445,
      "logps/chosen": -1.0065994262695312,
      "logps/rejected": -7.087155342102051,
      "loss": 3.0979,
      "nll_loss": 3.0794601440429688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10065993666648865,
      "rewards/margins": 0.6080557107925415,
      "rewards/rejected": -0.708715558052063,
      "step": 615
    },
    {
      "epoch": 0.383203732503888,
      "grad_norm": 0.43355652689933777,
      "learning_rate": 3.46e-05,
      "log_odds_chosen": 5.680283069610596,
      "log_odds_ratio": -0.3767112195491791,
      "logits/chosen": 0.03800162300467491,
      "logits/rejected": 0.27057909965515137,
      "logps/chosen": -1.3413360118865967,
      "logps/rejected": -6.765846252441406,
      "loss": 2.745,
      "nll_loss": 2.707333564758301,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13413360714912415,
      "rewards/margins": 0.542451024055481,
      "rewards/rejected": -0.6765846014022827,
      "step": 616
    },
    {
      "epoch": 0.3838258164852255,
      "grad_norm": 0.3831072151660919,
      "learning_rate": 3.4575e-05,
      "log_odds_chosen": 5.973001480102539,
      "log_odds_ratio": -0.13384968042373657,
      "logits/chosen": 0.09891879558563232,
      "logits/rejected": 0.39111006259918213,
      "logps/chosen": -1.1477863788604736,
      "logps/rejected": -6.674656867980957,
      "loss": 3.2631,
      "nll_loss": 3.2497549057006836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11477864533662796,
      "rewards/margins": 0.5526870489120483,
      "rewards/rejected": -0.6674657464027405,
      "step": 617
    },
    {
      "epoch": 0.38444790046656296,
      "grad_norm": 0.4581452012062073,
      "learning_rate": 3.455e-05,
      "log_odds_chosen": 5.603791236877441,
      "log_odds_ratio": -0.1752859354019165,
      "logits/chosen": 0.04793360084295273,
      "logits/rejected": 0.3412993550300598,
      "logps/chosen": -1.081144094467163,
      "logps/rejected": -6.33277702331543,
      "loss": 3.1135,
      "nll_loss": 3.0959632396698,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10811440646648407,
      "rewards/margins": 0.5251632928848267,
      "rewards/rejected": -0.6332777142524719,
      "step": 618
    },
    {
      "epoch": 0.38506998444790047,
      "grad_norm": 0.421712189912796,
      "learning_rate": 3.4525e-05,
      "log_odds_chosen": 5.768167972564697,
      "log_odds_ratio": -0.22745099663734436,
      "logits/chosen": 0.03324780613183975,
      "logits/rejected": 0.42689570784568787,
      "logps/chosen": -1.1148405075073242,
      "logps/rejected": -6.4015655517578125,
      "loss": 2.9409,
      "nll_loss": 2.9181556701660156,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11148406565189362,
      "rewards/margins": 0.5286725163459778,
      "rewards/rejected": -0.6401565670967102,
      "step": 619
    },
    {
      "epoch": 0.3856920684292379,
      "grad_norm": 0.4084291458129883,
      "learning_rate": 3.45e-05,
      "log_odds_chosen": 6.440693378448486,
      "log_odds_ratio": -0.1236582025885582,
      "logits/chosen": 0.10940997302532196,
      "logits/rejected": 0.420942485332489,
      "logps/chosen": -1.1581194400787354,
      "logps/rejected": -6.912134170532227,
      "loss": 3.3024,
      "nll_loss": 3.2900736331939697,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11581194400787354,
      "rewards/margins": 0.5754014849662781,
      "rewards/rejected": -0.6912134289741516,
      "step": 620
    },
    {
      "epoch": 0.38631415241057543,
      "grad_norm": 0.5068473219871521,
      "learning_rate": 3.4475000000000005e-05,
      "log_odds_chosen": 6.7337493896484375,
      "log_odds_ratio": -0.1989458054304123,
      "logits/chosen": 0.19425322115421295,
      "logits/rejected": 0.316842645406723,
      "logps/chosen": -1.122262716293335,
      "logps/rejected": -7.526772499084473,
      "loss": 3.0876,
      "nll_loss": 3.0677075386047363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11222627013921738,
      "rewards/margins": 0.6404510140419006,
      "rewards/rejected": -0.7526772618293762,
      "step": 621
    },
    {
      "epoch": 0.38693623639191294,
      "grad_norm": 0.38466599583625793,
      "learning_rate": 3.445e-05,
      "log_odds_chosen": 5.886903762817383,
      "log_odds_ratio": -0.2482847273349762,
      "logits/chosen": 0.2649783492088318,
      "logits/rejected": 0.7941878437995911,
      "logps/chosen": -1.2445883750915527,
      "logps/rejected": -6.8715128898620605,
      "loss": 3.1309,
      "nll_loss": 3.1060690879821777,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12445883452892303,
      "rewards/margins": 0.5626925230026245,
      "rewards/rejected": -0.687151312828064,
      "step": 622
    },
    {
      "epoch": 0.3875583203732504,
      "grad_norm": 0.40377840399742126,
      "learning_rate": 3.4425e-05,
      "log_odds_chosen": 8.402645111083984,
      "log_odds_ratio": -0.2527872323989868,
      "logits/chosen": 0.14706459641456604,
      "logits/rejected": 0.5948149561882019,
      "logps/chosen": -1.120314598083496,
      "logps/rejected": -9.159903526306152,
      "loss": 3.1371,
      "nll_loss": 3.111776828765869,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11203145235776901,
      "rewards/margins": 0.8039589524269104,
      "rewards/rejected": -0.9159903526306152,
      "step": 623
    },
    {
      "epoch": 0.3881804043545879,
      "grad_norm": 0.41054055094718933,
      "learning_rate": 3.4399999999999996e-05,
      "log_odds_chosen": 6.867403984069824,
      "log_odds_ratio": -0.08644460141658783,
      "logits/chosen": 0.2943398952484131,
      "logits/rejected": 0.6636469960212708,
      "logps/chosen": -0.9363681077957153,
      "logps/rejected": -7.202298641204834,
      "loss": 3.3252,
      "nll_loss": 3.3165183067321777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09363681823015213,
      "rewards/margins": 0.6265930533409119,
      "rewards/rejected": -0.7202298641204834,
      "step": 624
    },
    {
      "epoch": 0.38880248833592534,
      "grad_norm": 0.4753408432006836,
      "learning_rate": 3.4375e-05,
      "log_odds_chosen": 6.773974418640137,
      "log_odds_ratio": -0.2092844694852829,
      "logits/chosen": 0.29289454221725464,
      "logits/rejected": 0.7853541374206543,
      "logps/chosen": -1.3344194889068604,
      "logps/rejected": -7.880761623382568,
      "loss": 3.0419,
      "nll_loss": 3.0209245681762695,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1334419548511505,
      "rewards/margins": 0.6546341776847839,
      "rewards/rejected": -0.7880761623382568,
      "step": 625
    },
    {
      "epoch": 0.38942457231726285,
      "grad_norm": 0.38249650597572327,
      "learning_rate": 3.435e-05,
      "log_odds_chosen": 9.644584655761719,
      "log_odds_ratio": -0.07656724750995636,
      "logits/chosen": 0.32878345251083374,
      "logits/rejected": 0.9979957342147827,
      "logps/chosen": -0.9833849668502808,
      "logps/rejected": -10.156517028808594,
      "loss": 3.1942,
      "nll_loss": 3.186497688293457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09833849966526031,
      "rewards/margins": 0.917313277721405,
      "rewards/rejected": -1.015651822090149,
      "step": 626
    },
    {
      "epoch": 0.3900466562986003,
      "grad_norm": 0.5387607216835022,
      "learning_rate": 3.4325e-05,
      "log_odds_chosen": 7.158836364746094,
      "log_odds_ratio": -0.13461348414421082,
      "logits/chosen": 0.11595956236124039,
      "logits/rejected": 0.8866112232208252,
      "logps/chosen": -1.2095855474472046,
      "logps/rejected": -8.037429809570312,
      "loss": 2.1498,
      "nll_loss": 2.1363189220428467,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12095855176448822,
      "rewards/margins": 0.6827844977378845,
      "rewards/rejected": -0.8037430047988892,
      "step": 627
    },
    {
      "epoch": 0.3906687402799378,
      "grad_norm": 18.14006233215332,
      "learning_rate": 3.430000000000001e-05,
      "log_odds_chosen": 8.21702766418457,
      "log_odds_ratio": -0.22415418922901154,
      "logits/chosen": 0.5142999887466431,
      "logits/rejected": 0.9347355961799622,
      "logps/chosen": -3.1413345336914062,
      "logps/rejected": -11.10336685180664,
      "loss": 3.7885,
      "nll_loss": 3.7660984992980957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31413349509239197,
      "rewards/margins": 0.7962032556533813,
      "rewards/rejected": -1.1103367805480957,
      "step": 628
    },
    {
      "epoch": 0.39129082426127526,
      "grad_norm": 0.5293332934379578,
      "learning_rate": 3.4275e-05,
      "log_odds_chosen": 9.935027122497559,
      "log_odds_ratio": -0.0972500815987587,
      "logits/chosen": 0.372843861579895,
      "logits/rejected": 0.5359551906585693,
      "logps/chosen": -1.0797069072723389,
      "logps/rejected": -10.594209671020508,
      "loss": 3.561,
      "nll_loss": 3.551283121109009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10797069221735,
      "rewards/margins": 0.9514502882957458,
      "rewards/rejected": -1.0594210624694824,
      "step": 629
    },
    {
      "epoch": 0.39191290824261277,
      "grad_norm": 0.5737473368644714,
      "learning_rate": 3.4250000000000006e-05,
      "log_odds_chosen": 8.50829029083252,
      "log_odds_ratio": -0.14946213364601135,
      "logits/chosen": 0.21567851305007935,
      "logits/rejected": 0.841961681842804,
      "logps/chosen": -1.2254951000213623,
      "logps/rejected": -9.41191291809082,
      "loss": 2.6145,
      "nll_loss": 2.599510669708252,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12254951894283295,
      "rewards/margins": 0.8186418414115906,
      "rewards/rejected": -0.9411913156509399,
      "step": 630
    },
    {
      "epoch": 0.3925349922239502,
      "grad_norm": 0.5078374147415161,
      "learning_rate": 3.4225e-05,
      "log_odds_chosen": 8.175843238830566,
      "log_odds_ratio": -0.14807230234146118,
      "logits/chosen": 0.4468088746070862,
      "logits/rejected": 0.967327356338501,
      "logps/chosen": -0.9474899768829346,
      "logps/rejected": -8.668539047241211,
      "loss": 2.8416,
      "nll_loss": 2.8268425464630127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09474898874759674,
      "rewards/margins": 0.7721049785614014,
      "rewards/rejected": -0.8668538928031921,
      "step": 631
    },
    {
      "epoch": 0.3931570762052877,
      "grad_norm": 0.43191152811050415,
      "learning_rate": 3.4200000000000005e-05,
      "log_odds_chosen": 4.073611736297607,
      "log_odds_ratio": -0.16531451046466827,
      "logits/chosen": 0.35050833225250244,
      "logits/rejected": 0.45885440707206726,
      "logps/chosen": -1.4971977472305298,
      "logps/rejected": -5.289085388183594,
      "loss": 3.174,
      "nll_loss": 3.1574785709381104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14971977472305298,
      "rewards/margins": 0.3791887164115906,
      "rewards/rejected": -0.5289084911346436,
      "step": 632
    },
    {
      "epoch": 0.3937791601866252,
      "grad_norm": 0.48704293370246887,
      "learning_rate": 3.4175000000000004e-05,
      "log_odds_chosen": 4.4312849044799805,
      "log_odds_ratio": -0.15987420082092285,
      "logits/chosen": 0.3942926228046417,
      "logits/rejected": 0.5708335638046265,
      "logps/chosen": -1.2215713262557983,
      "logps/rejected": -5.378739833831787,
      "loss": 3.6228,
      "nll_loss": 3.6067724227905273,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12215713411569595,
      "rewards/margins": 0.41571688652038574,
      "rewards/rejected": -0.5378739833831787,
      "step": 633
    },
    {
      "epoch": 0.3944012441679627,
      "grad_norm": 0.5552240014076233,
      "learning_rate": 3.415e-05,
      "log_odds_chosen": 7.168707847595215,
      "log_odds_ratio": -0.04835962504148483,
      "logits/chosen": 0.1953887641429901,
      "logits/rejected": 0.6829716563224792,
      "logps/chosen": -0.9781539440155029,
      "logps/rejected": -7.654153347015381,
      "loss": 2.4954,
      "nll_loss": 2.4905707836151123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09781539440155029,
      "rewards/margins": 0.6675999164581299,
      "rewards/rejected": -0.765415370464325,
      "step": 634
    },
    {
      "epoch": 0.39502332814930013,
      "grad_norm": 0.43904614448547363,
      "learning_rate": 3.4125e-05,
      "log_odds_chosen": 7.5145721435546875,
      "log_odds_ratio": -0.01926579885184765,
      "logits/chosen": 0.2834244668483734,
      "logits/rejected": 0.6822664737701416,
      "logps/chosen": -1.2787011861801147,
      "logps/rejected": -8.410541534423828,
      "loss": 3.4046,
      "nll_loss": 3.4026920795440674,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1278701275587082,
      "rewards/margins": 0.7131839990615845,
      "rewards/rejected": -0.8410541415214539,
      "step": 635
    },
    {
      "epoch": 0.39564541213063764,
      "grad_norm": 0.4341355860233307,
      "learning_rate": 3.41e-05,
      "log_odds_chosen": 7.731707572937012,
      "log_odds_ratio": -0.02020195871591568,
      "logits/chosen": 0.2916351556777954,
      "logits/rejected": 0.7818939685821533,
      "logps/chosen": -0.9819329977035522,
      "logps/rejected": -8.171854972839355,
      "loss": 2.8691,
      "nll_loss": 2.8670694828033447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09819329530000687,
      "rewards/margins": 0.7189922332763672,
      "rewards/rejected": -0.8171855211257935,
      "step": 636
    },
    {
      "epoch": 0.3962674961119751,
      "grad_norm": 0.4297373294830322,
      "learning_rate": 3.4075e-05,
      "log_odds_chosen": 5.247067451477051,
      "log_odds_ratio": -0.35224947333335876,
      "logits/chosen": 0.16714119911193848,
      "logits/rejected": 0.5036913752555847,
      "logps/chosen": -1.042916178703308,
      "logps/rejected": -6.024389743804932,
      "loss": 2.7291,
      "nll_loss": 2.6938908100128174,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10429162532091141,
      "rewards/margins": 0.4981473386287689,
      "rewards/rejected": -0.6024389266967773,
      "step": 637
    },
    {
      "epoch": 0.3968895800933126,
      "grad_norm": 0.5243464112281799,
      "learning_rate": 3.405e-05,
      "log_odds_chosen": 8.308638572692871,
      "log_odds_ratio": -0.2812676429748535,
      "logits/chosen": 0.20677152276039124,
      "logits/rejected": 0.3971843421459198,
      "logps/chosen": -1.1305897235870361,
      "logps/rejected": -9.007345199584961,
      "loss": 3.0744,
      "nll_loss": 3.0463199615478516,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11305898427963257,
      "rewards/margins": 0.7876755595207214,
      "rewards/rejected": -0.9007344841957092,
      "step": 638
    },
    {
      "epoch": 0.39751166407465005,
      "grad_norm": 0.541507363319397,
      "learning_rate": 3.4025e-05,
      "log_odds_chosen": 5.680115699768066,
      "log_odds_ratio": -0.08863461017608643,
      "logits/chosen": 0.23920726776123047,
      "logits/rejected": 0.6990938782691956,
      "logps/chosen": -1.3689988851547241,
      "logps/rejected": -6.771450996398926,
      "loss": 2.8984,
      "nll_loss": 2.889554023742676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1368998885154724,
      "rewards/margins": 0.5402451753616333,
      "rewards/rejected": -0.6771451234817505,
      "step": 639
    },
    {
      "epoch": 0.39813374805598756,
      "grad_norm": 0.44972142577171326,
      "learning_rate": 3.4000000000000007e-05,
      "log_odds_chosen": 5.760073661804199,
      "log_odds_ratio": -0.20923081040382385,
      "logits/chosen": 0.28065040707588196,
      "logits/rejected": 0.6092424392700195,
      "logps/chosen": -1.1194320917129517,
      "logps/rejected": -6.457537651062012,
      "loss": 2.9208,
      "nll_loss": 2.899829864501953,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11194320023059845,
      "rewards/margins": 0.533810555934906,
      "rewards/rejected": -0.6457537412643433,
      "step": 640
    },
    {
      "epoch": 0.39875583203732506,
      "grad_norm": 0.45881423354148865,
      "learning_rate": 3.3975e-05,
      "log_odds_chosen": 8.45309066772461,
      "log_odds_ratio": -0.002736177993938327,
      "logits/chosen": 0.1473774015903473,
      "logits/rejected": 0.9392131567001343,
      "logps/chosen": -1.1920280456542969,
      "logps/rejected": -9.228044509887695,
      "loss": 2.4849,
      "nll_loss": 2.4846346378326416,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11920280009508133,
      "rewards/margins": 0.8036016225814819,
      "rewards/rejected": -0.9228044152259827,
      "step": 641
    },
    {
      "epoch": 0.3993779160186625,
      "grad_norm": 0.4157197177410126,
      "learning_rate": 3.3950000000000005e-05,
      "log_odds_chosen": 7.536615371704102,
      "log_odds_ratio": -0.09330558776855469,
      "logits/chosen": 0.31922441720962524,
      "logits/rejected": 0.7367990016937256,
      "logps/chosen": -1.2460134029388428,
      "logps/rejected": -8.397119522094727,
      "loss": 3.217,
      "nll_loss": 3.2076451778411865,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12460136413574219,
      "rewards/margins": 0.7151105403900146,
      "rewards/rejected": -0.8397119641304016,
      "step": 642
    },
    {
      "epoch": 0.4,
      "grad_norm": 0.3730640113353729,
      "learning_rate": 3.3925e-05,
      "log_odds_chosen": 7.549066543579102,
      "log_odds_ratio": -0.028249753639101982,
      "logits/chosen": 0.43140530586242676,
      "logits/rejected": 0.803489089012146,
      "logps/chosen": -0.825218141078949,
      "logps/rejected": -7.683599472045898,
      "loss": 3.0414,
      "nll_loss": 3.0385513305664062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08252181112766266,
      "rewards/margins": 0.6858382225036621,
      "rewards/rejected": -0.7683599591255188,
      "step": 643
    },
    {
      "epoch": 0.4006220839813375,
      "grad_norm": 0.5043125748634338,
      "learning_rate": 3.3900000000000004e-05,
      "log_odds_chosen": 12.2427978515625,
      "log_odds_ratio": -0.06448502838611603,
      "logits/chosen": 0.22592878341674805,
      "logits/rejected": 0.6281089782714844,
      "logps/chosen": -0.8235211968421936,
      "logps/rejected": -12.127344131469727,
      "loss": 3.0036,
      "nll_loss": 2.997192859649658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08235213160514832,
      "rewards/margins": 1.1303822994232178,
      "rewards/rejected": -1.2127344608306885,
      "step": 644
    },
    {
      "epoch": 0.401244167962675,
      "grad_norm": 0.5275508165359497,
      "learning_rate": 3.3875000000000003e-05,
      "log_odds_chosen": 8.873658180236816,
      "log_odds_ratio": -0.16682618856430054,
      "logits/chosen": 0.2479245662689209,
      "logits/rejected": 0.6735783219337463,
      "logps/chosen": -1.1058787107467651,
      "logps/rejected": -9.506242752075195,
      "loss": 2.9091,
      "nll_loss": 2.8924357891082764,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11058788001537323,
      "rewards/margins": 0.8400364518165588,
      "rewards/rejected": -0.9506243467330933,
      "step": 645
    },
    {
      "epoch": 0.40186625194401243,
      "grad_norm": 0.47356879711151123,
      "learning_rate": 3.385e-05,
      "log_odds_chosen": 6.3066816329956055,
      "log_odds_ratio": -0.24541525542736053,
      "logits/chosen": 0.4849867820739746,
      "logits/rejected": 0.8165839314460754,
      "logps/chosen": -1.1368855237960815,
      "logps/rejected": -6.978367805480957,
      "loss": 3.2928,
      "nll_loss": 3.268289089202881,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11368854343891144,
      "rewards/margins": 0.5841482877731323,
      "rewards/rejected": -0.6978368163108826,
      "step": 646
    },
    {
      "epoch": 0.40248833592534994,
      "grad_norm": 0.5155513882637024,
      "learning_rate": 3.3825e-05,
      "log_odds_chosen": 5.060737609863281,
      "log_odds_ratio": -0.2929985821247101,
      "logits/chosen": 0.2869529724121094,
      "logits/rejected": 0.567604124546051,
      "logps/chosen": -1.1311206817626953,
      "logps/rejected": -5.907896995544434,
      "loss": 3.0702,
      "nll_loss": 3.0408787727355957,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11311206221580505,
      "rewards/margins": 0.47767767310142517,
      "rewards/rejected": -0.5907896757125854,
      "step": 647
    },
    {
      "epoch": 0.4031104199066874,
      "grad_norm": 0.4957614243030548,
      "learning_rate": 3.38e-05,
      "log_odds_chosen": 7.915012836456299,
      "log_odds_ratio": -0.07648847997188568,
      "logits/chosen": 0.3334338963031769,
      "logits/rejected": 0.8731058239936829,
      "logps/chosen": -0.9914820790290833,
      "logps/rejected": -8.388053894042969,
      "loss": 2.8132,
      "nll_loss": 2.805509090423584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0991482138633728,
      "rewards/margins": 0.7396571636199951,
      "rewards/rejected": -0.8388054370880127,
      "step": 648
    },
    {
      "epoch": 0.4037325038880249,
      "grad_norm": 0.48051124811172485,
      "learning_rate": 3.3775e-05,
      "log_odds_chosen": 11.63776683807373,
      "log_odds_ratio": -0.05890395864844322,
      "logits/chosen": 0.15852481126785278,
      "logits/rejected": 0.6033845543861389,
      "logps/chosen": -0.6961344480514526,
      "logps/rejected": -11.388937950134277,
      "loss": 2.9851,
      "nll_loss": 2.9792487621307373,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06961344182491302,
      "rewards/margins": 1.0692802667617798,
      "rewards/rejected": -1.138893723487854,
      "step": 649
    },
    {
      "epoch": 0.40435458786936235,
      "grad_norm": 0.5910434722900391,
      "learning_rate": 3.375000000000001e-05,
      "log_odds_chosen": 13.171907424926758,
      "log_odds_ratio": -0.08223630487918854,
      "logits/chosen": 0.15894870460033417,
      "logits/rejected": 0.8937514424324036,
      "logps/chosen": -1.2927449941635132,
      "logps/rejected": -14.164291381835938,
      "loss": 2.5011,
      "nll_loss": 2.4928855895996094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12927450239658356,
      "rewards/margins": 1.2871546745300293,
      "rewards/rejected": -1.4164291620254517,
      "step": 650
    },
    {
      "epoch": 0.40497667185069985,
      "grad_norm": 2.0554184913635254,
      "learning_rate": 3.3725e-05,
      "log_odds_chosen": 13.773082733154297,
      "log_odds_ratio": -0.07272257655858994,
      "logits/chosen": 0.4365695118904114,
      "logits/rejected": 1.2860937118530273,
      "logps/chosen": -1.0678213834762573,
      "logps/rejected": -14.37375545501709,
      "loss": 2.8618,
      "nll_loss": 2.8545725345611572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10678213834762573,
      "rewards/margins": 1.330593466758728,
      "rewards/rejected": -1.437375545501709,
      "step": 651
    },
    {
      "epoch": 0.4055987558320373,
      "grad_norm": 2.0059454441070557,
      "learning_rate": 3.3700000000000006e-05,
      "log_odds_chosen": 13.936979293823242,
      "log_odds_ratio": -0.09149403125047684,
      "logits/chosen": 0.09907475113868713,
      "logits/rejected": 0.763411283493042,
      "logps/chosen": -1.1428678035736084,
      "logps/rejected": -14.637571334838867,
      "loss": 2.3492,
      "nll_loss": 2.3400137424468994,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11428678035736084,
      "rewards/margins": 1.3494703769683838,
      "rewards/rejected": -1.4637572765350342,
      "step": 652
    },
    {
      "epoch": 0.4062208398133748,
      "grad_norm": 0.46556660532951355,
      "learning_rate": 3.3675e-05,
      "log_odds_chosen": 7.606475830078125,
      "log_odds_ratio": -0.18322187662124634,
      "logits/chosen": 0.3579789996147156,
      "logits/rejected": 0.8734443783760071,
      "logps/chosen": -1.0546009540557861,
      "logps/rejected": -8.23729419708252,
      "loss": 3.08,
      "nll_loss": 3.0616695880889893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10546009242534637,
      "rewards/margins": 0.7182692885398865,
      "rewards/rejected": -0.8237293362617493,
      "step": 653
    },
    {
      "epoch": 0.40684292379471226,
      "grad_norm": 0.4319630563259125,
      "learning_rate": 3.3650000000000005e-05,
      "log_odds_chosen": 9.043044090270996,
      "log_odds_ratio": -0.17355716228485107,
      "logits/chosen": 0.38297635316848755,
      "logits/rejected": 0.8766739964485168,
      "logps/chosen": -1.290746808052063,
      "logps/rejected": -10.054768562316895,
      "loss": 3.6632,
      "nll_loss": 3.6458590030670166,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12907469272613525,
      "rewards/margins": 0.8764022588729858,
      "rewards/rejected": -1.005476951599121,
      "step": 654
    },
    {
      "epoch": 0.40746500777604977,
      "grad_norm": 0.586379885673523,
      "learning_rate": 3.3625000000000004e-05,
      "log_odds_chosen": 10.436687469482422,
      "log_odds_ratio": -0.08339973539113998,
      "logits/chosen": 0.01694488525390625,
      "logits/rejected": 0.7056764364242554,
      "logps/chosen": -1.175871729850769,
      "logps/rejected": -11.233123779296875,
      "loss": 2.2024,
      "nll_loss": 2.1940736770629883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11758717149496078,
      "rewards/margins": 1.0057252645492554,
      "rewards/rejected": -1.1233124732971191,
      "step": 655
    },
    {
      "epoch": 0.4080870917573872,
      "grad_norm": 0.5463007688522339,
      "learning_rate": 3.3600000000000004e-05,
      "log_odds_chosen": 14.558086395263672,
      "log_odds_ratio": -0.020145747810602188,
      "logits/chosen": 0.1619691252708435,
      "logits/rejected": 0.8365485668182373,
      "logps/chosen": -1.254634141921997,
      "logps/rejected": -15.396265029907227,
      "loss": 2.7958,
      "nll_loss": 2.793788433074951,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12546342611312866,
      "rewards/margins": 1.4141631126403809,
      "rewards/rejected": -1.5396264791488647,
      "step": 656
    },
    {
      "epoch": 0.40870917573872473,
      "grad_norm": 1.0127180814743042,
      "learning_rate": 3.3575e-05,
      "log_odds_chosen": 10.627214431762695,
      "log_odds_ratio": -0.025262095034122467,
      "logits/chosen": 0.15809279680252075,
      "logits/rejected": 1.015772819519043,
      "logps/chosen": -1.144571304321289,
      "logps/rejected": -11.349461555480957,
      "loss": 2.5338,
      "nll_loss": 2.5312976837158203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11445712298154831,
      "rewards/margins": 1.020488977432251,
      "rewards/rejected": -1.1349461078643799,
      "step": 657
    },
    {
      "epoch": 0.40933125972006223,
      "grad_norm": 0.5098369121551514,
      "learning_rate": 3.355e-05,
      "log_odds_chosen": 7.605299949645996,
      "log_odds_ratio": -0.18348553776741028,
      "logits/chosen": 0.14395418763160706,
      "logits/rejected": 0.305155873298645,
      "logps/chosen": -1.009643793106079,
      "logps/rejected": -8.116901397705078,
      "loss": 2.8782,
      "nll_loss": 2.859889507293701,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10096438229084015,
      "rewards/margins": 0.7107258439064026,
      "rewards/rejected": -0.8116902112960815,
      "step": 658
    },
    {
      "epoch": 0.4099533437013997,
      "grad_norm": 0.49793869256973267,
      "learning_rate": 3.3525e-05,
      "log_odds_chosen": 5.807119369506836,
      "log_odds_ratio": -0.18563133478164673,
      "logits/chosen": 0.1726893037557602,
      "logits/rejected": 0.5850039720535278,
      "logps/chosen": -1.2937705516815186,
      "logps/rejected": -6.849543571472168,
      "loss": 2.8515,
      "nll_loss": 2.8329498767852783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12937705218791962,
      "rewards/margins": 0.555577278137207,
      "rewards/rejected": -0.6849542856216431,
      "step": 659
    },
    {
      "epoch": 0.4105754276827372,
      "grad_norm": 0.4080248773097992,
      "learning_rate": 3.35e-05,
      "log_odds_chosen": 7.798429489135742,
      "log_odds_ratio": -0.033232737332582474,
      "logits/chosen": 0.16965456306934357,
      "logits/rejected": 0.5689667463302612,
      "logps/chosen": -0.8590775728225708,
      "logps/rejected": -8.06137466430664,
      "loss": 3.1818,
      "nll_loss": 3.178438663482666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08590776473283768,
      "rewards/margins": 0.7202296853065491,
      "rewards/rejected": -0.8061374425888062,
      "step": 660
    },
    {
      "epoch": 0.41119751166407464,
      "grad_norm": 1.0456312894821167,
      "learning_rate": 3.3475e-05,
      "log_odds_chosen": 4.1362528800964355,
      "log_odds_ratio": -0.4419694244861603,
      "logits/chosen": 0.1379992961883545,
      "logits/rejected": 0.17854338884353638,
      "logps/chosen": -1.8760737180709839,
      "logps/rejected": -5.827506065368652,
      "loss": 3.5102,
      "nll_loss": 3.4659581184387207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18760737776756287,
      "rewards/margins": 0.39514321088790894,
      "rewards/rejected": -0.5827505588531494,
      "step": 661
    },
    {
      "epoch": 0.41181959564541215,
      "grad_norm": 0.45937567949295044,
      "learning_rate": 3.345000000000001e-05,
      "log_odds_chosen": 6.3919997215271,
      "log_odds_ratio": -0.18915338814258575,
      "logits/chosen": 0.06113801896572113,
      "logits/rejected": 0.46323448419570923,
      "logps/chosen": -0.9827958345413208,
      "logps/rejected": -6.9341278076171875,
      "loss": 2.761,
      "nll_loss": 2.742082118988037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09827959537506104,
      "rewards/margins": 0.5951332449913025,
      "rewards/rejected": -0.6934128403663635,
      "step": 662
    },
    {
      "epoch": 0.4124416796267496,
      "grad_norm": 0.7443048357963562,
      "learning_rate": 3.3425e-05,
      "log_odds_chosen": 3.1607913970947266,
      "log_odds_ratio": -0.3298979699611664,
      "logits/chosen": -0.002762638032436371,
      "logits/rejected": 0.15435074269771576,
      "logps/chosen": -1.0491981506347656,
      "logps/rejected": -3.879762649536133,
      "loss": 2.6448,
      "nll_loss": 2.6117641925811768,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10491982102394104,
      "rewards/margins": 0.28305643796920776,
      "rewards/rejected": -0.3879762291908264,
      "step": 663
    },
    {
      "epoch": 0.4130637636080871,
      "grad_norm": 0.5198607444763184,
      "learning_rate": 3.3400000000000005e-05,
      "log_odds_chosen": 5.001226425170898,
      "log_odds_ratio": -0.33517199754714966,
      "logits/chosen": 0.09879479557275772,
      "logits/rejected": 0.3197081387042999,
      "logps/chosen": -1.149256944656372,
      "logps/rejected": -5.96102237701416,
      "loss": 2.7095,
      "nll_loss": 2.6760072708129883,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11492569744586945,
      "rewards/margins": 0.48117655515670776,
      "rewards/rejected": -0.596102237701416,
      "step": 664
    },
    {
      "epoch": 0.41368584758942456,
      "grad_norm": 0.6322446465492249,
      "learning_rate": 3.3375e-05,
      "log_odds_chosen": 8.142166137695312,
      "log_odds_ratio": -0.5253583788871765,
      "logits/chosen": 0.10554549843072891,
      "logits/rejected": 0.32331740856170654,
      "logps/chosen": -1.1191916465759277,
      "logps/rejected": -8.784271240234375,
      "loss": 3.2475,
      "nll_loss": 3.194953441619873,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11191916465759277,
      "rewards/margins": 0.7665079832077026,
      "rewards/rejected": -0.8784270882606506,
      "step": 665
    },
    {
      "epoch": 0.41430793157076207,
      "grad_norm": 0.3725062906742096,
      "learning_rate": 3.3350000000000004e-05,
      "log_odds_chosen": 2.328892707824707,
      "log_odds_ratio": -0.46152687072753906,
      "logits/chosen": 0.1010928526520729,
      "logits/rejected": 0.24071413278579712,
      "logps/chosen": -1.1850922107696533,
      "logps/rejected": -3.2830145359039307,
      "loss": 3.2168,
      "nll_loss": 3.170639753341675,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1185092180967331,
      "rewards/margins": 0.20979222655296326,
      "rewards/rejected": -0.32830148935317993,
      "step": 666
    },
    {
      "epoch": 0.4149300155520995,
      "grad_norm": 0.6286746263504028,
      "learning_rate": 3.3325000000000004e-05,
      "log_odds_chosen": 4.8994221687316895,
      "log_odds_ratio": -0.09839385747909546,
      "logits/chosen": 0.04399656876921654,
      "logits/rejected": 0.19438612461090088,
      "logps/chosen": -1.0532360076904297,
      "logps/rejected": -5.433768272399902,
      "loss": 2.3705,
      "nll_loss": 2.360666275024414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10532359033823013,
      "rewards/margins": 0.43805328011512756,
      "rewards/rejected": -0.5433768630027771,
      "step": 667
    },
    {
      "epoch": 0.415552099533437,
      "grad_norm": 0.509515106678009,
      "learning_rate": 3.33e-05,
      "log_odds_chosen": 1.514922857284546,
      "log_odds_ratio": -0.4025351405143738,
      "logits/chosen": 0.14341723918914795,
      "logits/rejected": 0.2947719693183899,
      "logps/chosen": -1.520890474319458,
      "logps/rejected": -2.891176223754883,
      "loss": 3.3778,
      "nll_loss": 3.3375864028930664,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15208904445171356,
      "rewards/margins": 0.13702860474586487,
      "rewards/rejected": -0.28911763429641724,
      "step": 668
    },
    {
      "epoch": 0.4161741835147745,
      "grad_norm": 0.42655518651008606,
      "learning_rate": 3.3275e-05,
      "log_odds_chosen": 5.49832820892334,
      "log_odds_ratio": -0.04279167205095291,
      "logits/chosen": 0.17684528231620789,
      "logits/rejected": 0.4185295104980469,
      "logps/chosen": -1.1025071144104004,
      "logps/rejected": -6.174997329711914,
      "loss": 3.0329,
      "nll_loss": 3.0286574363708496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11025071889162064,
      "rewards/margins": 0.5072489976882935,
      "rewards/rejected": -0.6174997091293335,
      "step": 669
    },
    {
      "epoch": 0.416796267496112,
      "grad_norm": 0.45711860060691833,
      "learning_rate": 3.325e-05,
      "log_odds_chosen": 5.088669300079346,
      "log_odds_ratio": -0.17721472680568695,
      "logits/chosen": 0.21003636717796326,
      "logits/rejected": 0.4002099633216858,
      "logps/chosen": -0.9949367046356201,
      "logps/rejected": -5.5031514167785645,
      "loss": 3.3491,
      "nll_loss": 3.33135724067688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09949367493391037,
      "rewards/margins": 0.45082151889801025,
      "rewards/rejected": -0.5503151416778564,
      "step": 670
    },
    {
      "epoch": 0.41741835147744943,
      "grad_norm": 0.4432086646556854,
      "learning_rate": 3.3225e-05,
      "log_odds_chosen": 7.916843414306641,
      "log_odds_ratio": -0.1536819189786911,
      "logits/chosen": 0.14663207530975342,
      "logits/rejected": 0.4883155822753906,
      "logps/chosen": -0.9252321124076843,
      "logps/rejected": -8.24629020690918,
      "loss": 2.6296,
      "nll_loss": 2.6141862869262695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09252320975065231,
      "rewards/margins": 0.7321057915687561,
      "rewards/rejected": -0.8246290683746338,
      "step": 671
    },
    {
      "epoch": 0.41804043545878694,
      "grad_norm": 0.37037086486816406,
      "learning_rate": 3.32e-05,
      "log_odds_chosen": 6.104862213134766,
      "log_odds_ratio": -0.2211923599243164,
      "logits/chosen": 0.14146625995635986,
      "logits/rejected": 0.2648002505302429,
      "logps/chosen": -1.257808804512024,
      "logps/rejected": -6.962691307067871,
      "loss": 3.0737,
      "nll_loss": 3.0516157150268555,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1257808804512024,
      "rewards/margins": 0.5704882144927979,
      "rewards/rejected": -0.696269154548645,
      "step": 672
    },
    {
      "epoch": 0.4186625194401244,
      "grad_norm": 0.3770119845867157,
      "learning_rate": 3.3175e-05,
      "log_odds_chosen": 7.011809349060059,
      "log_odds_ratio": -0.11317376792430878,
      "logits/chosen": 0.364218533039093,
      "logits/rejected": 0.4188498854637146,
      "logps/chosen": -0.8818821907043457,
      "logps/rejected": -7.176725387573242,
      "loss": 3.8569,
      "nll_loss": 3.845541477203369,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08818823099136353,
      "rewards/margins": 0.6294843554496765,
      "rewards/rejected": -0.71767258644104,
      "step": 673
    },
    {
      "epoch": 0.4192846034214619,
      "grad_norm": 0.3674079179763794,
      "learning_rate": 3.3150000000000006e-05,
      "log_odds_chosen": 4.395227432250977,
      "log_odds_ratio": -0.3885990381240845,
      "logits/chosen": 0.1841599941253662,
      "logits/rejected": 0.3264680504798889,
      "logps/chosen": -1.2810193300247192,
      "logps/rejected": -5.530620098114014,
      "loss": 3.2217,
      "nll_loss": 3.182861804962158,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12810194492340088,
      "rewards/margins": 0.42496004700660706,
      "rewards/rejected": -0.5530620217323303,
      "step": 674
    },
    {
      "epoch": 0.4199066874027994,
      "grad_norm": 0.3473321199417114,
      "learning_rate": 3.3125e-05,
      "log_odds_chosen": 7.033344745635986,
      "log_odds_ratio": -0.09653866291046143,
      "logits/chosen": 0.022332118824124336,
      "logits/rejected": 0.26113685965538025,
      "logps/chosen": -0.9488670825958252,
      "logps/rejected": -7.321063041687012,
      "loss": 2.7077,
      "nll_loss": 2.698029041290283,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09488672018051147,
      "rewards/margins": 0.6372196078300476,
      "rewards/rejected": -0.7321063280105591,
      "step": 675
    },
    {
      "epoch": 0.42052877138413686,
      "grad_norm": 0.5475688576698303,
      "learning_rate": 3.3100000000000005e-05,
      "log_odds_chosen": 4.550343036651611,
      "log_odds_ratio": -0.4450215697288513,
      "logits/chosen": 0.22917914390563965,
      "logits/rejected": 0.44544917345046997,
      "logps/chosen": -1.1535780429840088,
      "logps/rejected": -5.414106369018555,
      "loss": 3.0895,
      "nll_loss": 3.044969320297241,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11535780131816864,
      "rewards/margins": 0.42605286836624146,
      "rewards/rejected": -0.5414106249809265,
      "step": 676
    },
    {
      "epoch": 0.42115085536547436,
      "grad_norm": 0.6116684079170227,
      "learning_rate": 3.3075e-05,
      "log_odds_chosen": 2.995664596557617,
      "log_odds_ratio": -0.6276533603668213,
      "logits/chosen": 0.14690755307674408,
      "logits/rejected": 0.3131594657897949,
      "logps/chosen": -1.151829719543457,
      "logps/rejected": -3.948768138885498,
      "loss": 2.504,
      "nll_loss": 2.4411871433258057,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11518297344446182,
      "rewards/margins": 0.2796938121318817,
      "rewards/rejected": -0.39487677812576294,
      "step": 677
    },
    {
      "epoch": 0.4217729393468118,
      "grad_norm": 0.4240874946117401,
      "learning_rate": 3.3050000000000004e-05,
      "log_odds_chosen": 5.284928321838379,
      "log_odds_ratio": -0.183100625872612,
      "logits/chosen": 0.1283799111843109,
      "logits/rejected": 0.2122117280960083,
      "logps/chosen": -0.9375210404396057,
      "logps/rejected": -5.698101997375488,
      "loss": 3.3374,
      "nll_loss": 3.319121837615967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09375210851430893,
      "rewards/margins": 0.47605806589126587,
      "rewards/rejected": -0.569810152053833,
      "step": 678
    },
    {
      "epoch": 0.4223950233281493,
      "grad_norm": 0.3675094246864319,
      "learning_rate": 3.3025e-05,
      "log_odds_chosen": 8.166585922241211,
      "log_odds_ratio": -0.18817320466041565,
      "logits/chosen": 0.23240971565246582,
      "logits/rejected": 0.30073827505111694,
      "logps/chosen": -0.8773943185806274,
      "logps/rejected": -8.555867195129395,
      "loss": 3.5129,
      "nll_loss": 3.4940483570098877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08773943036794662,
      "rewards/margins": 0.7678471803665161,
      "rewards/rejected": -0.8555866479873657,
      "step": 679
    },
    {
      "epoch": 0.4230171073094868,
      "grad_norm": 0.3875802159309387,
      "learning_rate": 3.3e-05,
      "log_odds_chosen": 7.042316913604736,
      "log_odds_ratio": -0.33785781264305115,
      "logits/chosen": 0.2113608419895172,
      "logits/rejected": 0.20565426349639893,
      "logps/chosen": -1.0790364742279053,
      "logps/rejected": -7.6537089347839355,
      "loss": 3.4207,
      "nll_loss": 3.3869504928588867,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10790365934371948,
      "rewards/margins": 0.6574673056602478,
      "rewards/rejected": -0.7653709053993225,
      "step": 680
    },
    {
      "epoch": 0.4236391912908243,
      "grad_norm": 0.47894734144210815,
      "learning_rate": 3.2975e-05,
      "log_odds_chosen": 8.363826751708984,
      "log_odds_ratio": -0.121647410094738,
      "logits/chosen": 0.16830061376094818,
      "logits/rejected": 0.7813224196434021,
      "logps/chosen": -1.120478630065918,
      "logps/rejected": -9.116507530212402,
      "loss": 2.7327,
      "nll_loss": 2.720547676086426,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11204788088798523,
      "rewards/margins": 0.7996028661727905,
      "rewards/rejected": -0.9116507768630981,
      "step": 681
    },
    {
      "epoch": 0.42426127527216173,
      "grad_norm": 0.5742572546005249,
      "learning_rate": 3.295e-05,
      "log_odds_chosen": 8.133566856384277,
      "log_odds_ratio": -0.07514964044094086,
      "logits/chosen": 0.10974755883216858,
      "logits/rejected": 0.43608176708221436,
      "logps/chosen": -0.74546879529953,
      "logps/rejected": -8.146604537963867,
      "loss": 2.561,
      "nll_loss": 2.553471803665161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07454688102006912,
      "rewards/margins": 0.7401136159896851,
      "rewards/rejected": -0.8146604299545288,
      "step": 682
    },
    {
      "epoch": 0.42488335925349924,
      "grad_norm": 0.4656708240509033,
      "learning_rate": 3.2925e-05,
      "log_odds_chosen": 5.774636745452881,
      "log_odds_ratio": -0.13835880160331726,
      "logits/chosen": 0.3287077248096466,
      "logits/rejected": 0.6844013929367065,
      "logps/chosen": -1.2045843601226807,
      "logps/rejected": -6.656024932861328,
      "loss": 3.3318,
      "nll_loss": 3.3180010318756104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12045843154191971,
      "rewards/margins": 0.5451440811157227,
      "rewards/rejected": -0.6656025052070618,
      "step": 683
    },
    {
      "epoch": 0.4255054432348367,
      "grad_norm": 0.4819745421409607,
      "learning_rate": 3.29e-05,
      "log_odds_chosen": 11.101076126098633,
      "log_odds_ratio": -0.05002117529511452,
      "logits/chosen": 0.4243965446949005,
      "logits/rejected": 1.0198917388916016,
      "logps/chosen": -1.1497440338134766,
      "logps/rejected": -11.876899719238281,
      "loss": 3.3201,
      "nll_loss": 3.315087080001831,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11497440934181213,
      "rewards/margins": 1.0727156400680542,
      "rewards/rejected": -1.187690019607544,
      "step": 684
    },
    {
      "epoch": 0.4261275272161742,
      "grad_norm": 0.5185660719871521,
      "learning_rate": 3.2875e-05,
      "log_odds_chosen": 11.11820125579834,
      "log_odds_ratio": -0.0768592581152916,
      "logits/chosen": 0.29388269782066345,
      "logits/rejected": 0.9650092124938965,
      "logps/chosen": -1.116542100906372,
      "logps/rejected": -11.861469268798828,
      "loss": 2.4838,
      "nll_loss": 2.4760661125183105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11165420711040497,
      "rewards/margins": 1.0744926929473877,
      "rewards/rejected": -1.186146855354309,
      "step": 685
    },
    {
      "epoch": 0.42674961119751165,
      "grad_norm": 0.6650855541229248,
      "learning_rate": 3.2850000000000006e-05,
      "log_odds_chosen": 5.216774940490723,
      "log_odds_ratio": -0.18936312198638916,
      "logits/chosen": 0.33131909370422363,
      "logits/rejected": 0.5103744864463806,
      "logps/chosen": -1.0327638387680054,
      "logps/rejected": -5.904963493347168,
      "loss": 2.7629,
      "nll_loss": 2.7439844608306885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10327637195587158,
      "rewards/margins": 0.48721998929977417,
      "rewards/rejected": -0.5904964208602905,
      "step": 686
    },
    {
      "epoch": 0.42737169517884915,
      "grad_norm": 0.48068517446517944,
      "learning_rate": 3.2825e-05,
      "log_odds_chosen": 7.785678386688232,
      "log_odds_ratio": -0.4847630262374878,
      "logits/chosen": 0.31015443801879883,
      "logits/rejected": 0.8882300853729248,
      "logps/chosen": -1.257796049118042,
      "logps/rejected": -8.93236255645752,
      "loss": 3.1948,
      "nll_loss": 3.146324872970581,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12577959895133972,
      "rewards/margins": 0.767456591129303,
      "rewards/rejected": -0.8932361602783203,
      "step": 687
    },
    {
      "epoch": 0.4279937791601866,
      "grad_norm": 0.5043879747390747,
      "learning_rate": 3.2800000000000004e-05,
      "log_odds_chosen": 9.975924491882324,
      "log_odds_ratio": -0.09700936079025269,
      "logits/chosen": 0.4725090265274048,
      "logits/rejected": 0.9532057642936707,
      "logps/chosen": -1.3929022550582886,
      "logps/rejected": -11.091150283813477,
      "loss": 3.2531,
      "nll_loss": 3.2434072494506836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1392902284860611,
      "rewards/margins": 0.9698247909545898,
      "rewards/rejected": -1.1091151237487793,
      "step": 688
    },
    {
      "epoch": 0.4286158631415241,
      "grad_norm": 0.379336953163147,
      "learning_rate": 3.2775e-05,
      "log_odds_chosen": 6.284054279327393,
      "log_odds_ratio": -0.14844508469104767,
      "logits/chosen": 0.22702661156654358,
      "logits/rejected": 0.30767202377319336,
      "logps/chosen": -0.7351300120353699,
      "logps/rejected": -6.354801654815674,
      "loss": 3.1944,
      "nll_loss": 3.179506778717041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07351300120353699,
      "rewards/margins": 0.561967134475708,
      "rewards/rejected": -0.6354801058769226,
      "step": 689
    },
    {
      "epoch": 0.42923794712286156,
      "grad_norm": 0.3885799050331116,
      "learning_rate": 3.275e-05,
      "log_odds_chosen": 8.412586212158203,
      "log_odds_ratio": -0.14217258989810944,
      "logits/chosen": 0.307309627532959,
      "logits/rejected": 0.5339474081993103,
      "logps/chosen": -1.0682765245437622,
      "logps/rejected": -9.073349952697754,
      "loss": 3.3597,
      "nll_loss": 3.3454573154449463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10682766139507294,
      "rewards/margins": 0.8005073070526123,
      "rewards/rejected": -0.9073349833488464,
      "step": 690
    },
    {
      "epoch": 0.42986003110419907,
      "grad_norm": 0.4812905192375183,
      "learning_rate": 3.2725e-05,
      "log_odds_chosen": 4.128777503967285,
      "log_odds_ratio": -0.2909623384475708,
      "logits/chosen": 0.2246783971786499,
      "logits/rejected": 0.3128810524940491,
      "logps/chosen": -0.982389509677887,
      "logps/rejected": -4.593428134918213,
      "loss": 3.1373,
      "nll_loss": 3.1081809997558594,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09823895990848541,
      "rewards/margins": 0.361103892326355,
      "rewards/rejected": -0.4593428373336792,
      "step": 691
    },
    {
      "epoch": 0.4304821150855365,
      "grad_norm": 0.4728715121746063,
      "learning_rate": 3.27e-05,
      "log_odds_chosen": 8.828766822814941,
      "log_odds_ratio": -0.32048314809799194,
      "logits/chosen": 0.239055335521698,
      "logits/rejected": 0.45625340938568115,
      "logps/chosen": -0.9721330404281616,
      "logps/rejected": -9.417234420776367,
      "loss": 3.3518,
      "nll_loss": 3.319707155227661,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09721331298351288,
      "rewards/margins": 0.8445100784301758,
      "rewards/rejected": -0.9417234659194946,
      "step": 692
    },
    {
      "epoch": 0.431104199066874,
      "grad_norm": 0.6615074872970581,
      "learning_rate": 3.2675e-05,
      "log_odds_chosen": 3.707711696624756,
      "log_odds_ratio": -0.5199106931686401,
      "logits/chosen": 0.13658814132213593,
      "logits/rejected": 0.24072976410388947,
      "logps/chosen": -1.3815832138061523,
      "logps/rejected": -4.977440357208252,
      "loss": 3.5352,
      "nll_loss": 3.483212947845459,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13815832138061523,
      "rewards/margins": 0.359585702419281,
      "rewards/rejected": -0.49774405360221863,
      "step": 693
    },
    {
      "epoch": 0.43172628304821153,
      "grad_norm": 0.5314946174621582,
      "learning_rate": 3.265e-05,
      "log_odds_chosen": 12.457331657409668,
      "log_odds_ratio": -0.14057813584804535,
      "logits/chosen": 0.2213817536830902,
      "logits/rejected": 0.8500862121582031,
      "logps/chosen": -1.0103843212127686,
      "logps/rejected": -12.985528945922852,
      "loss": 2.7115,
      "nll_loss": 2.697409152984619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10103843361139297,
      "rewards/margins": 1.197514295578003,
      "rewards/rejected": -1.2985528707504272,
      "step": 694
    },
    {
      "epoch": 0.432348367029549,
      "grad_norm": 0.47115829586982727,
      "learning_rate": 3.2625e-05,
      "log_odds_chosen": 16.424076080322266,
      "log_odds_ratio": -0.10967248678207397,
      "logits/chosen": 0.17491212487220764,
      "logits/rejected": 0.6353819370269775,
      "logps/chosen": -1.1987203359603882,
      "logps/rejected": -17.27529525756836,
      "loss": 3.0507,
      "nll_loss": 3.0397396087646484,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11987203359603882,
      "rewards/margins": 1.6076574325561523,
      "rewards/rejected": -1.727529525756836,
      "step": 695
    },
    {
      "epoch": 0.4329704510108865,
      "grad_norm": 0.4146907329559326,
      "learning_rate": 3.26e-05,
      "log_odds_chosen": 10.058050155639648,
      "log_odds_ratio": -0.004779008217155933,
      "logits/chosen": 0.33498451113700867,
      "logits/rejected": 0.9141867160797119,
      "logps/chosen": -1.1475989818572998,
      "logps/rejected": -10.654457092285156,
      "loss": 3.33,
      "nll_loss": 3.329507350921631,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1147598996758461,
      "rewards/margins": 0.9506858587265015,
      "rewards/rejected": -1.0654456615447998,
      "step": 696
    },
    {
      "epoch": 0.43359253499222394,
      "grad_norm": 0.771777868270874,
      "learning_rate": 3.2575e-05,
      "log_odds_chosen": 5.501579761505127,
      "log_odds_ratio": -0.4520403742790222,
      "logits/chosen": 0.11421041190624237,
      "logits/rejected": 0.4084751307964325,
      "logps/chosen": -1.05423903465271,
      "logps/rejected": -6.390564441680908,
      "loss": 2.4735,
      "nll_loss": 2.4283266067504883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10542389750480652,
      "rewards/margins": 0.5336325764656067,
      "rewards/rejected": -0.6390565037727356,
      "step": 697
    },
    {
      "epoch": 0.43421461897356145,
      "grad_norm": 0.41613906621932983,
      "learning_rate": 3.2550000000000005e-05,
      "log_odds_chosen": 11.464178085327148,
      "log_odds_ratio": -0.3043498396873474,
      "logits/chosen": 0.23589307069778442,
      "logits/rejected": 0.7353752851486206,
      "logps/chosen": -1.2409882545471191,
      "logps/rejected": -12.499031066894531,
      "loss": 3.0398,
      "nll_loss": 3.0093822479248047,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12409882992506027,
      "rewards/margins": 1.1258044242858887,
      "rewards/rejected": -1.2499030828475952,
      "step": 698
    },
    {
      "epoch": 0.4348367029548989,
      "grad_norm": 0.41328713297843933,
      "learning_rate": 3.2525e-05,
      "log_odds_chosen": 10.817594528198242,
      "log_odds_ratio": -0.10718300193548203,
      "logits/chosen": 0.25043705105781555,
      "logits/rejected": 0.6431066393852234,
      "logps/chosen": -0.8873676061630249,
      "logps/rejected": -11.163918495178223,
      "loss": 3.1033,
      "nll_loss": 3.0926289558410645,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08873676508665085,
      "rewards/margins": 1.0276551246643066,
      "rewards/rejected": -1.1163920164108276,
      "step": 699
    },
    {
      "epoch": 0.4354587869362364,
      "grad_norm": 0.4063067138195038,
      "learning_rate": 3.2500000000000004e-05,
      "log_odds_chosen": 12.294190406799316,
      "log_odds_ratio": -0.12343909591436386,
      "logits/chosen": 0.33206403255462646,
      "logits/rejected": 0.7490406632423401,
      "logps/chosen": -0.9793272614479065,
      "logps/rejected": -12.811656951904297,
      "loss": 3.2405,
      "nll_loss": 3.2281508445739746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09793273359537125,
      "rewards/margins": 1.1832330226898193,
      "rewards/rejected": -1.2811657190322876,
      "step": 700
    },
    {
      "epoch": 0.43608087091757386,
      "grad_norm": 9.865860939025879,
      "learning_rate": 3.2474999999999997e-05,
      "log_odds_chosen": 8.678180694580078,
      "log_odds_ratio": -0.7480700016021729,
      "logits/chosen": 0.48775994777679443,
      "logits/rejected": 1.0069730281829834,
      "logps/chosen": -3.711902618408203,
      "logps/rejected": -12.224882125854492,
      "loss": 3.892,
      "nll_loss": 3.8171887397766113,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.37119027972221375,
      "rewards/margins": 0.8512980341911316,
      "rewards/rejected": -1.2224884033203125,
      "step": 701
    },
    {
      "epoch": 0.43670295489891137,
      "grad_norm": 15.485320091247559,
      "learning_rate": 3.245e-05,
      "log_odds_chosen": 10.974223136901855,
      "log_odds_ratio": -0.2399260401725769,
      "logits/chosen": 0.2739264965057373,
      "logits/rejected": 0.861457347869873,
      "logps/chosen": -1.5031821727752686,
      "logps/rejected": -12.213601112365723,
      "loss": 3.1174,
      "nll_loss": 3.093437671661377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1503182202577591,
      "rewards/margins": 1.0710419416427612,
      "rewards/rejected": -1.221360206604004,
      "step": 702
    },
    {
      "epoch": 0.4373250388802488,
      "grad_norm": 0.6337563395500183,
      "learning_rate": 3.2425e-05,
      "log_odds_chosen": 10.547877311706543,
      "log_odds_ratio": -0.17687174677848816,
      "logits/chosen": 0.16326065361499786,
      "logits/rejected": 0.6496604084968567,
      "logps/chosen": -1.749812364578247,
      "logps/rejected": -12.043559074401855,
      "loss": 3.144,
      "nll_loss": 3.126358985900879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1749812364578247,
      "rewards/margins": 1.0293747186660767,
      "rewards/rejected": -1.2043559551239014,
      "step": 703
    },
    {
      "epoch": 0.4379471228615863,
      "grad_norm": 0.4531756043434143,
      "learning_rate": 3.24e-05,
      "log_odds_chosen": 7.200477600097656,
      "log_odds_ratio": -0.31999287009239197,
      "logits/chosen": 0.2357894331216812,
      "logits/rejected": 0.5349158644676208,
      "logps/chosen": -1.258987545967102,
      "logps/rejected": -8.253671646118164,
      "loss": 3.0629,
      "nll_loss": 3.030871868133545,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12589874863624573,
      "rewards/margins": 0.6994683146476746,
      "rewards/rejected": -0.8253670930862427,
      "step": 704
    },
    {
      "epoch": 0.4385692068429238,
      "grad_norm": 0.9053670167922974,
      "learning_rate": 3.2375e-05,
      "log_odds_chosen": 10.318868637084961,
      "log_odds_ratio": -0.1663799285888672,
      "logits/chosen": 0.26653003692626953,
      "logits/rejected": 0.845885157585144,
      "logps/chosen": -1.1329401731491089,
      "logps/rejected": -11.080015182495117,
      "loss": 2.6064,
      "nll_loss": 2.589715003967285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11329401284456253,
      "rewards/margins": 0.994707465171814,
      "rewards/rejected": -1.108001470565796,
      "step": 705
    },
    {
      "epoch": 0.4391912908242613,
      "grad_norm": 0.7091290950775146,
      "learning_rate": 3.235e-05,
      "log_odds_chosen": 5.621170997619629,
      "log_odds_ratio": -0.23729656636714935,
      "logits/chosen": 0.16418294608592987,
      "logits/rejected": 0.46262502670288086,
      "logps/chosen": -1.284653663635254,
      "logps/rejected": -6.603672027587891,
      "loss": 2.5671,
      "nll_loss": 2.543408155441284,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12846536934375763,
      "rewards/margins": 0.5319017767906189,
      "rewards/rejected": -0.6603671908378601,
      "step": 706
    },
    {
      "epoch": 0.43981337480559873,
      "grad_norm": 0.4145105183124542,
      "learning_rate": 3.2325e-05,
      "log_odds_chosen": 11.086092948913574,
      "log_odds_ratio": -0.12887164950370789,
      "logits/chosen": 0.19201578199863434,
      "logits/rejected": 0.8350374102592468,
      "logps/chosen": -1.087384581565857,
      "logps/rejected": -11.732353210449219,
      "loss": 2.7902,
      "nll_loss": 2.77732515335083,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10873845964670181,
      "rewards/margins": 1.0644968748092651,
      "rewards/rejected": -1.1732354164123535,
      "step": 707
    },
    {
      "epoch": 0.44043545878693624,
      "grad_norm": 0.4064066708087921,
      "learning_rate": 3.2300000000000006e-05,
      "log_odds_chosen": 9.882148742675781,
      "log_odds_ratio": -0.08438535779714584,
      "logits/chosen": 0.2323889285326004,
      "logits/rejected": 0.5477718710899353,
      "logps/chosen": -1.0335191488265991,
      "logps/rejected": -10.476679801940918,
      "loss": 3.3664,
      "nll_loss": 3.3579728603363037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10335192084312439,
      "rewards/margins": 0.944316029548645,
      "rewards/rejected": -1.0476679801940918,
      "step": 708
    },
    {
      "epoch": 0.4410575427682737,
      "grad_norm": 0.46960577368736267,
      "learning_rate": 3.2275e-05,
      "log_odds_chosen": 9.509929656982422,
      "log_odds_ratio": -0.04786451533436775,
      "logits/chosen": 0.3471740186214447,
      "logits/rejected": 0.7918584942817688,
      "logps/chosen": -1.0728213787078857,
      "logps/rejected": -10.050264358520508,
      "loss": 3.4729,
      "nll_loss": 3.468099355697632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1072821393609047,
      "rewards/margins": 0.8977442979812622,
      "rewards/rejected": -1.0050263404846191,
      "step": 709
    },
    {
      "epoch": 0.4416796267496112,
      "grad_norm": 0.4803198277950287,
      "learning_rate": 3.2250000000000005e-05,
      "log_odds_chosen": 9.757268905639648,
      "log_odds_ratio": -0.057732146233320236,
      "logits/chosen": 0.23063698410987854,
      "logits/rejected": 0.6390030384063721,
      "logps/chosen": -1.1186230182647705,
      "logps/rejected": -10.331436157226562,
      "loss": 2.9499,
      "nll_loss": 2.944155693054199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11186229437589645,
      "rewards/margins": 0.9212814569473267,
      "rewards/rejected": -1.0331437587738037,
      "step": 710
    },
    {
      "epoch": 0.4423017107309487,
      "grad_norm": 0.5018370747566223,
      "learning_rate": 3.2225e-05,
      "log_odds_chosen": 7.726747512817383,
      "log_odds_ratio": -0.13159742951393127,
      "logits/chosen": 0.19366326928138733,
      "logits/rejected": 0.6917151808738708,
      "logps/chosen": -1.2360267639160156,
      "logps/rejected": -8.579374313354492,
      "loss": 2.7357,
      "nll_loss": 2.722532272338867,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12360267341136932,
      "rewards/margins": 0.7343348264694214,
      "rewards/rejected": -0.8579374551773071,
      "step": 711
    },
    {
      "epoch": 0.44292379471228616,
      "grad_norm": 0.38528966903686523,
      "learning_rate": 3.2200000000000003e-05,
      "log_odds_chosen": 7.1198835372924805,
      "log_odds_ratio": -0.08342044800519943,
      "logits/chosen": 0.08595260232686996,
      "logits/rejected": 0.31091731786727905,
      "logps/chosen": -0.8347682952880859,
      "logps/rejected": -7.294371604919434,
      "loss": 2.9365,
      "nll_loss": 2.9281294345855713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08347682654857635,
      "rewards/margins": 0.6459603309631348,
      "rewards/rejected": -0.7294372320175171,
      "step": 712
    },
    {
      "epoch": 0.44354587869362366,
      "grad_norm": 0.4061030447483063,
      "learning_rate": 3.2175e-05,
      "log_odds_chosen": 10.162424087524414,
      "log_odds_ratio": -0.04226357489824295,
      "logits/chosen": 0.4276748597621918,
      "logits/rejected": 0.6757638454437256,
      "logps/chosen": -1.3077592849731445,
      "logps/rejected": -11.131202697753906,
      "loss": 3.7205,
      "nll_loss": 3.7162415981292725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13077592849731445,
      "rewards/margins": 0.982344388961792,
      "rewards/rejected": -1.1131203174591064,
      "step": 713
    },
    {
      "epoch": 0.4441679626749611,
      "grad_norm": 0.4195731282234192,
      "learning_rate": 3.215e-05,
      "log_odds_chosen": 6.164430618286133,
      "log_odds_ratio": -0.18825969099998474,
      "logits/chosen": 0.3099673092365265,
      "logits/rejected": 0.4048096835613251,
      "logps/chosen": -1.0167288780212402,
      "logps/rejected": -6.674760818481445,
      "loss": 3.1177,
      "nll_loss": 3.098839282989502,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10167289525270462,
      "rewards/margins": 0.5658032298088074,
      "rewards/rejected": -0.6674761176109314,
      "step": 714
    },
    {
      "epoch": 0.4447900466562986,
      "grad_norm": 0.45698559284210205,
      "learning_rate": 3.2125e-05,
      "log_odds_chosen": 6.022528648376465,
      "log_odds_ratio": -0.13800564408302307,
      "logits/chosen": 0.34452202916145325,
      "logits/rejected": 0.5671758055686951,
      "logps/chosen": -1.0680052042007446,
      "logps/rejected": -6.720895290374756,
      "loss": 3.0917,
      "nll_loss": 3.077878475189209,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10680052638053894,
      "rewards/margins": 0.5652889609336853,
      "rewards/rejected": -0.6720895171165466,
      "step": 715
    },
    {
      "epoch": 0.4454121306376361,
      "grad_norm": 0.5258144736289978,
      "learning_rate": 3.21e-05,
      "log_odds_chosen": 7.468056678771973,
      "log_odds_ratio": -0.05660433694720268,
      "logits/chosen": 0.29723796248435974,
      "logits/rejected": 0.726962685585022,
      "logps/chosen": -1.1292157173156738,
      "logps/rejected": -8.210090637207031,
      "loss": 2.5706,
      "nll_loss": 2.564983367919922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1129215732216835,
      "rewards/margins": 0.7080875039100647,
      "rewards/rejected": -0.8210090398788452,
      "step": 716
    },
    {
      "epoch": 0.4460342146189736,
      "grad_norm": 0.6525778770446777,
      "learning_rate": 3.2075e-05,
      "log_odds_chosen": 8.634244918823242,
      "log_odds_ratio": -0.2098091095685959,
      "logits/chosen": 0.3678334951400757,
      "logits/rejected": 0.6341312527656555,
      "logps/chosen": -1.0692224502563477,
      "logps/rejected": -8.939923286437988,
      "loss": 3.4355,
      "nll_loss": 3.4145524501800537,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10692223906517029,
      "rewards/margins": 0.7870701551437378,
      "rewards/rejected": -0.8939923644065857,
      "step": 717
    },
    {
      "epoch": 0.44665629860031103,
      "grad_norm": 0.44750717282295227,
      "learning_rate": 3.205e-05,
      "log_odds_chosen": 3.406766653060913,
      "log_odds_ratio": -0.31649070978164673,
      "logits/chosen": 0.3066599369049072,
      "logits/rejected": 0.2885739207267761,
      "logps/chosen": -1.0535868406295776,
      "logps/rejected": -4.101262092590332,
      "loss": 3.3799,
      "nll_loss": 3.3482348918914795,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10535868257284164,
      "rewards/margins": 0.3047674894332886,
      "rewards/rejected": -0.4101262092590332,
      "step": 718
    },
    {
      "epoch": 0.44727838258164854,
      "grad_norm": 0.46216386556625366,
      "learning_rate": 3.2025e-05,
      "log_odds_chosen": 9.778112411499023,
      "log_odds_ratio": -0.0030376752838492393,
      "logits/chosen": 0.33077576756477356,
      "logits/rejected": 0.7403644323348999,
      "logps/chosen": -1.0820248126983643,
      "logps/rejected": -10.346675872802734,
      "loss": 3.2615,
      "nll_loss": 3.261215925216675,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1082024872303009,
      "rewards/margins": 0.9264651536941528,
      "rewards/rejected": -1.034667730331421,
      "step": 719
    },
    {
      "epoch": 0.447900466562986,
      "grad_norm": 0.749276340007782,
      "learning_rate": 3.2000000000000005e-05,
      "log_odds_chosen": 5.970105171203613,
      "log_odds_ratio": -0.3304394483566284,
      "logits/chosen": 0.16680698096752167,
      "logits/rejected": 0.3060448467731476,
      "logps/chosen": -1.435762882232666,
      "logps/rejected": -7.024744033813477,
      "loss": 2.9031,
      "nll_loss": 2.870039463043213,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14357629418373108,
      "rewards/margins": 0.5588980913162231,
      "rewards/rejected": -0.7024743556976318,
      "step": 720
    },
    {
      "epoch": 0.4485225505443235,
      "grad_norm": 0.539753794670105,
      "learning_rate": 3.1975e-05,
      "log_odds_chosen": 3.475248098373413,
      "log_odds_ratio": -0.529831051826477,
      "logits/chosen": 0.2159595638513565,
      "logits/rejected": 0.42169877886772156,
      "logps/chosen": -1.6044065952301025,
      "logps/rejected": -5.004334926605225,
      "loss": 3.2259,
      "nll_loss": 3.172966241836548,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16044065356254578,
      "rewards/margins": 0.33999279141426086,
      "rewards/rejected": -0.5004334449768066,
      "step": 721
    },
    {
      "epoch": 0.44914463452566095,
      "grad_norm": 0.452215313911438,
      "learning_rate": 3.1950000000000004e-05,
      "log_odds_chosen": 7.956733226776123,
      "log_odds_ratio": -0.3105076849460602,
      "logits/chosen": 0.16990184783935547,
      "logits/rejected": 0.48880094289779663,
      "logps/chosen": -1.11954927444458,
      "logps/rejected": -8.724953651428223,
      "loss": 2.9063,
      "nll_loss": 2.8752598762512207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1119549423456192,
      "rewards/margins": 0.7605404257774353,
      "rewards/rejected": -0.8724953532218933,
      "step": 722
    },
    {
      "epoch": 0.44976671850699845,
      "grad_norm": 0.5523648262023926,
      "learning_rate": 3.1925e-05,
      "log_odds_chosen": 6.903494358062744,
      "log_odds_ratio": -0.2759135067462921,
      "logits/chosen": 0.0760892778635025,
      "logits/rejected": 0.32330164313316345,
      "logps/chosen": -1.1168400049209595,
      "logps/rejected": -7.690395832061768,
      "loss": 2.3726,
      "nll_loss": 2.344987392425537,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11168399453163147,
      "rewards/margins": 0.6573556065559387,
      "rewards/rejected": -0.7690396308898926,
      "step": 723
    },
    {
      "epoch": 0.4503888024883359,
      "grad_norm": 0.4526844322681427,
      "learning_rate": 3.19e-05,
      "log_odds_chosen": 7.047250270843506,
      "log_odds_ratio": -0.1251072883605957,
      "logits/chosen": 0.1379421055316925,
      "logits/rejected": 0.5924164652824402,
      "logps/chosen": -1.2250146865844727,
      "logps/rejected": -7.931028842926025,
      "loss": 2.6565,
      "nll_loss": 2.643982172012329,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12250147014856339,
      "rewards/margins": 0.6706014275550842,
      "rewards/rejected": -0.7931028604507446,
      "step": 724
    },
    {
      "epoch": 0.4510108864696734,
      "grad_norm": 0.5021094083786011,
      "learning_rate": 3.1875e-05,
      "log_odds_chosen": 9.610315322875977,
      "log_odds_ratio": -0.24255721271038055,
      "logits/chosen": 0.3092350363731384,
      "logits/rejected": 0.9811382293701172,
      "logps/chosen": -1.3039846420288086,
      "logps/rejected": -10.703886032104492,
      "loss": 2.5964,
      "nll_loss": 2.572134256362915,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1303984671831131,
      "rewards/margins": 0.9399901628494263,
      "rewards/rejected": -1.0703885555267334,
      "step": 725
    },
    {
      "epoch": 0.45163297045101086,
      "grad_norm": 6.453852653503418,
      "learning_rate": 3.185e-05,
      "log_odds_chosen": 9.646251678466797,
      "log_odds_ratio": -0.11725477129220963,
      "logits/chosen": 0.49717283248901367,
      "logits/rejected": 0.8251327276229858,
      "logps/chosen": -1.5012346506118774,
      "logps/rejected": -10.704265594482422,
      "loss": 3.4432,
      "nll_loss": 3.431502103805542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1501234769821167,
      "rewards/margins": 0.9203031659126282,
      "rewards/rejected": -1.0704265832901,
      "step": 726
    },
    {
      "epoch": 0.45225505443234837,
      "grad_norm": 0.407627135515213,
      "learning_rate": 3.1825e-05,
      "log_odds_chosen": 6.178986549377441,
      "log_odds_ratio": -0.25438594818115234,
      "logits/chosen": 0.1292475461959839,
      "logits/rejected": 0.43548843264579773,
      "logps/chosen": -0.9622341990470886,
      "logps/rejected": -6.761863708496094,
      "loss": 2.5614,
      "nll_loss": 2.5359420776367188,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09622342884540558,
      "rewards/margins": 0.5799629092216492,
      "rewards/rejected": -0.6761863231658936,
      "step": 727
    },
    {
      "epoch": 0.4528771384136858,
      "grad_norm": 0.5671166777610779,
      "learning_rate": 3.18e-05,
      "log_odds_chosen": 9.908763885498047,
      "log_odds_ratio": -0.15785574913024902,
      "logits/chosen": 0.35942667722702026,
      "logits/rejected": 0.8982104063034058,
      "logps/chosen": -1.2879177331924438,
      "logps/rejected": -10.947426795959473,
      "loss": 3.2627,
      "nll_loss": 3.2469372749328613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12879177927970886,
      "rewards/margins": 0.9659509062767029,
      "rewards/rejected": -1.094742774963379,
      "step": 728
    },
    {
      "epoch": 0.4534992223950233,
      "grad_norm": 0.4187334179878235,
      "learning_rate": 3.1775e-05,
      "log_odds_chosen": 6.857314586639404,
      "log_odds_ratio": -0.12401160597801208,
      "logits/chosen": 0.29477638006210327,
      "logits/rejected": 0.567139744758606,
      "logps/chosen": -0.97846519947052,
      "logps/rejected": -7.214424133300781,
      "loss": 3.2737,
      "nll_loss": 3.2612991333007812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09784651547670364,
      "rewards/margins": 0.6235959529876709,
      "rewards/rejected": -0.721442461013794,
      "step": 729
    },
    {
      "epoch": 0.45412130637636083,
      "grad_norm": 0.43191951513290405,
      "learning_rate": 3.175e-05,
      "log_odds_chosen": 11.218259811401367,
      "log_odds_ratio": -0.068722665309906,
      "logits/chosen": 0.43568962812423706,
      "logits/rejected": 1.0645880699157715,
      "logps/chosen": -1.2674078941345215,
      "logps/rejected": -12.176996231079102,
      "loss": 3.3798,
      "nll_loss": 3.372969388961792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12674078345298767,
      "rewards/margins": 1.090958833694458,
      "rewards/rejected": -1.217699646949768,
      "step": 730
    },
    {
      "epoch": 0.4547433903576983,
      "grad_norm": 1.9741532802581787,
      "learning_rate": 3.1725e-05,
      "log_odds_chosen": 7.206732273101807,
      "log_odds_ratio": -0.13450881838798523,
      "logits/chosen": 0.30000701546669006,
      "logits/rejected": 0.480673611164093,
      "logps/chosen": -1.4034446477890015,
      "logps/rejected": -8.346330642700195,
      "loss": 3.3633,
      "nll_loss": 3.3498318195343018,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14034447073936462,
      "rewards/margins": 0.6942886114120483,
      "rewards/rejected": -0.8346331119537354,
      "step": 731
    },
    {
      "epoch": 0.4553654743390358,
      "grad_norm": 0.3933872878551483,
      "learning_rate": 3.1700000000000005e-05,
      "log_odds_chosen": 9.562494277954102,
      "log_odds_ratio": -0.07410041987895966,
      "logits/chosen": 0.34438636898994446,
      "logits/rejected": 0.6179068088531494,
      "logps/chosen": -0.7376449704170227,
      "logps/rejected": -9.597999572753906,
      "loss": 3.4239,
      "nll_loss": 3.4165120124816895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07376450300216675,
      "rewards/margins": 0.8860355615615845,
      "rewards/rejected": -0.9598000049591064,
      "step": 732
    },
    {
      "epoch": 0.45598755832037324,
      "grad_norm": 0.4037986695766449,
      "learning_rate": 3.1675e-05,
      "log_odds_chosen": 7.081986427307129,
      "log_odds_ratio": -0.10163398832082748,
      "logits/chosen": 0.18077480792999268,
      "logits/rejected": 0.4592246115207672,
      "logps/chosen": -0.9602686166763306,
      "logps/rejected": -7.555391311645508,
      "loss": 2.9457,
      "nll_loss": 2.935537576675415,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09602686762809753,
      "rewards/margins": 0.6595123410224915,
      "rewards/rejected": -0.7555391788482666,
      "step": 733
    },
    {
      "epoch": 0.45660964230171075,
      "grad_norm": 0.5620434284210205,
      "learning_rate": 3.1650000000000004e-05,
      "log_odds_chosen": 5.531484603881836,
      "log_odds_ratio": -0.32697606086730957,
      "logits/chosen": 0.1813567876815796,
      "logits/rejected": 0.3070491850376129,
      "logps/chosen": -1.2444071769714355,
      "logps/rejected": -6.4295549392700195,
      "loss": 3.2722,
      "nll_loss": 3.239509105682373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12444071471691132,
      "rewards/margins": 0.5185148119926453,
      "rewards/rejected": -0.642955482006073,
      "step": 734
    },
    {
      "epoch": 0.4572317262830482,
      "grad_norm": 0.4302683174610138,
      "learning_rate": 3.1624999999999996e-05,
      "log_odds_chosen": 6.523910999298096,
      "log_odds_ratio": -0.14284959435462952,
      "logits/chosen": 0.04424915090203285,
      "logits/rejected": 0.13490360975265503,
      "logps/chosen": -0.9167971611022949,
      "logps/rejected": -6.748834133148193,
      "loss": 3.1965,
      "nll_loss": 3.1821680068969727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09167972207069397,
      "rewards/margins": 0.5832037329673767,
      "rewards/rejected": -0.6748834848403931,
      "step": 735
    },
    {
      "epoch": 0.4578538102643857,
      "grad_norm": 0.6089425086975098,
      "learning_rate": 3.16e-05,
      "log_odds_chosen": 5.9218363761901855,
      "log_odds_ratio": -0.2695227563381195,
      "logits/chosen": 0.2000441998243332,
      "logits/rejected": 0.4127127230167389,
      "logps/chosen": -1.3729795217514038,
      "logps/rejected": -7.12251091003418,
      "loss": 3.1966,
      "nll_loss": 3.1696414947509766,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13729795813560486,
      "rewards/margins": 0.5749530792236328,
      "rewards/rejected": -0.7122510671615601,
      "step": 736
    },
    {
      "epoch": 0.45847589424572316,
      "grad_norm": 0.5367254018783569,
      "learning_rate": 3.1575e-05,
      "log_odds_chosen": 6.432736396789551,
      "log_odds_ratio": -0.16896378993988037,
      "logits/chosen": 0.06318940222263336,
      "logits/rejected": 0.297978013753891,
      "logps/chosen": -1.222300410270691,
      "logps/rejected": -7.110202312469482,
      "loss": 3.0173,
      "nll_loss": 3.0003702640533447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12223003804683685,
      "rewards/margins": 0.5887902975082397,
      "rewards/rejected": -0.7110202312469482,
      "step": 737
    },
    {
      "epoch": 0.45909797822706067,
      "grad_norm": 4.201266288757324,
      "learning_rate": 3.155e-05,
      "log_odds_chosen": 5.493893623352051,
      "log_odds_ratio": -0.45646530389785767,
      "logits/chosen": 0.21096059679985046,
      "logits/rejected": 0.23016388714313507,
      "logps/chosen": -1.8003606796264648,
      "logps/rejected": -7.092996597290039,
      "loss": 3.7453,
      "nll_loss": 3.6996266841888428,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18003606796264648,
      "rewards/margins": 0.5292636156082153,
      "rewards/rejected": -0.7092996835708618,
      "step": 738
    },
    {
      "epoch": 0.4597200622083981,
      "grad_norm": 0.41682955622673035,
      "learning_rate": 3.1525e-05,
      "log_odds_chosen": 3.3369433879852295,
      "log_odds_ratio": -0.2148256152868271,
      "logits/chosen": 0.12079880386590958,
      "logits/rejected": 0.19746747612953186,
      "logps/chosen": -1.0625683069229126,
      "logps/rejected": -3.9425437450408936,
      "loss": 3.5171,
      "nll_loss": 3.4956541061401367,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10625684261322021,
      "rewards/margins": 0.2879975736141205,
      "rewards/rejected": -0.3942543864250183,
      "step": 739
    },
    {
      "epoch": 0.4603421461897356,
      "grad_norm": 0.589731752872467,
      "learning_rate": 3.15e-05,
      "log_odds_chosen": 3.7177412509918213,
      "log_odds_ratio": -0.39108750224113464,
      "logits/chosen": -0.020437847822904587,
      "logits/rejected": 0.1704711765050888,
      "logps/chosen": -1.1689900159835815,
      "logps/rejected": -4.5871758460998535,
      "loss": 2.4873,
      "nll_loss": 2.4481828212738037,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11689899861812592,
      "rewards/margins": 0.34181857109069824,
      "rewards/rejected": -0.45871755480766296,
      "step": 740
    },
    {
      "epoch": 0.4609642301710731,
      "grad_norm": 0.6688488721847534,
      "learning_rate": 3.1475e-05,
      "log_odds_chosen": 4.912108421325684,
      "log_odds_ratio": -0.13216131925582886,
      "logits/chosen": 0.22601565718650818,
      "logits/rejected": 0.3431988060474396,
      "logps/chosen": -1.2438921928405762,
      "logps/rejected": -5.821355819702148,
      "loss": 3.6307,
      "nll_loss": 3.617436408996582,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12438922375440598,
      "rewards/margins": 0.45774632692337036,
      "rewards/rejected": -0.5821355581283569,
      "step": 741
    },
    {
      "epoch": 0.4615863141524106,
      "grad_norm": 0.433036744594574,
      "learning_rate": 3.145e-05,
      "log_odds_chosen": 3.096634864807129,
      "log_odds_ratio": -0.2476002722978592,
      "logits/chosen": 0.043339017778635025,
      "logits/rejected": -0.03985293209552765,
      "logps/chosen": -0.9964567422866821,
      "logps/rejected": -3.488633155822754,
      "loss": 3.2937,
      "nll_loss": 3.26889705657959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09964566677808762,
      "rewards/margins": 0.2492176592350006,
      "rewards/rejected": -0.34886330366134644,
      "step": 742
    },
    {
      "epoch": 0.46220839813374803,
      "grad_norm": 0.6005847454071045,
      "learning_rate": 3.1425e-05,
      "log_odds_chosen": 6.896453857421875,
      "log_odds_ratio": -0.1344524621963501,
      "logits/chosen": 0.08107919245958328,
      "logits/rejected": 0.26698002219200134,
      "logps/chosen": -0.9853373169898987,
      "logps/rejected": -7.432549953460693,
      "loss": 2.531,
      "nll_loss": 2.5175111293792725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09853372722864151,
      "rewards/margins": 0.644721269607544,
      "rewards/rejected": -0.7432550191879272,
      "step": 743
    },
    {
      "epoch": 0.46283048211508554,
      "grad_norm": 0.5875562429428101,
      "learning_rate": 3.1400000000000004e-05,
      "log_odds_chosen": 6.2522687911987305,
      "log_odds_ratio": -0.14717459678649902,
      "logits/chosen": 0.0802212506532669,
      "logits/rejected": 0.36846601963043213,
      "logps/chosen": -1.2279167175292969,
      "logps/rejected": -7.11676025390625,
      "loss": 2.9711,
      "nll_loss": 2.9563703536987305,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12279167771339417,
      "rewards/margins": 0.5888843536376953,
      "rewards/rejected": -0.7116760015487671,
      "step": 744
    },
    {
      "epoch": 0.463452566096423,
      "grad_norm": 0.6242512464523315,
      "learning_rate": 3.1375e-05,
      "log_odds_chosen": 8.841582298278809,
      "log_odds_ratio": -0.2243223786354065,
      "logits/chosen": 0.14888420701026917,
      "logits/rejected": 0.574272096157074,
      "logps/chosen": -1.1364891529083252,
      "logps/rejected": -9.670944213867188,
      "loss": 2.973,
      "nll_loss": 2.9505813121795654,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1136489063501358,
      "rewards/margins": 0.8534455299377441,
      "rewards/rejected": -0.9670944213867188,
      "step": 745
    },
    {
      "epoch": 0.4640746500777605,
      "grad_norm": 0.39740875363349915,
      "learning_rate": 3.135e-05,
      "log_odds_chosen": 6.4931817054748535,
      "log_odds_ratio": -0.07528342306613922,
      "logits/chosen": 0.23848633468151093,
      "logits/rejected": 0.4403890371322632,
      "logps/chosen": -1.1312795877456665,
      "logps/rejected": -7.221208572387695,
      "loss": 3.546,
      "nll_loss": 3.5384349822998047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11312796920537949,
      "rewards/margins": 0.608992874622345,
      "rewards/rejected": -0.7221208214759827,
      "step": 746
    },
    {
      "epoch": 0.464696734059098,
      "grad_norm": 0.5737400650978088,
      "learning_rate": 3.1324999999999996e-05,
      "log_odds_chosen": 7.452863693237305,
      "log_odds_ratio": -0.15671104192733765,
      "logits/chosen": 0.03789973258972168,
      "logits/rejected": 0.1605812907218933,
      "logps/chosen": -0.8898376226425171,
      "logps/rejected": -7.745999336242676,
      "loss": 3.0129,
      "nll_loss": 2.997223377227783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08898375928401947,
      "rewards/margins": 0.6856162548065186,
      "rewards/rejected": -0.7745999097824097,
      "step": 747
    },
    {
      "epoch": 0.46531881804043546,
      "grad_norm": 0.4806496500968933,
      "learning_rate": 3.13e-05,
      "log_odds_chosen": 5.818041801452637,
      "log_odds_ratio": -0.1867186278104782,
      "logits/chosen": 0.1712789386510849,
      "logits/rejected": 0.29497987031936646,
      "logps/chosen": -0.8955323696136475,
      "logps/rejected": -6.180495262145996,
      "loss": 3.3477,
      "nll_loss": 3.3290135860443115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08955323696136475,
      "rewards/margins": 0.5284963846206665,
      "rewards/rejected": -0.6180496215820312,
      "step": 748
    },
    {
      "epoch": 0.46594090202177296,
      "grad_norm": 0.6791718006134033,
      "learning_rate": 3.1275e-05,
      "log_odds_chosen": 5.693668842315674,
      "log_odds_ratio": -0.15988114476203918,
      "logits/chosen": 0.11964002251625061,
      "logits/rejected": 0.40675055980682373,
      "logps/chosen": -0.9922550916671753,
      "logps/rejected": -6.27172327041626,
      "loss": 2.7005,
      "nll_loss": 2.68455171585083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09922550618648529,
      "rewards/margins": 0.5279468297958374,
      "rewards/rejected": -0.6271723508834839,
      "step": 749
    },
    {
      "epoch": 0.4665629860031104,
      "grad_norm": 0.6238471865653992,
      "learning_rate": 3.125e-05,
      "log_odds_chosen": 3.5400168895721436,
      "log_odds_ratio": -0.42935463786125183,
      "logits/chosen": 0.0866260677576065,
      "logits/rejected": 0.37089765071868896,
      "logps/chosen": -1.1119719743728638,
      "logps/rejected": -4.440646171569824,
      "loss": 2.7092,
      "nll_loss": 2.666236400604248,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11119719594717026,
      "rewards/margins": 0.33286744356155396,
      "rewards/rejected": -0.4440646469593048,
      "step": 750
    },
    {
      "epoch": 0.4671850699844479,
      "grad_norm": 0.5068716406822205,
      "learning_rate": 3.122500000000001e-05,
      "log_odds_chosen": 10.899909019470215,
      "log_odds_ratio": -0.00871206633746624,
      "logits/chosen": -0.01668960601091385,
      "logits/rejected": 0.5333446264266968,
      "logps/chosen": -1.2172091007232666,
      "logps/rejected": -11.635497093200684,
      "loss": 2.5222,
      "nll_loss": 2.521331310272217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12172091007232666,
      "rewards/margins": 1.0418287515640259,
      "rewards/rejected": -1.1635496616363525,
      "step": 751
    },
    {
      "epoch": 0.46780715396578537,
      "grad_norm": 1.516399621963501,
      "learning_rate": 3.12e-05,
      "log_odds_chosen": 7.541528701782227,
      "log_odds_ratio": -0.1825413852930069,
      "logits/chosen": 0.006808616686612368,
      "logits/rejected": 0.20838546752929688,
      "logps/chosen": -1.2133069038391113,
      "logps/rejected": -8.299160957336426,
      "loss": 2.7663,
      "nll_loss": 2.7480356693267822,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12133070081472397,
      "rewards/margins": 0.7085853815078735,
      "rewards/rejected": -0.8299161195755005,
      "step": 752
    },
    {
      "epoch": 0.4684292379471229,
      "grad_norm": 0.43352144956588745,
      "learning_rate": 3.1175000000000006e-05,
      "log_odds_chosen": 7.610816955566406,
      "log_odds_ratio": -0.33424389362335205,
      "logits/chosen": 0.17765414714813232,
      "logits/rejected": 0.3236091732978821,
      "logps/chosen": -1.1976127624511719,
      "logps/rejected": -8.585079193115234,
      "loss": 3.4346,
      "nll_loss": 3.4012205600738525,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11976128816604614,
      "rewards/margins": 0.738746702671051,
      "rewards/rejected": -0.8585079908370972,
      "step": 753
    },
    {
      "epoch": 0.46905132192846033,
      "grad_norm": 0.4248764216899872,
      "learning_rate": 3.115e-05,
      "log_odds_chosen": 8.949850082397461,
      "log_odds_ratio": -0.18056482076644897,
      "logits/chosen": 0.1752690225839615,
      "logits/rejected": 0.3496873676776886,
      "logps/chosen": -0.8166213035583496,
      "logps/rejected": -9.080907821655273,
      "loss": 3.2551,
      "nll_loss": 3.23703932762146,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0816621258854866,
      "rewards/margins": 0.8264287710189819,
      "rewards/rejected": -0.9080908894538879,
      "step": 754
    },
    {
      "epoch": 0.46967340590979784,
      "grad_norm": 0.5243033170700073,
      "learning_rate": 3.1125000000000004e-05,
      "log_odds_chosen": 12.745326042175293,
      "log_odds_ratio": -0.11361424624919891,
      "logits/chosen": 0.1282683163881302,
      "logits/rejected": 0.7375682592391968,
      "logps/chosen": -1.205803394317627,
      "logps/rejected": -13.61036205291748,
      "loss": 2.9496,
      "nll_loss": 2.938230276107788,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12058034539222717,
      "rewards/margins": 1.2404558658599854,
      "rewards/rejected": -1.3610363006591797,
      "step": 755
    },
    {
      "epoch": 0.4702954898911353,
      "grad_norm": 0.5427597761154175,
      "learning_rate": 3.1100000000000004e-05,
      "log_odds_chosen": 10.578163146972656,
      "log_odds_ratio": -0.34553229808807373,
      "logits/chosen": 0.16474664211273193,
      "logits/rejected": 0.526168704032898,
      "logps/chosen": -1.3434865474700928,
      "logps/rejected": -11.748912811279297,
      "loss": 3.4079,
      "nll_loss": 3.3733925819396973,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13434866070747375,
      "rewards/margins": 1.040542721748352,
      "rewards/rejected": -1.1748913526535034,
      "step": 756
    },
    {
      "epoch": 0.4709175738724728,
      "grad_norm": 0.47395461797714233,
      "learning_rate": 3.1075e-05,
      "log_odds_chosen": 4.500313758850098,
      "log_odds_ratio": -0.33959078788757324,
      "logits/chosen": 0.07310894876718521,
      "logits/rejected": 0.23603522777557373,
      "logps/chosen": -1.0726346969604492,
      "logps/rejected": -5.353082656860352,
      "loss": 2.9118,
      "nll_loss": 2.8778131008148193,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1072634682059288,
      "rewards/margins": 0.428044855594635,
      "rewards/rejected": -0.535308301448822,
      "step": 757
    },
    {
      "epoch": 0.47153965785381025,
      "grad_norm": 0.5754133462905884,
      "learning_rate": 3.105e-05,
      "log_odds_chosen": 9.807731628417969,
      "log_odds_ratio": -0.16157642006874084,
      "logits/chosen": 0.05956118553876877,
      "logits/rejected": 0.5986432433128357,
      "logps/chosen": -1.3279142379760742,
      "logps/rejected": -10.884170532226562,
      "loss": 3.1166,
      "nll_loss": 3.100430488586426,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1327914297580719,
      "rewards/margins": 0.9556256532669067,
      "rewards/rejected": -1.0884170532226562,
      "step": 758
    },
    {
      "epoch": 0.47216174183514775,
      "grad_norm": 0.48073136806488037,
      "learning_rate": 3.1025e-05,
      "log_odds_chosen": 7.494086265563965,
      "log_odds_ratio": -0.2150120735168457,
      "logits/chosen": 0.2095496952533722,
      "logits/rejected": 0.4400617778301239,
      "logps/chosen": -1.2367106676101685,
      "logps/rejected": -8.398553848266602,
      "loss": 3.5941,
      "nll_loss": 3.5726468563079834,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12367106974124908,
      "rewards/margins": 0.7161843776702881,
      "rewards/rejected": -0.839855432510376,
      "step": 759
    },
    {
      "epoch": 0.4727838258164852,
      "grad_norm": 0.5509316325187683,
      "learning_rate": 3.1e-05,
      "log_odds_chosen": 7.781867504119873,
      "log_odds_ratio": -0.1630517691373825,
      "logits/chosen": 0.12625464797019958,
      "logits/rejected": 0.6091369390487671,
      "logps/chosen": -1.2734575271606445,
      "logps/rejected": -8.757940292358398,
      "loss": 2.8018,
      "nll_loss": 2.7855429649353027,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1273457556962967,
      "rewards/margins": 0.748448371887207,
      "rewards/rejected": -0.8757941126823425,
      "step": 760
    },
    {
      "epoch": 0.4734059097978227,
      "grad_norm": 0.5899271368980408,
      "learning_rate": 3.0975e-05,
      "log_odds_chosen": 2.6958911418914795,
      "log_odds_ratio": -0.5016304850578308,
      "logits/chosen": 0.13957029581069946,
      "logits/rejected": 0.23938274383544922,
      "logps/chosen": -1.2764240503311157,
      "logps/rejected": -3.8734121322631836,
      "loss": 3.2298,
      "nll_loss": 3.1796460151672363,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1276424080133438,
      "rewards/margins": 0.2596988081932068,
      "rewards/rejected": -0.3873412013053894,
      "step": 761
    },
    {
      "epoch": 0.47402799377916016,
      "grad_norm": 0.5521773099899292,
      "learning_rate": 3.095e-05,
      "log_odds_chosen": 3.7203664779663086,
      "log_odds_ratio": -0.439362496137619,
      "logits/chosen": -0.036664508283138275,
      "logits/rejected": 0.10708197951316833,
      "logps/chosen": -0.9352907538414001,
      "logps/rejected": -4.322984218597412,
      "loss": 2.5678,
      "nll_loss": 2.52388596534729,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09352907538414001,
      "rewards/margins": 0.3387693762779236,
      "rewards/rejected": -0.4322984218597412,
      "step": 762
    },
    {
      "epoch": 0.47465007776049767,
      "grad_norm": 0.5061217546463013,
      "learning_rate": 3.0925000000000006e-05,
      "log_odds_chosen": 12.82931137084961,
      "log_odds_ratio": -0.07551414519548416,
      "logits/chosen": 0.04354046285152435,
      "logits/rejected": 0.9819294810295105,
      "logps/chosen": -1.4580919742584229,
      "logps/rejected": -13.935245513916016,
      "loss": 2.7455,
      "nll_loss": 2.7379651069641113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14580918848514557,
      "rewards/margins": 1.2477153539657593,
      "rewards/rejected": -1.393524408340454,
      "step": 763
    },
    {
      "epoch": 0.4752721617418352,
      "grad_norm": 0.6883053183555603,
      "learning_rate": 3.09e-05,
      "log_odds_chosen": 7.964182376861572,
      "log_odds_ratio": -0.46369829773902893,
      "logits/chosen": 0.12046404182910919,
      "logits/rejected": 0.6274876594543457,
      "logps/chosen": -1.3564209938049316,
      "logps/rejected": -9.131847381591797,
      "loss": 2.7681,
      "nll_loss": 2.7217345237731934,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13564209640026093,
      "rewards/margins": 0.7775427103042603,
      "rewards/rejected": -0.9131847620010376,
      "step": 764
    },
    {
      "epoch": 0.4758942457231726,
      "grad_norm": 0.44783955812454224,
      "learning_rate": 3.0875000000000005e-05,
      "log_odds_chosen": 9.665321350097656,
      "log_odds_ratio": -0.1914052665233612,
      "logits/chosen": 0.05238525569438934,
      "logits/rejected": 0.6160183548927307,
      "logps/chosen": -1.1813290119171143,
      "logps/rejected": -10.583022117614746,
      "loss": 2.9187,
      "nll_loss": 2.8995184898376465,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11813290417194366,
      "rewards/margins": 0.9401693940162659,
      "rewards/rejected": -1.0583022832870483,
      "step": 765
    },
    {
      "epoch": 0.47651632970451013,
      "grad_norm": 0.48133954405784607,
      "learning_rate": 3.0850000000000004e-05,
      "log_odds_chosen": 10.398935317993164,
      "log_odds_ratio": -0.2939707934856415,
      "logits/chosen": 0.196578711271286,
      "logits/rejected": 0.7910341024398804,
      "logps/chosen": -1.453446626663208,
      "logps/rejected": -11.644519805908203,
      "loss": 3.0305,
      "nll_loss": 3.0011115074157715,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14534467458724976,
      "rewards/margins": 1.0191072225570679,
      "rewards/rejected": -1.1644519567489624,
      "step": 766
    },
    {
      "epoch": 0.4771384136858476,
      "grad_norm": 0.7429364323616028,
      "learning_rate": 3.0825000000000004e-05,
      "log_odds_chosen": 8.819571495056152,
      "log_odds_ratio": -0.17902621626853943,
      "logits/chosen": 0.0832553580403328,
      "logits/rejected": 0.9124725461006165,
      "logps/chosen": -1.139229416847229,
      "logps/rejected": -9.641944885253906,
      "loss": 2.4093,
      "nll_loss": 2.391360282897949,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11392293870449066,
      "rewards/margins": 0.8502715826034546,
      "rewards/rejected": -0.9641945958137512,
      "step": 767
    },
    {
      "epoch": 0.4777604976671851,
      "grad_norm": 0.45216140151023865,
      "learning_rate": 3.08e-05,
      "log_odds_chosen": 10.36209774017334,
      "log_odds_ratio": -0.033273592591285706,
      "logits/chosen": 0.0013059796765446663,
      "logits/rejected": 0.4774538576602936,
      "logps/chosen": -0.9916945695877075,
      "logps/rejected": -10.634331703186035,
      "loss": 2.6309,
      "nll_loss": 2.627547264099121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09916945546865463,
      "rewards/margins": 0.9642637372016907,
      "rewards/rejected": -1.063433289527893,
      "step": 768
    },
    {
      "epoch": 0.47838258164852254,
      "grad_norm": 0.6611552834510803,
      "learning_rate": 3.0775e-05,
      "log_odds_chosen": 13.858925819396973,
      "log_odds_ratio": -0.059990376234054565,
      "logits/chosen": 0.07334032654762268,
      "logits/rejected": 0.49123767018318176,
      "logps/chosen": -0.9776344299316406,
      "logps/rejected": -14.378206253051758,
      "loss": 3.101,
      "nll_loss": 3.0949764251708984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09776344150304794,
      "rewards/margins": 1.340057134628296,
      "rewards/rejected": -1.437820553779602,
      "step": 769
    },
    {
      "epoch": 0.47900466562986005,
      "grad_norm": 2.7856853008270264,
      "learning_rate": 3.075e-05,
      "log_odds_chosen": 11.939106941223145,
      "log_odds_ratio": -0.11589374393224716,
      "logits/chosen": 0.24869629740715027,
      "logits/rejected": 0.7966226935386658,
      "logps/chosen": -1.8126963376998901,
      "logps/rejected": -13.446033477783203,
      "loss": 3.6146,
      "nll_loss": 3.603039264678955,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18126963078975677,
      "rewards/margins": 1.163333773612976,
      "rewards/rejected": -1.3446035385131836,
      "step": 770
    },
    {
      "epoch": 0.4796267496111975,
      "grad_norm": 0.4278852641582489,
      "learning_rate": 3.0725e-05,
      "log_odds_chosen": 16.7786865234375,
      "log_odds_ratio": -0.15743397176265717,
      "logits/chosen": 0.27345868945121765,
      "logits/rejected": 0.9987517595291138,
      "logps/chosen": -0.9886462092399597,
      "logps/rejected": -17.29168128967285,
      "loss": 3.3687,
      "nll_loss": 3.3529460430145264,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09886462986469269,
      "rewards/margins": 1.6303035020828247,
      "rewards/rejected": -1.729168176651001,
      "step": 771
    },
    {
      "epoch": 0.480248833592535,
      "grad_norm": 0.4929329454898834,
      "learning_rate": 3.07e-05,
      "log_odds_chosen": 13.330595970153809,
      "log_odds_ratio": -0.08029691874980927,
      "logits/chosen": 0.1175004094839096,
      "logits/rejected": 0.8302413821220398,
      "logps/chosen": -1.2270184755325317,
      "logps/rejected": -14.165221214294434,
      "loss": 2.9649,
      "nll_loss": 2.956918478012085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12270184606313705,
      "rewards/margins": 1.2938203811645508,
      "rewards/rejected": -1.4165221452713013,
      "step": 772
    },
    {
      "epoch": 0.48087091757387246,
      "grad_norm": 0.4613707661628723,
      "learning_rate": 3.067500000000001e-05,
      "log_odds_chosen": 17.801786422729492,
      "log_odds_ratio": -0.0008926771115511656,
      "logits/chosen": 0.22919896245002747,
      "logits/rejected": 1.450141429901123,
      "logps/chosen": -1.3054852485656738,
      "logps/rejected": -18.71816635131836,
      "loss": 2.5083,
      "nll_loss": 2.5082015991210938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13054853677749634,
      "rewards/margins": 1.7412681579589844,
      "rewards/rejected": -1.8718167543411255,
      "step": 773
    },
    {
      "epoch": 0.48149300155520997,
      "grad_norm": 0.4349910020828247,
      "learning_rate": 3.065e-05,
      "log_odds_chosen": 16.952882766723633,
      "log_odds_ratio": -0.07804927229881287,
      "logits/chosen": -0.0008221510797739029,
      "logits/rejected": 1.0307400226593018,
      "logps/chosen": -1.0544698238372803,
      "logps/rejected": -17.315820693969727,
      "loss": 2.7886,
      "nll_loss": 2.7808361053466797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10544697940349579,
      "rewards/margins": 1.6261351108551025,
      "rewards/rejected": -1.7315820455551147,
      "step": 774
    },
    {
      "epoch": 0.4821150855365474,
      "grad_norm": 0.45079338550567627,
      "learning_rate": 3.0625000000000006e-05,
      "log_odds_chosen": 6.942625045776367,
      "log_odds_ratio": -0.2755284011363983,
      "logits/chosen": 0.1854054182767868,
      "logits/rejected": 0.424145370721817,
      "logps/chosen": -1.04248046875,
      "logps/rejected": -7.667983055114746,
      "loss": 3.4672,
      "nll_loss": 3.4396426677703857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1042480394244194,
      "rewards/margins": 0.6625502705574036,
      "rewards/rejected": -0.7667983174324036,
      "step": 775
    },
    {
      "epoch": 0.4827371695178849,
      "grad_norm": 1.2325921058654785,
      "learning_rate": 3.06e-05,
      "log_odds_chosen": 17.07767105102539,
      "log_odds_ratio": -0.07737934589385986,
      "logits/chosen": 0.209869846701622,
      "logits/rejected": 1.219171404838562,
      "logps/chosen": -1.1802769899368286,
      "logps/rejected": -17.847057342529297,
      "loss": 2.9043,
      "nll_loss": 2.89656138420105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1180276945233345,
      "rewards/margins": 1.6666781902313232,
      "rewards/rejected": -1.7847058773040771,
      "step": 776
    },
    {
      "epoch": 0.4833592534992224,
      "grad_norm": 0.4120030999183655,
      "learning_rate": 3.0575000000000005e-05,
      "log_odds_chosen": 13.921056747436523,
      "log_odds_ratio": -0.22028662264347076,
      "logits/chosen": 0.2784203588962555,
      "logits/rejected": 1.1734328269958496,
      "logps/chosen": -1.1785310506820679,
      "logps/rejected": -14.79477596282959,
      "loss": 3.0951,
      "nll_loss": 3.0730538368225098,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11785309761762619,
      "rewards/margins": 1.3616244792938232,
      "rewards/rejected": -1.4794776439666748,
      "step": 777
    },
    {
      "epoch": 0.4839813374805599,
      "grad_norm": 0.5162467956542969,
      "learning_rate": 3.0550000000000004e-05,
      "log_odds_chosen": 12.977121353149414,
      "log_odds_ratio": -0.20910266041755676,
      "logits/chosen": 0.2095002979040146,
      "logits/rejected": 0.5365079641342163,
      "logps/chosen": -1.314969539642334,
      "logps/rejected": -14.050061225891113,
      "loss": 3.1444,
      "nll_loss": 3.123497486114502,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13149695098400116,
      "rewards/margins": 1.2735092639923096,
      "rewards/rejected": -1.4050061702728271,
      "step": 778
    },
    {
      "epoch": 0.48460342146189733,
      "grad_norm": 0.4763834774494171,
      "learning_rate": 3.0525e-05,
      "log_odds_chosen": 20.971172332763672,
      "log_odds_ratio": -2.3841880647523794e-07,
      "logits/chosen": 0.3068065345287323,
      "logits/rejected": 1.4703824520111084,
      "logps/chosen": -1.2722759246826172,
      "logps/rejected": -21.902528762817383,
      "loss": 3.1538,
      "nll_loss": 3.153752088546753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12722758948802948,
      "rewards/margins": 2.0630252361297607,
      "rewards/rejected": -2.1902527809143066,
      "step": 779
    },
    {
      "epoch": 0.48522550544323484,
      "grad_norm": 0.45172974467277527,
      "learning_rate": 3.05e-05,
      "log_odds_chosen": 11.323856353759766,
      "log_odds_ratio": -0.1366121768951416,
      "logits/chosen": 0.18574324250221252,
      "logits/rejected": 0.7099617719650269,
      "logps/chosen": -0.6922950744628906,
      "logps/rejected": -11.323362350463867,
      "loss": 3.2448,
      "nll_loss": 3.2311220169067383,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06922950595617294,
      "rewards/margins": 1.0631067752838135,
      "rewards/rejected": -1.1323362588882446,
      "step": 780
    },
    {
      "epoch": 0.4858475894245723,
      "grad_norm": 0.32547134160995483,
      "learning_rate": 3.0475000000000002e-05,
      "log_odds_chosen": 13.120780944824219,
      "log_odds_ratio": -0.2054150402545929,
      "logits/chosen": 0.3427540361881256,
      "logits/rejected": 0.6512297987937927,
      "logps/chosen": -1.148818016052246,
      "logps/rejected": -13.904681205749512,
      "loss": 3.6809,
      "nll_loss": 3.6603195667266846,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11488181352615356,
      "rewards/margins": 1.2755863666534424,
      "rewards/rejected": -1.3904681205749512,
      "step": 781
    },
    {
      "epoch": 0.4864696734059098,
      "grad_norm": 0.4283800721168518,
      "learning_rate": 3.045e-05,
      "log_odds_chosen": 15.075250625610352,
      "log_odds_ratio": -0.05576830729842186,
      "logits/chosen": 0.07458168268203735,
      "logits/rejected": 0.8056328296661377,
      "logps/chosen": -1.0403079986572266,
      "logps/rejected": -15.61276912689209,
      "loss": 2.6876,
      "nll_loss": 2.682063341140747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1040308028459549,
      "rewards/margins": 1.4572460651397705,
      "rewards/rejected": -1.561276912689209,
      "step": 782
    },
    {
      "epoch": 0.4870917573872473,
      "grad_norm": 7.5606160163879395,
      "learning_rate": 3.0425000000000004e-05,
      "log_odds_chosen": 15.987709999084473,
      "log_odds_ratio": -0.09875954687595367,
      "logits/chosen": 0.29279085993766785,
      "logits/rejected": 1.0353281497955322,
      "logps/chosen": -1.5241639614105225,
      "logps/rejected": -17.19584846496582,
      "loss": 3.2454,
      "nll_loss": 3.2354772090911865,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15241639316082,
      "rewards/margins": 1.5671684741973877,
      "rewards/rejected": -1.7195848226547241,
      "step": 783
    },
    {
      "epoch": 0.48771384136858476,
      "grad_norm": 0.6330939531326294,
      "learning_rate": 3.04e-05,
      "log_odds_chosen": 7.304858684539795,
      "log_odds_ratio": -0.5831460952758789,
      "logits/chosen": 0.21207121014595032,
      "logits/rejected": 0.46470144391059875,
      "logps/chosen": -1.3569834232330322,
      "logps/rejected": -8.61102294921875,
      "loss": 3.3432,
      "nll_loss": 3.284919261932373,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1356983482837677,
      "rewards/margins": 0.725403904914856,
      "rewards/rejected": -0.8611023426055908,
      "step": 784
    },
    {
      "epoch": 0.48833592534992226,
      "grad_norm": 0.43546679615974426,
      "learning_rate": 3.0375000000000003e-05,
      "log_odds_chosen": 7.85859489440918,
      "log_odds_ratio": -0.39412838220596313,
      "logits/chosen": 0.12173283845186234,
      "logits/rejected": 0.5120882391929626,
      "logps/chosen": -1.2129175662994385,
      "logps/rejected": -8.616832733154297,
      "loss": 3.2616,
      "nll_loss": 3.2221570014953613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12129175662994385,
      "rewards/margins": 0.7403914928436279,
      "rewards/rejected": -0.861683189868927,
      "step": 785
    },
    {
      "epoch": 0.4889580093312597,
      "grad_norm": 0.5060476064682007,
      "learning_rate": 3.035e-05,
      "log_odds_chosen": 8.088102340698242,
      "log_odds_ratio": -0.1689227670431137,
      "logits/chosen": 0.18627770245075226,
      "logits/rejected": 0.8996896743774414,
      "logps/chosen": -1.2733261585235596,
      "logps/rejected": -9.087573051452637,
      "loss": 3.05,
      "nll_loss": 3.0331501960754395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12733261287212372,
      "rewards/margins": 0.7814247012138367,
      "rewards/rejected": -0.908757209777832,
      "step": 786
    },
    {
      "epoch": 0.4895800933125972,
      "grad_norm": 0.5102691054344177,
      "learning_rate": 3.0325000000000002e-05,
      "log_odds_chosen": 9.025958061218262,
      "log_odds_ratio": -0.18867120146751404,
      "logits/chosen": 0.2037818729877472,
      "logits/rejected": 0.6893376708030701,
      "logps/chosen": -1.0546302795410156,
      "logps/rejected": -9.737627983093262,
      "loss": 2.8651,
      "nll_loss": 2.84627366065979,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10546302795410156,
      "rewards/margins": 0.8682998418807983,
      "rewards/rejected": -0.9737628698348999,
      "step": 787
    },
    {
      "epoch": 0.49020217729393467,
      "grad_norm": 0.43984538316726685,
      "learning_rate": 3.03e-05,
      "log_odds_chosen": 10.870232582092285,
      "log_odds_ratio": -0.10683636367321014,
      "logits/chosen": 0.13067655265331268,
      "logits/rejected": 0.9502900838851929,
      "logps/chosen": -0.9612005352973938,
      "logps/rejected": -11.30851936340332,
      "loss": 2.3156,
      "nll_loss": 2.304933547973633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09612004458904266,
      "rewards/margins": 1.0347318649291992,
      "rewards/rejected": -1.1308519840240479,
      "step": 788
    },
    {
      "epoch": 0.4908242612752722,
      "grad_norm": 0.41433024406433105,
      "learning_rate": 3.0275000000000004e-05,
      "log_odds_chosen": 7.59851598739624,
      "log_odds_ratio": -0.12431486696004868,
      "logits/chosen": 0.08263087272644043,
      "logits/rejected": 0.3450954556465149,
      "logps/chosen": -1.0177639722824097,
      "logps/rejected": -8.221351623535156,
      "loss": 3.0721,
      "nll_loss": 3.0597083568573,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10177639871835709,
      "rewards/margins": 0.7203587293624878,
      "rewards/rejected": -0.8221350908279419,
      "step": 789
    },
    {
      "epoch": 0.49144634525660963,
      "grad_norm": 0.480323851108551,
      "learning_rate": 3.025e-05,
      "log_odds_chosen": 3.6505227088928223,
      "log_odds_ratio": -0.4678669273853302,
      "logits/chosen": 0.12035234272480011,
      "logits/rejected": 0.24431809782981873,
      "logps/chosen": -1.2138129472732544,
      "logps/rejected": -4.56633996963501,
      "loss": 2.912,
      "nll_loss": 2.865224599838257,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12138129770755768,
      "rewards/margins": 0.33525270223617554,
      "rewards/rejected": -0.456633985042572,
      "step": 790
    },
    {
      "epoch": 0.49206842923794714,
      "grad_norm": 0.5126906037330627,
      "learning_rate": 3.0225000000000003e-05,
      "log_odds_chosen": 14.473326683044434,
      "log_odds_ratio": -7.871995330788195e-05,
      "logits/chosen": 0.212377667427063,
      "logits/rejected": 1.0169252157211304,
      "logps/chosen": -1.15001380443573,
      "logps/rejected": -15.20506763458252,
      "loss": 3.0556,
      "nll_loss": 3.0555496215820312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.115001380443573,
      "rewards/margins": 1.4055054187774658,
      "rewards/rejected": -1.520506739616394,
      "step": 791
    },
    {
      "epoch": 0.4926905132192846,
      "grad_norm": 0.5260676145553589,
      "learning_rate": 3.02e-05,
      "log_odds_chosen": 9.164556503295898,
      "log_odds_ratio": -0.20497088134288788,
      "logits/chosen": 0.14021986722946167,
      "logits/rejected": 0.6322394609451294,
      "logps/chosen": -1.1137261390686035,
      "logps/rejected": -9.867830276489258,
      "loss": 3.3855,
      "nll_loss": 3.3650379180908203,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11137261986732483,
      "rewards/margins": 0.8754103183746338,
      "rewards/rejected": -0.986782968044281,
      "step": 792
    },
    {
      "epoch": 0.4933125972006221,
      "grad_norm": 1.3812450170516968,
      "learning_rate": 3.0175e-05,
      "log_odds_chosen": 7.82905912399292,
      "log_odds_ratio": -0.5109982490539551,
      "logits/chosen": 0.2852308452129364,
      "logits/rejected": 0.6695021390914917,
      "logps/chosen": -1.8619670867919922,
      "logps/rejected": -9.53437614440918,
      "loss": 3.489,
      "nll_loss": 3.4378857612609863,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1861967146396637,
      "rewards/margins": 0.7672408819198608,
      "rewards/rejected": -0.9534375667572021,
      "step": 793
    },
    {
      "epoch": 0.49393468118195955,
      "grad_norm": 0.47631436586380005,
      "learning_rate": 3.015e-05,
      "log_odds_chosen": 10.75469970703125,
      "log_odds_ratio": -0.24386245012283325,
      "logits/chosen": 0.14563864469528198,
      "logits/rejected": 0.9450300335884094,
      "logps/chosen": -1.1033132076263428,
      "logps/rejected": -11.575775146484375,
      "loss": 2.6249,
      "nll_loss": 2.6005167961120605,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11033132672309875,
      "rewards/margins": 1.0472462177276611,
      "rewards/rejected": -1.1575775146484375,
      "step": 794
    },
    {
      "epoch": 0.49455676516329705,
      "grad_norm": 0.4538606107234955,
      "learning_rate": 3.0125000000000004e-05,
      "log_odds_chosen": 8.877276420593262,
      "log_odds_ratio": -0.19892564415931702,
      "logits/chosen": 0.2505427896976471,
      "logits/rejected": 0.9329760670661926,
      "logps/chosen": -1.0743852853775024,
      "logps/rejected": -9.534322738647461,
      "loss": 2.7392,
      "nll_loss": 2.7193334102630615,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10743853449821472,
      "rewards/margins": 0.8459938764572144,
      "rewards/rejected": -0.9534323215484619,
      "step": 795
    },
    {
      "epoch": 0.4951788491446345,
      "grad_norm": 0.5034440755844116,
      "learning_rate": 3.01e-05,
      "log_odds_chosen": 6.5391740798950195,
      "log_odds_ratio": -0.3063565492630005,
      "logits/chosen": 0.1824992597103119,
      "logits/rejected": 0.32490694522857666,
      "logps/chosen": -1.5990245342254639,
      "logps/rejected": -7.915644645690918,
      "loss": 3.681,
      "nll_loss": 3.6503705978393555,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1599024385213852,
      "rewards/margins": 0.6316620111465454,
      "rewards/rejected": -0.7915644645690918,
      "step": 796
    },
    {
      "epoch": 0.495800933125972,
      "grad_norm": 0.4987301528453827,
      "learning_rate": 3.0075000000000003e-05,
      "log_odds_chosen": 13.378631591796875,
      "log_odds_ratio": -0.15653789043426514,
      "logits/chosen": 0.19882658123970032,
      "logits/rejected": 1.0811190605163574,
      "logps/chosen": -0.9882148504257202,
      "logps/rejected": -13.973001480102539,
      "loss": 2.8765,
      "nll_loss": 2.860856056213379,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0988214910030365,
      "rewards/margins": 1.2984788417816162,
      "rewards/rejected": -1.3973002433776855,
      "step": 797
    },
    {
      "epoch": 0.49642301710730946,
      "grad_norm": 0.35390564799308777,
      "learning_rate": 3.0050000000000002e-05,
      "log_odds_chosen": 11.163237571716309,
      "log_odds_ratio": -0.11993933469057083,
      "logits/chosen": 0.24885781109333038,
      "logits/rejected": 0.7615518569946289,
      "logps/chosen": -1.0514477491378784,
      "logps/rejected": -11.754963874816895,
      "loss": 3.3127,
      "nll_loss": 3.3006787300109863,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10514476895332336,
      "rewards/margins": 1.0703516006469727,
      "rewards/rejected": -1.1754963397979736,
      "step": 798
    },
    {
      "epoch": 0.49704510108864697,
      "grad_norm": 0.46855729818344116,
      "learning_rate": 3.0025000000000005e-05,
      "log_odds_chosen": 14.509871482849121,
      "log_odds_ratio": -0.06569898873567581,
      "logits/chosen": 0.13805624842643738,
      "logits/rejected": 0.8150652050971985,
      "logps/chosen": -1.496653437614441,
      "logps/rejected": -15.731345176696777,
      "loss": 3.0396,
      "nll_loss": 3.0330357551574707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14966535568237305,
      "rewards/margins": 1.4234691858291626,
      "rewards/rejected": -1.573134422302246,
      "step": 799
    },
    {
      "epoch": 0.4976671850699845,
      "grad_norm": 0.418714702129364,
      "learning_rate": 3e-05,
      "log_odds_chosen": 6.016261577606201,
      "log_odds_ratio": -0.35451096296310425,
      "logits/chosen": 0.012198593467473984,
      "logits/rejected": 0.27333053946495056,
      "logps/chosen": -1.07492995262146,
      "logps/rejected": -6.799544334411621,
      "loss": 2.815,
      "nll_loss": 2.7795400619506836,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10749299824237823,
      "rewards/margins": 0.5724614262580872,
      "rewards/rejected": -0.6799544095993042,
      "step": 800
    },
    {
      "epoch": 0.4982892690513219,
      "grad_norm": 0.45310264825820923,
      "learning_rate": 2.9975000000000004e-05,
      "log_odds_chosen": 12.761698722839355,
      "log_odds_ratio": -0.23440547287464142,
      "logits/chosen": 0.07036770135164261,
      "logits/rejected": 0.8573406338691711,
      "logps/chosen": -1.2407668828964233,
      "logps/rejected": -13.764123916625977,
      "loss": 2.4648,
      "nll_loss": 2.4413938522338867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.124076709151268,
      "rewards/margins": 1.252335786819458,
      "rewards/rejected": -1.3764123916625977,
      "step": 801
    },
    {
      "epoch": 0.49891135303265943,
      "grad_norm": 0.47506943345069885,
      "learning_rate": 2.995e-05,
      "log_odds_chosen": 9.560124397277832,
      "log_odds_ratio": -0.45071908831596375,
      "logits/chosen": 0.16752222180366516,
      "logits/rejected": 0.6287195086479187,
      "logps/chosen": -1.2660199403762817,
      "logps/rejected": -10.690040588378906,
      "loss": 2.8457,
      "nll_loss": 2.8006627559661865,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12660199403762817,
      "rewards/margins": 0.942402184009552,
      "rewards/rejected": -1.0690041780471802,
      "step": 802
    },
    {
      "epoch": 0.4995334370139969,
      "grad_norm": 0.47123369574546814,
      "learning_rate": 2.9925000000000002e-05,
      "log_odds_chosen": 7.347402572631836,
      "log_odds_ratio": -0.20708920061588287,
      "logits/chosen": 0.1796862781047821,
      "logits/rejected": 0.46771129965782166,
      "logps/chosen": -0.9793645739555359,
      "logps/rejected": -7.834238052368164,
      "loss": 3.2493,
      "nll_loss": 3.2286105155944824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09793645888566971,
      "rewards/margins": 0.6854873895645142,
      "rewards/rejected": -0.7834238409996033,
      "step": 803
    },
    {
      "epoch": 0.5001555209953343,
      "grad_norm": 0.4224500358104706,
      "learning_rate": 2.9900000000000002e-05,
      "log_odds_chosen": 9.55636215209961,
      "log_odds_ratio": -0.20841999351978302,
      "logits/chosen": 0.25128045678138733,
      "logits/rejected": 0.9328861832618713,
      "logps/chosen": -1.126585602760315,
      "logps/rejected": -10.326457977294922,
      "loss": 3.3699,
      "nll_loss": 3.3491053581237793,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11265856772661209,
      "rewards/margins": 0.9199872612953186,
      "rewards/rejected": -1.03264582157135,
      "step": 804
    },
    {
      "epoch": 0.5007776049766719,
      "grad_norm": 0.6790135502815247,
      "learning_rate": 2.9875000000000004e-05,
      "log_odds_chosen": 10.101418495178223,
      "log_odds_ratio": -0.3796658217906952,
      "logits/chosen": 0.1424853503704071,
      "logits/rejected": 0.5358273386955261,
      "logps/chosen": -1.3166006803512573,
      "logps/rejected": -11.287535667419434,
      "loss": 2.6253,
      "nll_loss": 2.5873122215270996,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1316600739955902,
      "rewards/margins": 0.9970934391021729,
      "rewards/rejected": -1.1287535429000854,
      "step": 805
    },
    {
      "epoch": 0.5013996889580093,
      "grad_norm": 0.5378172993659973,
      "learning_rate": 2.985e-05,
      "log_odds_chosen": 12.793437957763672,
      "log_odds_ratio": -0.16563086211681366,
      "logits/chosen": 0.2660048007965088,
      "logits/rejected": 1.0401332378387451,
      "logps/chosen": -1.2829506397247314,
      "logps/rejected": -13.792176246643066,
      "loss": 2.8793,
      "nll_loss": 2.8627707958221436,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12829506397247314,
      "rewards/margins": 1.250922679901123,
      "rewards/rejected": -1.3792178630828857,
      "step": 806
    },
    {
      "epoch": 0.5020217729393468,
      "grad_norm": 0.44759759306907654,
      "learning_rate": 2.9825000000000003e-05,
      "log_odds_chosen": 8.082733154296875,
      "log_odds_ratio": -0.29205119609832764,
      "logits/chosen": 0.08207601308822632,
      "logits/rejected": 0.7699632048606873,
      "logps/chosen": -1.0160481929779053,
      "logps/rejected": -8.517202377319336,
      "loss": 2.4715,
      "nll_loss": 2.442284107208252,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10160481929779053,
      "rewards/margins": 0.7501154541969299,
      "rewards/rejected": -0.8517202734947205,
      "step": 807
    },
    {
      "epoch": 0.5026438569206843,
      "grad_norm": 12.664875030517578,
      "learning_rate": 2.98e-05,
      "log_odds_chosen": 9.370549201965332,
      "log_odds_ratio": -0.25263848900794983,
      "logits/chosen": 0.18772029876708984,
      "logits/rejected": 0.5844571590423584,
      "logps/chosen": -1.6902470588684082,
      "logps/rejected": -10.821871757507324,
      "loss": 3.2462,
      "nll_loss": 3.220900774002075,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16902472078800201,
      "rewards/margins": 0.9131625294685364,
      "rewards/rejected": -1.0821871757507324,
      "step": 808
    },
    {
      "epoch": 0.5032659409020218,
      "grad_norm": 0.785666823387146,
      "learning_rate": 2.9775000000000002e-05,
      "log_odds_chosen": 11.042867660522461,
      "log_odds_ratio": -0.21754157543182373,
      "logits/chosen": 0.30363738536834717,
      "logits/rejected": 0.7481663227081299,
      "logps/chosen": -1.3917642831802368,
      "logps/rejected": -12.1473388671875,
      "loss": 3.4199,
      "nll_loss": 3.398167133331299,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13917642831802368,
      "rewards/margins": 1.0755574703216553,
      "rewards/rejected": -1.2147338390350342,
      "step": 809
    },
    {
      "epoch": 0.5038880248833593,
      "grad_norm": 0.6624091267585754,
      "learning_rate": 2.975e-05,
      "log_odds_chosen": 12.086601257324219,
      "log_odds_ratio": -0.16491734981536865,
      "logits/chosen": 0.1394706517457962,
      "logits/rejected": 0.6047402620315552,
      "logps/chosen": -1.2884442806243896,
      "logps/rejected": -13.085152626037598,
      "loss": 2.5779,
      "nll_loss": 2.561450481414795,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12884442508220673,
      "rewards/margins": 1.179670810699463,
      "rewards/rejected": -1.3085153102874756,
      "step": 810
    },
    {
      "epoch": 0.5045101088646967,
      "grad_norm": 0.35120460391044617,
      "learning_rate": 2.9725000000000004e-05,
      "log_odds_chosen": 6.397805690765381,
      "log_odds_ratio": -0.39501500129699707,
      "logits/chosen": 0.25109362602233887,
      "logits/rejected": 0.48641854524612427,
      "logps/chosen": -1.0249440670013428,
      "logps/rejected": -7.021490097045898,
      "loss": 3.4616,
      "nll_loss": 3.4220848083496094,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10249441117048264,
      "rewards/margins": 0.5996546149253845,
      "rewards/rejected": -0.7021490335464478,
      "step": 811
    },
    {
      "epoch": 0.5051321928460342,
      "grad_norm": 0.5609656572341919,
      "learning_rate": 2.97e-05,
      "log_odds_chosen": 15.698429107666016,
      "log_odds_ratio": -0.017468160018324852,
      "logits/chosen": 0.3717779815196991,
      "logits/rejected": 1.3647180795669556,
      "logps/chosen": -1.1899399757385254,
      "logps/rejected": -16.393325805664062,
      "loss": 3.1869,
      "nll_loss": 3.1851115226745605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11899399757385254,
      "rewards/margins": 1.520338773727417,
      "rewards/rejected": -1.63933265209198,
      "step": 812
    },
    {
      "epoch": 0.5057542768273717,
      "grad_norm": 0.436008095741272,
      "learning_rate": 2.9675000000000003e-05,
      "log_odds_chosen": 17.379844665527344,
      "log_odds_ratio": -0.048989683389663696,
      "logits/chosen": 0.2973853647708893,
      "logits/rejected": 1.2079492807388306,
      "logps/chosen": -1.2121381759643555,
      "logps/rejected": -18.229488372802734,
      "loss": 3.0875,
      "nll_loss": 3.0826025009155273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12121382355690002,
      "rewards/margins": 1.7017349004745483,
      "rewards/rejected": -1.822948694229126,
      "step": 813
    },
    {
      "epoch": 0.5063763608087092,
      "grad_norm": 0.4556023180484772,
      "learning_rate": 2.965e-05,
      "log_odds_chosen": 17.50226402282715,
      "log_odds_ratio": -0.00011185341281816363,
      "logits/chosen": 0.230424702167511,
      "logits/rejected": 0.8504374027252197,
      "logps/chosen": -1.146848440170288,
      "logps/rejected": -18.064584732055664,
      "loss": 3.1448,
      "nll_loss": 3.1448068618774414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11468484997749329,
      "rewards/margins": 1.6917736530303955,
      "rewards/rejected": -1.8064584732055664,
      "step": 814
    },
    {
      "epoch": 0.5069984447900466,
      "grad_norm": 0.3557608723640442,
      "learning_rate": 2.9625000000000002e-05,
      "log_odds_chosen": 9.623047828674316,
      "log_odds_ratio": -0.2141369879245758,
      "logits/chosen": 0.22592505812644958,
      "logits/rejected": 0.46705248951911926,
      "logps/chosen": -1.2357347011566162,
      "logps/rejected": -10.610895156860352,
      "loss": 3.3321,
      "nll_loss": 3.3106772899627686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12357348203659058,
      "rewards/margins": 0.9375160336494446,
      "rewards/rejected": -1.0610895156860352,
      "step": 815
    },
    {
      "epoch": 0.5076205287713841,
      "grad_norm": 0.4675641655921936,
      "learning_rate": 2.96e-05,
      "log_odds_chosen": 14.062185287475586,
      "log_odds_ratio": -0.10161672532558441,
      "logits/chosen": 0.18339157104492188,
      "logits/rejected": 1.1679893732070923,
      "logps/chosen": -1.142388105392456,
      "logps/rejected": -14.841217994689941,
      "loss": 2.7793,
      "nll_loss": 2.769094705581665,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11423881351947784,
      "rewards/margins": 1.3698830604553223,
      "rewards/rejected": -1.4841217994689941,
      "step": 816
    },
    {
      "epoch": 0.5082426127527216,
      "grad_norm": 0.3435609042644501,
      "learning_rate": 2.9575000000000004e-05,
      "log_odds_chosen": 11.574292182922363,
      "log_odds_ratio": -0.1866357922554016,
      "logits/chosen": 0.3857420086860657,
      "logits/rejected": 1.0843833684921265,
      "logps/chosen": -1.4344037771224976,
      "logps/rejected": -12.822274208068848,
      "loss": 3.5625,
      "nll_loss": 3.543877124786377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1434403955936432,
      "rewards/margins": 1.1387871503829956,
      "rewards/rejected": -1.2822275161743164,
      "step": 817
    },
    {
      "epoch": 0.5088646967340591,
      "grad_norm": 0.463178813457489,
      "learning_rate": 2.955e-05,
      "log_odds_chosen": 11.81800651550293,
      "log_odds_ratio": -0.18851415812969208,
      "logits/chosen": 0.285900741815567,
      "logits/rejected": 1.0203773975372314,
      "logps/chosen": -1.2619681358337402,
      "logps/rejected": -12.845138549804688,
      "loss": 3.0084,
      "nll_loss": 2.9895620346069336,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12619681656360626,
      "rewards/margins": 1.1583170890808105,
      "rewards/rejected": -1.2845139503479004,
      "step": 818
    },
    {
      "epoch": 0.5094867807153965,
      "grad_norm": 0.38956987857818604,
      "learning_rate": 2.9525000000000003e-05,
      "log_odds_chosen": 14.569768905639648,
      "log_odds_ratio": -0.0029117946978658438,
      "logits/chosen": 0.19263651967048645,
      "logits/rejected": 1.1464442014694214,
      "logps/chosen": -0.9850395917892456,
      "logps/rejected": -15.035454750061035,
      "loss": 2.6472,
      "nll_loss": 2.646864891052246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0985039621591568,
      "rewards/margins": 1.4050414562225342,
      "rewards/rejected": -1.5035455226898193,
      "step": 819
    },
    {
      "epoch": 0.5101088646967341,
      "grad_norm": 0.3614479601383209,
      "learning_rate": 2.95e-05,
      "log_odds_chosen": 13.27166748046875,
      "log_odds_ratio": -0.0006157811731100082,
      "logits/chosen": 0.2117461860179901,
      "logits/rejected": 1.3962442874908447,
      "logps/chosen": -1.2849613428115845,
      "logps/rejected": -14.16645622253418,
      "loss": 3.0077,
      "nll_loss": 3.0076475143432617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12849614024162292,
      "rewards/margins": 1.2881494760513306,
      "rewards/rejected": -1.4166457653045654,
      "step": 820
    },
    {
      "epoch": 0.5107309486780716,
      "grad_norm": 0.4717678129673004,
      "learning_rate": 2.9475e-05,
      "log_odds_chosen": 10.178583145141602,
      "log_odds_ratio": -0.13750000298023224,
      "logits/chosen": 0.20223943889141083,
      "logits/rejected": 1.0256826877593994,
      "logps/chosen": -0.9736385345458984,
      "logps/rejected": -10.707265853881836,
      "loss": 2.6019,
      "nll_loss": 2.5881075859069824,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09736385941505432,
      "rewards/margins": 0.9733626842498779,
      "rewards/rejected": -1.0707266330718994,
      "step": 821
    },
    {
      "epoch": 0.511353032659409,
      "grad_norm": 0.37724214792251587,
      "learning_rate": 2.945e-05,
      "log_odds_chosen": 15.004103660583496,
      "log_odds_ratio": -0.0004430253757163882,
      "logits/chosen": 0.3318445086479187,
      "logits/rejected": 0.974646806716919,
      "logps/chosen": -1.1180328130722046,
      "logps/rejected": -15.67713451385498,
      "loss": 3.4453,
      "nll_loss": 3.4452362060546875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11180329322814941,
      "rewards/margins": 1.4559102058410645,
      "rewards/rejected": -1.5677136182785034,
      "step": 822
    },
    {
      "epoch": 0.5119751166407465,
      "grad_norm": 0.4549922049045563,
      "learning_rate": 2.9425000000000004e-05,
      "log_odds_chosen": 15.360553741455078,
      "log_odds_ratio": -0.050239816308021545,
      "logits/chosen": 0.44666236639022827,
      "logits/rejected": 1.4149678945541382,
      "logps/chosen": -1.1860952377319336,
      "logps/rejected": -15.948168754577637,
      "loss": 3.2089,
      "nll_loss": 3.203843593597412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11860951781272888,
      "rewards/margins": 1.4762073755264282,
      "rewards/rejected": -1.5948166847229004,
      "step": 823
    },
    {
      "epoch": 0.512597200622084,
      "grad_norm": 0.535690426826477,
      "learning_rate": 2.94e-05,
      "log_odds_chosen": 8.157598495483398,
      "log_odds_ratio": -0.32862338423728943,
      "logits/chosen": 0.19047969579696655,
      "logits/rejected": 0.8336578011512756,
      "logps/chosen": -1.151352882385254,
      "logps/rejected": -8.862951278686523,
      "loss": 2.6731,
      "nll_loss": 2.6401925086975098,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11513528972864151,
      "rewards/margins": 0.771159827709198,
      "rewards/rejected": -0.8862950801849365,
      "step": 824
    },
    {
      "epoch": 0.5132192846034215,
      "grad_norm": 0.38767126202583313,
      "learning_rate": 2.9375000000000003e-05,
      "log_odds_chosen": 8.195636749267578,
      "log_odds_ratio": -0.2015450894832611,
      "logits/chosen": 0.21387703716754913,
      "logits/rejected": 0.6429730653762817,
      "logps/chosen": -1.1645689010620117,
      "logps/rejected": -8.997748374938965,
      "loss": 3.2326,
      "nll_loss": 3.212407112121582,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11645688116550446,
      "rewards/margins": 0.7833179831504822,
      "rewards/rejected": -0.8997748494148254,
      "step": 825
    },
    {
      "epoch": 0.5138413685847589,
      "grad_norm": 0.5092079043388367,
      "learning_rate": 2.935e-05,
      "log_odds_chosen": 5.449245929718018,
      "log_odds_ratio": -0.11578687280416489,
      "logits/chosen": 0.24035300314426422,
      "logits/rejected": 0.5837621092796326,
      "logps/chosen": -1.1935652494430542,
      "logps/rejected": -6.307559490203857,
      "loss": 3.0066,
      "nll_loss": 2.9950246810913086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11935652047395706,
      "rewards/margins": 0.5113994479179382,
      "rewards/rejected": -0.6307559013366699,
      "step": 826
    },
    {
      "epoch": 0.5144634525660964,
      "grad_norm": 0.6121360063552856,
      "learning_rate": 2.9325e-05,
      "log_odds_chosen": 8.71769905090332,
      "log_odds_ratio": -0.22162342071533203,
      "logits/chosen": 0.2978130578994751,
      "logits/rejected": 0.6275981664657593,
      "logps/chosen": -1.168921709060669,
      "logps/rejected": -9.59075927734375,
      "loss": 3.1936,
      "nll_loss": 3.171393871307373,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11689215898513794,
      "rewards/margins": 0.8421837687492371,
      "rewards/rejected": -0.959075927734375,
      "step": 827
    },
    {
      "epoch": 0.5150855365474339,
      "grad_norm": 0.4831790626049042,
      "learning_rate": 2.93e-05,
      "log_odds_chosen": 14.293399810791016,
      "log_odds_ratio": -0.09568853676319122,
      "logits/chosen": 0.3437972664833069,
      "logits/rejected": 1.154407262802124,
      "logps/chosen": -1.0440468788146973,
      "logps/rejected": -14.955587387084961,
      "loss": 3.0785,
      "nll_loss": 3.068910837173462,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10440470278263092,
      "rewards/margins": 1.3911540508270264,
      "rewards/rejected": -1.495558500289917,
      "step": 828
    },
    {
      "epoch": 0.5157076205287714,
      "grad_norm": 0.8993392586708069,
      "learning_rate": 2.9275000000000003e-05,
      "log_odds_chosen": 12.711183547973633,
      "log_odds_ratio": -0.09046018123626709,
      "logits/chosen": 0.41627803444862366,
      "logits/rejected": 1.3701362609863281,
      "logps/chosen": -1.0129324197769165,
      "logps/rejected": -13.203245162963867,
      "loss": 2.6354,
      "nll_loss": 2.626333236694336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10129325091838837,
      "rewards/margins": 1.2190312147140503,
      "rewards/rejected": -1.3203246593475342,
      "step": 829
    },
    {
      "epoch": 0.5163297045101088,
      "grad_norm": 0.5971187949180603,
      "learning_rate": 2.925e-05,
      "log_odds_chosen": 7.836894989013672,
      "log_odds_ratio": -0.3798048496246338,
      "logits/chosen": 0.3004685938358307,
      "logits/rejected": 0.724410355091095,
      "logps/chosen": -1.366215705871582,
      "logps/rejected": -9.068084716796875,
      "loss": 3.2621,
      "nll_loss": 3.224073886871338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13662157952785492,
      "rewards/margins": 0.7701869606971741,
      "rewards/rejected": -0.9068085551261902,
      "step": 830
    },
    {
      "epoch": 0.5169517884914463,
      "grad_norm": 0.36452698707580566,
      "learning_rate": 2.9225000000000002e-05,
      "log_odds_chosen": 15.304241180419922,
      "log_odds_ratio": -0.0003950538521166891,
      "logits/chosen": 0.3968985080718994,
      "logits/rejected": 1.0415889024734497,
      "logps/chosen": -1.0593630075454712,
      "logps/rejected": -15.823348999023438,
      "loss": 3.9988,
      "nll_loss": 3.9988064765930176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10593629628419876,
      "rewards/margins": 1.4763985872268677,
      "rewards/rejected": -1.5823347568511963,
      "step": 831
    },
    {
      "epoch": 0.5175738724727839,
      "grad_norm": 1.9738948345184326,
      "learning_rate": 2.9199999999999998e-05,
      "log_odds_chosen": 12.84214973449707,
      "log_odds_ratio": -0.1361134946346283,
      "logits/chosen": 0.08501138538122177,
      "logits/rejected": 0.4218248426914215,
      "logps/chosen": -1.936566710472107,
      "logps/rejected": -14.241930961608887,
      "loss": 3.0152,
      "nll_loss": 3.0015807151794434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19365663826465607,
      "rewards/margins": 1.2305364608764648,
      "rewards/rejected": -1.4241931438446045,
      "step": 832
    },
    {
      "epoch": 0.5181959564541213,
      "grad_norm": 0.4944245219230652,
      "learning_rate": 2.9175e-05,
      "log_odds_chosen": 20.146743774414062,
      "log_odds_ratio": -9.030426554090809e-06,
      "logits/chosen": 0.2090873271226883,
      "logits/rejected": 1.6611508131027222,
      "logps/chosen": -1.001164197921753,
      "logps/rejected": -20.59023094177246,
      "loss": 2.5188,
      "nll_loss": 2.5187838077545166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10011641681194305,
      "rewards/margins": 1.958906650543213,
      "rewards/rejected": -2.059023141860962,
      "step": 833
    },
    {
      "epoch": 0.5188180404354588,
      "grad_norm": 1.0386806726455688,
      "learning_rate": 2.915e-05,
      "log_odds_chosen": 10.240373611450195,
      "log_odds_ratio": -0.17523886263370514,
      "logits/chosen": 0.2517136335372925,
      "logits/rejected": 0.601931095123291,
      "logps/chosen": -1.1842358112335205,
      "logps/rejected": -11.045554161071777,
      "loss": 3.3807,
      "nll_loss": 3.363182544708252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11842358112335205,
      "rewards/margins": 0.9861319661140442,
      "rewards/rejected": -1.1045554876327515,
      "step": 834
    },
    {
      "epoch": 0.5194401244167963,
      "grad_norm": 4.311429023742676,
      "learning_rate": 2.9125000000000003e-05,
      "log_odds_chosen": 12.021062850952148,
      "log_odds_ratio": -0.011027473025023937,
      "logits/chosen": 0.22837401926517487,
      "logits/rejected": 0.7856446504592896,
      "logps/chosen": -1.6762373447418213,
      "logps/rejected": -13.32023811340332,
      "loss": 2.9735,
      "nll_loss": 2.9724273681640625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16762374341487885,
      "rewards/margins": 1.1644001007080078,
      "rewards/rejected": -1.3320238590240479,
      "step": 835
    },
    {
      "epoch": 0.5200622083981338,
      "grad_norm": 1.8414102792739868,
      "learning_rate": 2.91e-05,
      "log_odds_chosen": 13.802828788757324,
      "log_odds_ratio": -0.06723160296678543,
      "logits/chosen": 0.19973208010196686,
      "logits/rejected": 1.1031876802444458,
      "logps/chosen": -1.8014975786209106,
      "logps/rejected": -15.22963809967041,
      "loss": 2.9892,
      "nll_loss": 2.982433319091797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18014976382255554,
      "rewards/margins": 1.3428139686584473,
      "rewards/rejected": -1.5229637622833252,
      "step": 836
    },
    {
      "epoch": 0.5206842923794712,
      "grad_norm": 0.47048988938331604,
      "learning_rate": 2.9075000000000002e-05,
      "log_odds_chosen": 16.698774337768555,
      "log_odds_ratio": -1.1175909548910568e-06,
      "logits/chosen": 0.33147627115249634,
      "logits/rejected": 1.3666659593582153,
      "logps/chosen": -1.1629831790924072,
      "logps/rejected": -17.404766082763672,
      "loss": 3.2272,
      "nll_loss": 3.2272276878356934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11629832535982132,
      "rewards/margins": 1.624178171157837,
      "rewards/rejected": -1.7404766082763672,
      "step": 837
    },
    {
      "epoch": 0.5213063763608087,
      "grad_norm": 1.6325106620788574,
      "learning_rate": 2.9049999999999998e-05,
      "log_odds_chosen": 4.975150108337402,
      "log_odds_ratio": -0.32594966888427734,
      "logits/chosen": 0.16253390908241272,
      "logits/rejected": 0.3543585538864136,
      "logps/chosen": -1.0561513900756836,
      "logps/rejected": -5.589508056640625,
      "loss": 3.1341,
      "nll_loss": 3.1014583110809326,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10561515390872955,
      "rewards/margins": 0.453335702419281,
      "rewards/rejected": -0.5589507818222046,
      "step": 838
    },
    {
      "epoch": 0.5219284603421462,
      "grad_norm": 0.4523632228374481,
      "learning_rate": 2.9025e-05,
      "log_odds_chosen": 13.034244537353516,
      "log_odds_ratio": -0.0004373805713839829,
      "logits/chosen": 0.23802269995212555,
      "logits/rejected": 1.02839994430542,
      "logps/chosen": -1.0669258832931519,
      "logps/rejected": -13.587167739868164,
      "loss": 2.9686,
      "nll_loss": 2.968569755554199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1066925898194313,
      "rewards/margins": 1.2520242929458618,
      "rewards/rejected": -1.3587168455123901,
      "step": 839
    },
    {
      "epoch": 0.5225505443234837,
      "grad_norm": 0.46523481607437134,
      "learning_rate": 2.9e-05,
      "log_odds_chosen": 5.310853958129883,
      "log_odds_ratio": -0.19879606366157532,
      "logits/chosen": 0.12425953149795532,
      "logits/rejected": 0.24234220385551453,
      "logps/chosen": -0.9893929362297058,
      "logps/rejected": -5.892968654632568,
      "loss": 3.0965,
      "nll_loss": 3.0765981674194336,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09893929213285446,
      "rewards/margins": 0.49035757780075073,
      "rewards/rejected": -0.5892968773841858,
      "step": 840
    },
    {
      "epoch": 0.5231726283048211,
      "grad_norm": 0.43103811144828796,
      "learning_rate": 2.8975000000000003e-05,
      "log_odds_chosen": 7.345240592956543,
      "log_odds_ratio": -0.3413081169128418,
      "logits/chosen": 0.015453029423952103,
      "logits/rejected": 0.28171202540397644,
      "logps/chosen": -0.8644816279411316,
      "logps/rejected": -7.795204162597656,
      "loss": 2.733,
      "nll_loss": 2.6988906860351562,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08644817024469376,
      "rewards/margins": 0.6930723190307617,
      "rewards/rejected": -0.7795204520225525,
      "step": 841
    },
    {
      "epoch": 0.5237947122861586,
      "grad_norm": 0.4897030293941498,
      "learning_rate": 2.895e-05,
      "log_odds_chosen": 4.109506607055664,
      "log_odds_ratio": -0.5368872284889221,
      "logits/chosen": 0.2146613895893097,
      "logits/rejected": 0.4014909565448761,
      "logps/chosen": -1.3654534816741943,
      "logps/rejected": -5.355642318725586,
      "loss": 3.0509,
      "nll_loss": 2.997218132019043,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1365453451871872,
      "rewards/margins": 0.39901888370513916,
      "rewards/rejected": -0.5355641841888428,
      "step": 842
    },
    {
      "epoch": 0.5244167962674962,
      "grad_norm": 0.47838544845581055,
      "learning_rate": 2.8925000000000002e-05,
      "log_odds_chosen": 2.614036798477173,
      "log_odds_ratio": -0.4196464419364929,
      "logits/chosen": 0.04910755902528763,
      "logits/rejected": 0.2252475917339325,
      "logps/chosen": -1.306127667427063,
      "logps/rejected": -3.7415530681610107,
      "loss": 2.6839,
      "nll_loss": 2.641903877258301,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13061276078224182,
      "rewards/margins": 0.24354252219200134,
      "rewards/rejected": -0.37415528297424316,
      "step": 843
    },
    {
      "epoch": 0.5250388802488336,
      "grad_norm": 0.329008549451828,
      "learning_rate": 2.8899999999999998e-05,
      "log_odds_chosen": 4.632389068603516,
      "log_odds_ratio": -0.3140389621257782,
      "logits/chosen": 0.2015816867351532,
      "logits/rejected": 0.3459731340408325,
      "logps/chosen": -1.283074975013733,
      "logps/rejected": -5.729971408843994,
      "loss": 3.4623,
      "nll_loss": 3.430915355682373,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1283074915409088,
      "rewards/margins": 0.44468966126441956,
      "rewards/rejected": -0.5729971528053284,
      "step": 844
    },
    {
      "epoch": 0.5256609642301711,
      "grad_norm": 0.43985334038734436,
      "learning_rate": 2.8875e-05,
      "log_odds_chosen": 1.6101486682891846,
      "log_odds_ratio": -0.4912552237510681,
      "logits/chosen": 0.13369233906269073,
      "logits/rejected": 0.1671290248632431,
      "logps/chosen": -1.0167124271392822,
      "logps/rejected": -2.167614698410034,
      "loss": 2.8231,
      "nll_loss": 2.7740092277526855,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1016712337732315,
      "rewards/margins": 0.11509023606777191,
      "rewards/rejected": -0.2167614847421646,
      "step": 845
    },
    {
      "epoch": 0.5262830482115085,
      "grad_norm": 0.4271375834941864,
      "learning_rate": 2.885e-05,
      "log_odds_chosen": 7.256542205810547,
      "log_odds_ratio": -0.037149470299482346,
      "logits/chosen": 0.24337813258171082,
      "logits/rejected": 0.6470229625701904,
      "logps/chosen": -0.8154711723327637,
      "logps/rejected": -7.411262512207031,
      "loss": 3.2228,
      "nll_loss": 3.219062328338623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08154712617397308,
      "rewards/margins": 0.6595790386199951,
      "rewards/rejected": -0.7411262392997742,
      "step": 846
    },
    {
      "epoch": 0.5269051321928461,
      "grad_norm": 1.1752288341522217,
      "learning_rate": 2.8825000000000003e-05,
      "log_odds_chosen": 5.8892974853515625,
      "log_odds_ratio": -0.18083639442920685,
      "logits/chosen": 0.22460246086120605,
      "logits/rejected": 0.6375922560691833,
      "logps/chosen": -1.37664794921875,
      "logps/rejected": -7.0137481689453125,
      "loss": 3.0231,
      "nll_loss": 3.0050535202026367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1376648098230362,
      "rewards/margins": 0.5637100338935852,
      "rewards/rejected": -0.701374888420105,
      "step": 847
    },
    {
      "epoch": 0.5275272161741835,
      "grad_norm": 0.3938504457473755,
      "learning_rate": 2.88e-05,
      "log_odds_chosen": 6.437436580657959,
      "log_odds_ratio": -0.1518944650888443,
      "logits/chosen": 0.2168705314397812,
      "logits/rejected": 0.40676745772361755,
      "logps/chosen": -1.264991283416748,
      "logps/rejected": -7.434615135192871,
      "loss": 3.5069,
      "nll_loss": 3.4916627407073975,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12649914622306824,
      "rewards/margins": 0.6169624328613281,
      "rewards/rejected": -0.7434616088867188,
      "step": 848
    },
    {
      "epoch": 0.528149300155521,
      "grad_norm": 0.36738061904907227,
      "learning_rate": 2.8775e-05,
      "log_odds_chosen": 4.217384338378906,
      "log_odds_ratio": -0.19647136330604553,
      "logits/chosen": 0.1279819905757904,
      "logits/rejected": 0.30928248167037964,
      "logps/chosen": -1.111844539642334,
      "logps/rejected": -4.994659423828125,
      "loss": 3.0897,
      "nll_loss": 3.0701029300689697,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11118445545434952,
      "rewards/margins": 0.3882814049720764,
      "rewards/rejected": -0.4994658827781677,
      "step": 849
    },
    {
      "epoch": 0.5287713841368584,
      "grad_norm": 0.49180299043655396,
      "learning_rate": 2.8749999999999997e-05,
      "log_odds_chosen": 3.4940242767333984,
      "log_odds_ratio": -0.29053792357444763,
      "logits/chosen": 0.3198105990886688,
      "logits/rejected": 0.5212329030036926,
      "logps/chosen": -1.107250452041626,
      "logps/rejected": -4.281674385070801,
      "loss": 3.3457,
      "nll_loss": 3.3166162967681885,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1107250452041626,
      "rewards/margins": 0.317442387342453,
      "rewards/rejected": -0.4281674325466156,
      "step": 850
    },
    {
      "epoch": 0.529393468118196,
      "grad_norm": 0.5136086940765381,
      "learning_rate": 2.8725e-05,
      "log_odds_chosen": 4.076441764831543,
      "log_odds_ratio": -0.2946397066116333,
      "logits/chosen": -0.006016634404659271,
      "logits/rejected": 0.35069453716278076,
      "logps/chosen": -1.2343401908874512,
      "logps/rejected": -5.004025936126709,
      "loss": 2.3004,
      "nll_loss": 2.270923376083374,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12343402206897736,
      "rewards/margins": 0.3769685626029968,
      "rewards/rejected": -0.500402569770813,
      "step": 851
    },
    {
      "epoch": 0.5300155520995334,
      "grad_norm": 0.42883893847465515,
      "learning_rate": 2.87e-05,
      "log_odds_chosen": 2.978586196899414,
      "log_odds_ratio": -0.30816155672073364,
      "logits/chosen": 0.10813058167695999,
      "logits/rejected": 0.12170670181512833,
      "logps/chosen": -0.927642822265625,
      "logps/rejected": -3.484469413757324,
      "loss": 3.1087,
      "nll_loss": 3.0778818130493164,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09276428818702698,
      "rewards/margins": 0.25568264722824097,
      "rewards/rejected": -0.34844693541526794,
      "step": 852
    },
    {
      "epoch": 0.5306376360808709,
      "grad_norm": 0.4703337550163269,
      "learning_rate": 2.8675000000000002e-05,
      "log_odds_chosen": 4.520643711090088,
      "log_odds_ratio": -0.3384997248649597,
      "logits/chosen": 0.1783015877008438,
      "logits/rejected": 0.42561283707618713,
      "logps/chosen": -1.0047498941421509,
      "logps/rejected": -5.196669101715088,
      "loss": 2.9576,
      "nll_loss": 2.923743486404419,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10047499090433121,
      "rewards/margins": 0.41919195652008057,
      "rewards/rejected": -0.5196669101715088,
      "step": 853
    },
    {
      "epoch": 0.5312597200622085,
      "grad_norm": 0.5501247644424438,
      "learning_rate": 2.865e-05,
      "log_odds_chosen": 3.5761749744415283,
      "log_odds_ratio": -0.1852397471666336,
      "logits/chosen": 0.2851634919643402,
      "logits/rejected": 0.3833601772785187,
      "logps/chosen": -1.2000057697296143,
      "logps/rejected": -4.437448024749756,
      "loss": 3.2604,
      "nll_loss": 3.241886615753174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12000058591365814,
      "rewards/margins": 0.3237442076206207,
      "rewards/rejected": -0.44374483823776245,
      "step": 854
    },
    {
      "epoch": 0.5318818040435459,
      "grad_norm": 0.4883931577205658,
      "learning_rate": 2.8625e-05,
      "log_odds_chosen": 4.188264846801758,
      "log_odds_ratio": -0.2911498546600342,
      "logits/chosen": 0.11234842240810394,
      "logits/rejected": 0.4352606236934662,
      "logps/chosen": -1.0508315563201904,
      "logps/rejected": -4.962338447570801,
      "loss": 2.8563,
      "nll_loss": 2.827171564102173,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1050831526517868,
      "rewards/margins": 0.39115068316459656,
      "rewards/rejected": -0.49623382091522217,
      "step": 855
    },
    {
      "epoch": 0.5325038880248834,
      "grad_norm": 0.523665189743042,
      "learning_rate": 2.86e-05,
      "log_odds_chosen": 5.883430004119873,
      "log_odds_ratio": -0.3124341666698456,
      "logits/chosen": 0.18846668303012848,
      "logits/rejected": 0.35798412561416626,
      "logps/chosen": -0.9777987003326416,
      "logps/rejected": -6.380369663238525,
      "loss": 3.1017,
      "nll_loss": 3.070420026779175,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09777987003326416,
      "rewards/margins": 0.5402571558952332,
      "rewards/rejected": -0.6380370259284973,
      "step": 856
    },
    {
      "epoch": 0.5331259720062208,
      "grad_norm": 0.4752536714076996,
      "learning_rate": 2.8575000000000003e-05,
      "log_odds_chosen": 4.861764907836914,
      "log_odds_ratio": -0.13584004342556,
      "logits/chosen": 0.29713207483291626,
      "logits/rejected": 0.5332407355308533,
      "logps/chosen": -1.4573677778244019,
      "logps/rejected": -6.071244239807129,
      "loss": 3.228,
      "nll_loss": 3.2143874168395996,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14573678374290466,
      "rewards/margins": 0.46138763427734375,
      "rewards/rejected": -0.6071244478225708,
      "step": 857
    },
    {
      "epoch": 0.5337480559875584,
      "grad_norm": 0.3933459222316742,
      "learning_rate": 2.855e-05,
      "log_odds_chosen": 5.766116142272949,
      "log_odds_ratio": -0.2710520327091217,
      "logits/chosen": 0.2104707956314087,
      "logits/rejected": 0.3261168599128723,
      "logps/chosen": -0.9775829315185547,
      "logps/rejected": -6.416999816894531,
      "loss": 3.2929,
      "nll_loss": 3.2657809257507324,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09775829315185547,
      "rewards/margins": 0.5439417362213135,
      "rewards/rejected": -0.6416999697685242,
      "step": 858
    },
    {
      "epoch": 0.5343701399688958,
      "grad_norm": 0.5813342332839966,
      "learning_rate": 2.8525000000000002e-05,
      "log_odds_chosen": 4.442496299743652,
      "log_odds_ratio": -0.17784219980239868,
      "logits/chosen": 0.24876517057418823,
      "logits/rejected": 0.58121657371521,
      "logps/chosen": -1.5985615253448486,
      "logps/rejected": -5.860179901123047,
      "loss": 3.1737,
      "nll_loss": 3.155898094177246,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1598561555147171,
      "rewards/margins": 0.42616188526153564,
      "rewards/rejected": -0.5860180258750916,
      "step": 859
    },
    {
      "epoch": 0.5349922239502333,
      "grad_norm": 0.4372238218784332,
      "learning_rate": 2.8499999999999998e-05,
      "log_odds_chosen": 5.981117248535156,
      "log_odds_ratio": -0.1759411096572876,
      "logits/chosen": 0.1712474822998047,
      "logits/rejected": 0.4489181637763977,
      "logps/chosen": -1.0640121698379517,
      "logps/rejected": -6.628488540649414,
      "loss": 3.1263,
      "nll_loss": 3.1087098121643066,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.106401227414608,
      "rewards/margins": 0.5564476251602173,
      "rewards/rejected": -0.6628488302230835,
      "step": 860
    },
    {
      "epoch": 0.5356143079315707,
      "grad_norm": 0.5723143815994263,
      "learning_rate": 2.8475e-05,
      "log_odds_chosen": 3.2769057750701904,
      "log_odds_ratio": -0.3183494210243225,
      "logits/chosen": 0.15005794167518616,
      "logits/rejected": 0.4674699604511261,
      "logps/chosen": -1.1343958377838135,
      "logps/rejected": -4.155270576477051,
      "loss": 2.7241,
      "nll_loss": 2.6922266483306885,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11343959718942642,
      "rewards/margins": 0.3020874857902527,
      "rewards/rejected": -0.4155270755290985,
      "step": 861
    },
    {
      "epoch": 0.5362363919129083,
      "grad_norm": 0.3801248073577881,
      "learning_rate": 2.845e-05,
      "log_odds_chosen": 5.401886940002441,
      "log_odds_ratio": -0.1325007528066635,
      "logits/chosen": 0.2797131836414337,
      "logits/rejected": 0.7725595235824585,
      "logps/chosen": -1.2554233074188232,
      "logps/rejected": -6.341432094573975,
      "loss": 3.4174,
      "nll_loss": 3.4041857719421387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12554234266281128,
      "rewards/margins": 0.5086008906364441,
      "rewards/rejected": -0.6341432332992554,
      "step": 862
    },
    {
      "epoch": 0.5368584758942457,
      "grad_norm": 0.45833510160446167,
      "learning_rate": 2.8425000000000003e-05,
      "log_odds_chosen": 7.4684834480285645,
      "log_odds_ratio": -0.2111528515815735,
      "logits/chosen": 0.3720274269580841,
      "logits/rejected": 1.0443079471588135,
      "logps/chosen": -1.074413776397705,
      "logps/rejected": -8.263851165771484,
      "loss": 3.4695,
      "nll_loss": 3.448345184326172,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10744138062000275,
      "rewards/margins": 0.7189437747001648,
      "rewards/rejected": -0.8263851404190063,
      "step": 863
    },
    {
      "epoch": 0.5374805598755832,
      "grad_norm": 0.5950579047203064,
      "learning_rate": 2.84e-05,
      "log_odds_chosen": 6.653730392456055,
      "log_odds_ratio": -0.2213490754365921,
      "logits/chosen": 0.1977083534002304,
      "logits/rejected": 0.6113790273666382,
      "logps/chosen": -1.0264054536819458,
      "logps/rejected": -7.271251678466797,
      "loss": 2.6901,
      "nll_loss": 2.667936086654663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1026405543088913,
      "rewards/margins": 0.624484658241272,
      "rewards/rejected": -0.7271252274513245,
      "step": 864
    },
    {
      "epoch": 0.5381026438569206,
      "grad_norm": 0.5189838409423828,
      "learning_rate": 2.8375000000000002e-05,
      "log_odds_chosen": 4.811558246612549,
      "log_odds_ratio": -0.20530752837657928,
      "logits/chosen": 0.29856374859809875,
      "logits/rejected": 0.7025430202484131,
      "logps/chosen": -1.3187425136566162,
      "logps/rejected": -5.89030122756958,
      "loss": 3.147,
      "nll_loss": 3.1264843940734863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13187426328659058,
      "rewards/margins": 0.45715591311454773,
      "rewards/rejected": -0.5890301465988159,
      "step": 865
    },
    {
      "epoch": 0.5387247278382582,
      "grad_norm": 0.5219860076904297,
      "learning_rate": 2.8349999999999998e-05,
      "log_odds_chosen": 10.06355094909668,
      "log_odds_ratio": -0.06661475449800491,
      "logits/chosen": 0.16469168663024902,
      "logits/rejected": 0.5855368971824646,
      "logps/chosen": -0.9745451807975769,
      "logps/rejected": -10.555267333984375,
      "loss": 3.0044,
      "nll_loss": 2.997767448425293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09745452553033829,
      "rewards/margins": 0.9580721855163574,
      "rewards/rejected": -1.0555267333984375,
      "step": 866
    },
    {
      "epoch": 0.5393468118195957,
      "grad_norm": 0.40929824113845825,
      "learning_rate": 2.8325e-05,
      "log_odds_chosen": 11.310032844543457,
      "log_odds_ratio": -0.1569271832704544,
      "logits/chosen": 0.28702273964881897,
      "logits/rejected": 0.9445208311080933,
      "logps/chosen": -1.1780142784118652,
      "logps/rejected": -12.161791801452637,
      "loss": 3.249,
      "nll_loss": 3.2333545684814453,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11780144274234772,
      "rewards/margins": 1.0983777046203613,
      "rewards/rejected": -1.2161791324615479,
      "step": 867
    },
    {
      "epoch": 0.5399688958009331,
      "grad_norm": 0.5718206763267517,
      "learning_rate": 2.83e-05,
      "log_odds_chosen": 9.48852252960205,
      "log_odds_ratio": -0.23974888026714325,
      "logits/chosen": 0.20256325602531433,
      "logits/rejected": 0.7612900733947754,
      "logps/chosen": -1.1071330308914185,
      "logps/rejected": -10.291450500488281,
      "loss": 2.7763,
      "nll_loss": 2.752324104309082,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11071330308914185,
      "rewards/margins": 0.91843181848526,
      "rewards/rejected": -1.0291451215744019,
      "step": 868
    },
    {
      "epoch": 0.5405909797822706,
      "grad_norm": 0.5120429396629333,
      "learning_rate": 2.8275000000000003e-05,
      "log_odds_chosen": 7.586127281188965,
      "log_odds_ratio": -0.3463227152824402,
      "logits/chosen": 0.20187543332576752,
      "logits/rejected": 0.6514379382133484,
      "logps/chosen": -1.0916359424591064,
      "logps/rejected": -8.361661911010742,
      "loss": 3.122,
      "nll_loss": 3.087322235107422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10916360467672348,
      "rewards/margins": 0.7270025014877319,
      "rewards/rejected": -0.8361661434173584,
      "step": 869
    },
    {
      "epoch": 0.5412130637636081,
      "grad_norm": 0.5298638939857483,
      "learning_rate": 2.825e-05,
      "log_odds_chosen": 10.231219291687012,
      "log_odds_ratio": -0.11733568459749222,
      "logits/chosen": 0.16062955558300018,
      "logits/rejected": 0.6653136610984802,
      "logps/chosen": -1.0218132734298706,
      "logps/rejected": -10.77658748626709,
      "loss": 2.9773,
      "nll_loss": 2.9655981063842773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1021813303232193,
      "rewards/margins": 0.9754774570465088,
      "rewards/rejected": -1.0776586532592773,
      "step": 870
    },
    {
      "epoch": 0.5418351477449456,
      "grad_norm": 0.629223644733429,
      "learning_rate": 2.8225e-05,
      "log_odds_chosen": 12.822935104370117,
      "log_odds_ratio": -0.16422484815120697,
      "logits/chosen": 0.18336540460586548,
      "logits/rejected": 1.1963590383529663,
      "logps/chosen": -1.1354261636734009,
      "logps/rejected": -13.514695167541504,
      "loss": 2.4868,
      "nll_loss": 2.4704039096832275,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11354262381792068,
      "rewards/margins": 1.2379268407821655,
      "rewards/rejected": -1.3514695167541504,
      "step": 871
    },
    {
      "epoch": 0.542457231726283,
      "grad_norm": 1.4472790956497192,
      "learning_rate": 2.8199999999999998e-05,
      "log_odds_chosen": 15.860309600830078,
      "log_odds_ratio": -0.14200741052627563,
      "logits/chosen": 0.1716628074645996,
      "logits/rejected": 0.962594211101532,
      "logps/chosen": -1.1267319917678833,
      "logps/rejected": -16.692829132080078,
      "loss": 2.6449,
      "nll_loss": 2.630686044692993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11267320811748505,
      "rewards/margins": 1.5566097497940063,
      "rewards/rejected": -1.6692829132080078,
      "step": 872
    },
    {
      "epoch": 0.5430793157076206,
      "grad_norm": 0.4921049475669861,
      "learning_rate": 2.8175e-05,
      "log_odds_chosen": 16.460073471069336,
      "log_odds_ratio": -0.06028711423277855,
      "logits/chosen": 0.33112722635269165,
      "logits/rejected": 1.0481317043304443,
      "logps/chosen": -1.0758942365646362,
      "logps/rejected": -16.93272590637207,
      "loss": 3.3084,
      "nll_loss": 3.30238676071167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10758942365646362,
      "rewards/margins": 1.5856831073760986,
      "rewards/rejected": -1.693272590637207,
      "step": 873
    },
    {
      "epoch": 0.543701399688958,
      "grad_norm": 0.47205042839050293,
      "learning_rate": 2.815e-05,
      "log_odds_chosen": 10.865059852600098,
      "log_odds_ratio": -0.231788769364357,
      "logits/chosen": 0.28823232650756836,
      "logits/rejected": 1.0439519882202148,
      "logps/chosen": -1.0718377828598022,
      "logps/rejected": -11.658307075500488,
      "loss": 3.0113,
      "nll_loss": 2.988163948059082,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1071837842464447,
      "rewards/margins": 1.0586470365524292,
      "rewards/rejected": -1.1658308506011963,
      "step": 874
    },
    {
      "epoch": 0.5443234836702955,
      "grad_norm": 0.370347797870636,
      "learning_rate": 2.8125000000000003e-05,
      "log_odds_chosen": 10.927844047546387,
      "log_odds_ratio": -0.1950116902589798,
      "logits/chosen": 0.17625781893730164,
      "logits/rejected": 0.5121840238571167,
      "logps/chosen": -0.975417971611023,
      "logps/rejected": -11.16118049621582,
      "loss": 3.09,
      "nll_loss": 3.0705158710479736,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09754179418087006,
      "rewards/margins": 1.0185761451721191,
      "rewards/rejected": -1.1161179542541504,
      "step": 875
    },
    {
      "epoch": 0.5449455676516329,
      "grad_norm": 0.4597725570201874,
      "learning_rate": 2.8100000000000005e-05,
      "log_odds_chosen": 9.441105842590332,
      "log_odds_ratio": -0.18627074360847473,
      "logits/chosen": 0.2629444897174835,
      "logits/rejected": 0.7703899145126343,
      "logps/chosen": -1.3209762573242188,
      "logps/rejected": -10.505077362060547,
      "loss": 3.265,
      "nll_loss": 3.2463271617889404,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13209763169288635,
      "rewards/margins": 0.9184101223945618,
      "rewards/rejected": -1.0505077838897705,
      "step": 876
    },
    {
      "epoch": 0.5455676516329705,
      "grad_norm": 0.3739025890827179,
      "learning_rate": 2.8075e-05,
      "log_odds_chosen": 11.734682083129883,
      "log_odds_ratio": -0.07055188715457916,
      "logits/chosen": 0.22402915358543396,
      "logits/rejected": 0.826654314994812,
      "logps/chosen": -1.6761119365692139,
      "logps/rejected": -13.128270149230957,
      "loss": 3.2389,
      "nll_loss": 3.2317991256713867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16761119663715363,
      "rewards/margins": 1.1452157497406006,
      "rewards/rejected": -1.3128271102905273,
      "step": 877
    },
    {
      "epoch": 0.546189735614308,
      "grad_norm": 0.4357503354549408,
      "learning_rate": 2.8050000000000004e-05,
      "log_odds_chosen": 18.032630920410156,
      "log_odds_ratio": -6.927158392500132e-05,
      "logits/chosen": 0.4070179760456085,
      "logits/rejected": 1.3222649097442627,
      "logps/chosen": -1.4783521890640259,
      "logps/rejected": -19.151775360107422,
      "loss": 3.4047,
      "nll_loss": 3.404698133468628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14783522486686707,
      "rewards/margins": 1.7673425674438477,
      "rewards/rejected": -1.915177822113037,
      "step": 878
    },
    {
      "epoch": 0.5468118195956454,
      "grad_norm": 0.4408838748931885,
      "learning_rate": 2.8025e-05,
      "log_odds_chosen": 10.542194366455078,
      "log_odds_ratio": -0.28352150321006775,
      "logits/chosen": 0.25072023272514343,
      "logits/rejected": 1.1159553527832031,
      "logps/chosen": -1.1630507707595825,
      "logps/rejected": -11.427528381347656,
      "loss": 2.7491,
      "nll_loss": 2.7207608222961426,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11630507558584213,
      "rewards/margins": 1.0264477729797363,
      "rewards/rejected": -1.1427528858184814,
      "step": 879
    },
    {
      "epoch": 0.5474339035769828,
      "grad_norm": 0.5349311828613281,
      "learning_rate": 2.8000000000000003e-05,
      "log_odds_chosen": 14.291757583618164,
      "log_odds_ratio": -0.251430481672287,
      "logits/chosen": 0.32711905241012573,
      "logits/rejected": 0.9382522702217102,
      "logps/chosen": -1.1399996280670166,
      "logps/rejected": -15.159124374389648,
      "loss": 2.9631,
      "nll_loss": 2.937985420227051,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11399996280670166,
      "rewards/margins": 1.4019126892089844,
      "rewards/rejected": -1.5159125328063965,
      "step": 880
    },
    {
      "epoch": 0.5480559875583204,
      "grad_norm": 0.4650278091430664,
      "learning_rate": 2.7975000000000002e-05,
      "log_odds_chosen": 14.79272747039795,
      "log_odds_ratio": -0.06423240154981613,
      "logits/chosen": 0.3779009282588959,
      "logits/rejected": 1.0615414381027222,
      "logps/chosen": -1.2400293350219727,
      "logps/rejected": -15.624247550964355,
      "loss": 3.1322,
      "nll_loss": 3.1258246898651123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12400293350219727,
      "rewards/margins": 1.4384218454360962,
      "rewards/rejected": -1.562424898147583,
      "step": 881
    },
    {
      "epoch": 0.5486780715396579,
      "grad_norm": 0.4733143746852875,
      "learning_rate": 2.7950000000000005e-05,
      "log_odds_chosen": 6.083606719970703,
      "log_odds_ratio": -0.33826759457588196,
      "logits/chosen": 0.2951495349407196,
      "logits/rejected": 0.5478895902633667,
      "logps/chosen": -1.0794161558151245,
      "logps/rejected": -6.8474345207214355,
      "loss": 2.9234,
      "nll_loss": 2.889528751373291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10794162005186081,
      "rewards/margins": 0.5768018364906311,
      "rewards/rejected": -0.6847434639930725,
      "step": 882
    },
    {
      "epoch": 0.5493001555209953,
      "grad_norm": 0.42593225836753845,
      "learning_rate": 2.7925e-05,
      "log_odds_chosen": 12.911933898925781,
      "log_odds_ratio": -0.3391897976398468,
      "logits/chosen": 0.31484758853912354,
      "logits/rejected": 1.1003917455673218,
      "logps/chosen": -1.1400558948516846,
      "logps/rejected": -13.681197166442871,
      "loss": 2.7707,
      "nll_loss": 2.7367684841156006,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11400558799505234,
      "rewards/margins": 1.2541141510009766,
      "rewards/rejected": -1.368119716644287,
      "step": 883
    },
    {
      "epoch": 0.5499222395023328,
      "grad_norm": 0.44411978125572205,
      "learning_rate": 2.7900000000000004e-05,
      "log_odds_chosen": 15.06024169921875,
      "log_odds_ratio": -0.13867679238319397,
      "logits/chosen": 0.2806948125362396,
      "logits/rejected": 1.4216599464416504,
      "logps/chosen": -1.1676044464111328,
      "logps/rejected": -15.850728034973145,
      "loss": 2.7267,
      "nll_loss": 2.7128076553344727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11676044762134552,
      "rewards/margins": 1.468312382698059,
      "rewards/rejected": -1.5850728750228882,
      "step": 884
    },
    {
      "epoch": 0.5505443234836703,
      "grad_norm": 1.44214928150177,
      "learning_rate": 2.7875e-05,
      "log_odds_chosen": 12.520509719848633,
      "log_odds_ratio": -0.18546557426452637,
      "logits/chosen": 0.16061872243881226,
      "logits/rejected": 0.9799090027809143,
      "logps/chosen": -1.3326960802078247,
      "logps/rejected": -13.587608337402344,
      "loss": 2.9357,
      "nll_loss": 2.917177677154541,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13326960802078247,
      "rewards/margins": 1.2254914045333862,
      "rewards/rejected": -1.3587608337402344,
      "step": 885
    },
    {
      "epoch": 0.5511664074650078,
      "grad_norm": 0.4010365605354309,
      "learning_rate": 2.7850000000000003e-05,
      "log_odds_chosen": 17.20555877685547,
      "log_odds_ratio": -8.836646884446964e-06,
      "logits/chosen": 0.32361698150634766,
      "logits/rejected": 1.3668720722198486,
      "logps/chosen": -1.1861801147460938,
      "logps/rejected": -17.992464065551758,
      "loss": 3.2253,
      "nll_loss": 3.2252705097198486,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11861801147460938,
      "rewards/margins": 1.6806282997131348,
      "rewards/rejected": -1.7992461919784546,
      "step": 886
    },
    {
      "epoch": 0.5517884914463452,
      "grad_norm": 0.5873351693153381,
      "learning_rate": 2.7825000000000002e-05,
      "log_odds_chosen": 8.508808135986328,
      "log_odds_ratio": -0.3058502972126007,
      "logits/chosen": 0.2773277759552002,
      "logits/rejected": 0.7884806394577026,
      "logps/chosen": -1.3308159112930298,
      "logps/rejected": -9.67809009552002,
      "loss": 2.7651,
      "nll_loss": 2.7345235347747803,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1330816000699997,
      "rewards/margins": 0.83472740650177,
      "rewards/rejected": -0.9678090214729309,
      "step": 887
    },
    {
      "epoch": 0.5524105754276827,
      "grad_norm": 22.026321411132812,
      "learning_rate": 2.7800000000000005e-05,
      "log_odds_chosen": 13.509963035583496,
      "log_odds_ratio": -0.002110434928908944,
      "logits/chosen": 0.3400443196296692,
      "logits/rejected": 1.213066577911377,
      "logps/chosen": -1.1196520328521729,
      "logps/rejected": -13.974618911743164,
      "loss": 2.9615,
      "nll_loss": 2.961289882659912,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11196519434452057,
      "rewards/margins": 1.2854965925216675,
      "rewards/rejected": -1.3974618911743164,
      "step": 888
    },
    {
      "epoch": 0.5530326594090202,
      "grad_norm": 3.287785291671753,
      "learning_rate": 2.7775e-05,
      "log_odds_chosen": 10.978760719299316,
      "log_odds_ratio": -0.5777610540390015,
      "logits/chosen": 0.35481563210487366,
      "logits/rejected": 0.9291172027587891,
      "logps/chosen": -2.050435781478882,
      "logps/rejected": -12.973872184753418,
      "loss": 3.2977,
      "nll_loss": 3.2398900985717773,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20504358410835266,
      "rewards/margins": 1.0923435688018799,
      "rewards/rejected": -1.2973871231079102,
      "step": 889
    },
    {
      "epoch": 0.5536547433903577,
      "grad_norm": 0.43079885840415955,
      "learning_rate": 2.7750000000000004e-05,
      "log_odds_chosen": 13.530800819396973,
      "log_odds_ratio": -0.01729394681751728,
      "logits/chosen": 0.22219036519527435,
      "logits/rejected": 0.8625215888023376,
      "logps/chosen": -1.6603683233261108,
      "logps/rejected": -14.933816909790039,
      "loss": 3.3577,
      "nll_loss": 3.3560125827789307,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16603684425354004,
      "rewards/margins": 1.3273450136184692,
      "rewards/rejected": -1.4933817386627197,
      "step": 890
    },
    {
      "epoch": 0.5542768273716951,
      "grad_norm": 0.4820476770401001,
      "learning_rate": 2.7725e-05,
      "log_odds_chosen": 7.27010440826416,
      "log_odds_ratio": -0.21835504472255707,
      "logits/chosen": 0.2366558313369751,
      "logits/rejected": 0.615390419960022,
      "logps/chosen": -1.1921238899230957,
      "logps/rejected": -8.159709930419922,
      "loss": 3.1871,
      "nll_loss": 3.1652331352233887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11921238899230957,
      "rewards/margins": 0.6967586278915405,
      "rewards/rejected": -0.8159710168838501,
      "step": 891
    },
    {
      "epoch": 0.5548989113530327,
      "grad_norm": 0.4553963840007782,
      "learning_rate": 2.7700000000000002e-05,
      "log_odds_chosen": 9.985837936401367,
      "log_odds_ratio": -0.2571437954902649,
      "logits/chosen": 0.23121318221092224,
      "logits/rejected": 0.8296804428100586,
      "logps/chosen": -1.2417988777160645,
      "logps/rejected": -11.013862609863281,
      "loss": 2.8727,
      "nll_loss": 2.846977710723877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1241798847913742,
      "rewards/margins": 0.977206289768219,
      "rewards/rejected": -1.1013861894607544,
      "step": 892
    },
    {
      "epoch": 0.5555209953343702,
      "grad_norm": 0.5255298018455505,
      "learning_rate": 2.7675000000000002e-05,
      "log_odds_chosen": 13.86887264251709,
      "log_odds_ratio": -0.19372327625751495,
      "logits/chosen": 0.30388227105140686,
      "logits/rejected": 1.4697858095169067,
      "logps/chosen": -1.3105946779251099,
      "logps/rejected": -14.92329216003418,
      "loss": 2.4972,
      "nll_loss": 2.47778058052063,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.131059467792511,
      "rewards/margins": 1.3612698316574097,
      "rewards/rejected": -1.4923293590545654,
      "step": 893
    },
    {
      "epoch": 0.5561430793157076,
      "grad_norm": 0.5121700763702393,
      "learning_rate": 2.7650000000000005e-05,
      "log_odds_chosen": 3.2167305946350098,
      "log_odds_ratio": -0.32042694091796875,
      "logits/chosen": 0.25697678327560425,
      "logits/rejected": 0.38786739110946655,
      "logps/chosen": -1.0599229335784912,
      "logps/rejected": -3.9423530101776123,
      "loss": 2.8183,
      "nll_loss": 2.786297082901001,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10599227994680405,
      "rewards/margins": 0.28824299573898315,
      "rewards/rejected": -0.3942352831363678,
      "step": 894
    },
    {
      "epoch": 0.5567651632970451,
      "grad_norm": 5.089456558227539,
      "learning_rate": 2.7625e-05,
      "log_odds_chosen": 8.74205493927002,
      "log_odds_ratio": -0.15869277715682983,
      "logits/chosen": 0.32882875204086304,
      "logits/rejected": 1.0555585622787476,
      "logps/chosen": -1.5479052066802979,
      "logps/rejected": -9.963598251342773,
      "loss": 2.7022,
      "nll_loss": 2.686356782913208,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15479052066802979,
      "rewards/margins": 0.8415693044662476,
      "rewards/rejected": -0.9963598847389221,
      "step": 895
    },
    {
      "epoch": 0.5573872472783826,
      "grad_norm": 0.523443341255188,
      "learning_rate": 2.7600000000000003e-05,
      "log_odds_chosen": 15.575971603393555,
      "log_odds_ratio": -0.08751137554645538,
      "logits/chosen": 0.38994699716567993,
      "logits/rejected": 1.6484946012496948,
      "logps/chosen": -1.126055121421814,
      "logps/rejected": -16.346158981323242,
      "loss": 2.8432,
      "nll_loss": 2.834432363510132,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1126055121421814,
      "rewards/margins": 1.5220104455947876,
      "rewards/rejected": -1.6346158981323242,
      "step": 896
    },
    {
      "epoch": 0.5580093312597201,
      "grad_norm": 0.43732067942619324,
      "learning_rate": 2.7575e-05,
      "log_odds_chosen": 12.99947452545166,
      "log_odds_ratio": -0.22496089339256287,
      "logits/chosen": 0.3272874653339386,
      "logits/rejected": 0.942588210105896,
      "logps/chosen": -1.0477204322814941,
      "logps/rejected": -13.669449806213379,
      "loss": 3.2135,
      "nll_loss": 3.191025495529175,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10477204620838165,
      "rewards/margins": 1.2621729373931885,
      "rewards/rejected": -1.3669450283050537,
      "step": 897
    },
    {
      "epoch": 0.5586314152410575,
      "grad_norm": 0.6751424670219421,
      "learning_rate": 2.7550000000000002e-05,
      "log_odds_chosen": 8.845113754272461,
      "log_odds_ratio": -0.1459217518568039,
      "logits/chosen": 0.12097790837287903,
      "logits/rejected": 0.7129504084587097,
      "logps/chosen": -0.9098332524299622,
      "logps/rejected": -9.02649974822998,
      "loss": 2.4552,
      "nll_loss": 2.440615653991699,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0909833312034607,
      "rewards/margins": 0.81166672706604,
      "rewards/rejected": -0.9026500582695007,
      "step": 898
    },
    {
      "epoch": 0.559253499222395,
      "grad_norm": 0.5536679029464722,
      "learning_rate": 2.7525e-05,
      "log_odds_chosen": 10.608327865600586,
      "log_odds_ratio": -0.01897694170475006,
      "logits/chosen": 0.22658635675907135,
      "logits/rejected": 0.7577848434448242,
      "logps/chosen": -1.0816148519515991,
      "logps/rejected": -11.110265731811523,
      "loss": 2.8688,
      "nll_loss": 2.866927146911621,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10816149413585663,
      "rewards/margins": 1.0028650760650635,
      "rewards/rejected": -1.111026644706726,
      "step": 899
    },
    {
      "epoch": 0.5598755832037325,
      "grad_norm": 0.43533676862716675,
      "learning_rate": 2.7500000000000004e-05,
      "log_odds_chosen": 7.546494483947754,
      "log_odds_ratio": -0.1429726481437683,
      "logits/chosen": 0.20976758003234863,
      "logits/rejected": 0.4699644446372986,
      "logps/chosen": -1.0662420988082886,
      "logps/rejected": -8.181591033935547,
      "loss": 3.1895,
      "nll_loss": 3.1751976013183594,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10662420839071274,
      "rewards/margins": 0.7115349769592285,
      "rewards/rejected": -0.8181591629981995,
      "step": 900
    },
    {
      "epoch": 0.56049766718507,
      "grad_norm": 0.5101000666618347,
      "learning_rate": 2.7475e-05,
      "log_odds_chosen": 9.346063613891602,
      "log_odds_ratio": -0.16607795655727386,
      "logits/chosen": 0.22395896911621094,
      "logits/rejected": 0.6142349243164062,
      "logps/chosen": -1.1865246295928955,
      "logps/rejected": -9.847481727600098,
      "loss": 3.1556,
      "nll_loss": 3.1389565467834473,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11865246295928955,
      "rewards/margins": 0.8660956621170044,
      "rewards/rejected": -0.9847480654716492,
      "step": 901
    },
    {
      "epoch": 0.5611197511664074,
      "grad_norm": 0.44190743565559387,
      "learning_rate": 2.7450000000000003e-05,
      "log_odds_chosen": 7.044838905334473,
      "log_odds_ratio": -0.14595672488212585,
      "logits/chosen": 0.1050267368555069,
      "logits/rejected": 0.3443432152271271,
      "logps/chosen": -1.0475255250930786,
      "logps/rejected": -7.505648612976074,
      "loss": 3.0451,
      "nll_loss": 3.030492067337036,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1047525554895401,
      "rewards/margins": 0.6458122730255127,
      "rewards/rejected": -0.7505648136138916,
      "step": 902
    },
    {
      "epoch": 0.5617418351477449,
      "grad_norm": 0.3499910235404968,
      "learning_rate": 2.7425e-05,
      "log_odds_chosen": 7.8855743408203125,
      "log_odds_ratio": -0.2606319189071655,
      "logits/chosen": 0.29135677218437195,
      "logits/rejected": 0.577876091003418,
      "logps/chosen": -0.9390688538551331,
      "logps/rejected": -8.381467819213867,
      "loss": 3.7401,
      "nll_loss": 3.7140378952026367,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09390688687562943,
      "rewards/margins": 0.7442399263381958,
      "rewards/rejected": -0.8381468057632446,
      "step": 903
    },
    {
      "epoch": 0.5623639191290825,
      "grad_norm": 0.5649048089981079,
      "learning_rate": 2.7400000000000002e-05,
      "log_odds_chosen": 8.967251777648926,
      "log_odds_ratio": -0.03768826648592949,
      "logits/chosen": 0.2890816330909729,
      "logits/rejected": 0.7596768140792847,
      "logps/chosen": -0.9239016175270081,
      "logps/rejected": -9.305893898010254,
      "loss": 3.2545,
      "nll_loss": 3.2506906986236572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09239016473293304,
      "rewards/margins": 0.838199257850647,
      "rewards/rejected": -0.9305894374847412,
      "step": 904
    },
    {
      "epoch": 0.5629860031104199,
      "grad_norm": 0.45023804903030396,
      "learning_rate": 2.7375e-05,
      "log_odds_chosen": 6.16334342956543,
      "log_odds_ratio": -0.2813335359096527,
      "logits/chosen": 0.16055354475975037,
      "logits/rejected": 0.49607011675834656,
      "logps/chosen": -1.1517943143844604,
      "logps/rejected": -6.91542911529541,
      "loss": 2.9652,
      "nll_loss": 2.9370510578155518,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11517943441867828,
      "rewards/margins": 0.5763634443283081,
      "rewards/rejected": -0.6915428638458252,
      "step": 905
    },
    {
      "epoch": 0.5636080870917574,
      "grad_norm": 0.4151860475540161,
      "learning_rate": 2.7350000000000004e-05,
      "log_odds_chosen": 5.84874153137207,
      "log_odds_ratio": -0.30869588255882263,
      "logits/chosen": 0.17200124263763428,
      "logits/rejected": 0.48454806208610535,
      "logps/chosen": -1.2069495916366577,
      "logps/rejected": -6.622710227966309,
      "loss": 3.2825,
      "nll_loss": 3.2516021728515625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12069497257471085,
      "rewards/margins": 0.5415760278701782,
      "rewards/rejected": -0.6622710227966309,
      "step": 906
    },
    {
      "epoch": 0.5642301710730949,
      "grad_norm": 0.3901362717151642,
      "learning_rate": 2.7325e-05,
      "log_odds_chosen": 8.91002368927002,
      "log_odds_ratio": -0.09111232310533524,
      "logits/chosen": 0.22789613902568817,
      "logits/rejected": 0.7879712581634521,
      "logps/chosen": -1.3444089889526367,
      "logps/rejected": -9.875727653503418,
      "loss": 3.2262,
      "nll_loss": 3.2170474529266357,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13444089889526367,
      "rewards/margins": 0.8531317710876465,
      "rewards/rejected": -0.9875727295875549,
      "step": 907
    },
    {
      "epoch": 0.5648522550544324,
      "grad_norm": 0.4596397280693054,
      "learning_rate": 2.7300000000000003e-05,
      "log_odds_chosen": 6.89910888671875,
      "log_odds_ratio": -0.14412494003772736,
      "logits/chosen": 0.2593741714954376,
      "logits/rejected": 0.8184172511100769,
      "logps/chosen": -1.1923887729644775,
      "logps/rejected": -7.755105018615723,
      "loss": 2.9366,
      "nll_loss": 2.9221863746643066,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11923889070749283,
      "rewards/margins": 0.6562716960906982,
      "rewards/rejected": -0.7755105495452881,
      "step": 908
    },
    {
      "epoch": 0.5654743390357698,
      "grad_norm": 0.5999178290367126,
      "learning_rate": 2.7275e-05,
      "log_odds_chosen": 11.099242210388184,
      "log_odds_ratio": -0.17223107814788818,
      "logits/chosen": 0.16461758315563202,
      "logits/rejected": 0.345947265625,
      "logps/chosen": -1.1326367855072021,
      "logps/rejected": -11.848851203918457,
      "loss": 3.0254,
      "nll_loss": 3.008146047592163,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11326368153095245,
      "rewards/margins": 1.0716214179992676,
      "rewards/rejected": -1.1848851442337036,
      "step": 909
    },
    {
      "epoch": 0.5660964230171073,
      "grad_norm": 0.37448394298553467,
      "learning_rate": 2.725e-05,
      "log_odds_chosen": 5.122501850128174,
      "log_odds_ratio": -0.26253485679626465,
      "logits/chosen": 0.1458125114440918,
      "logits/rejected": 0.38186800479888916,
      "logps/chosen": -0.9692260026931763,
      "logps/rejected": -5.624997138977051,
      "loss": 2.946,
      "nll_loss": 2.9197349548339844,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09692259877920151,
      "rewards/margins": 0.4655771255493164,
      "rewards/rejected": -0.5624997019767761,
      "step": 910
    },
    {
      "epoch": 0.5667185069984448,
      "grad_norm": 0.46044209599494934,
      "learning_rate": 2.7225e-05,
      "log_odds_chosen": 6.9234795570373535,
      "log_odds_ratio": -0.21885745227336884,
      "logits/chosen": 0.09006873518228531,
      "logits/rejected": 0.40548866987228394,
      "logps/chosen": -0.9413871169090271,
      "logps/rejected": -7.406796455383301,
      "loss": 2.6326,
      "nll_loss": 2.6107070446014404,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09413871169090271,
      "rewards/margins": 0.6465408802032471,
      "rewards/rejected": -0.7406796216964722,
      "step": 911
    },
    {
      "epoch": 0.5673405909797823,
      "grad_norm": 0.48035767674446106,
      "learning_rate": 2.7200000000000004e-05,
      "log_odds_chosen": 9.26119613647461,
      "log_odds_ratio": -0.17187556624412537,
      "logits/chosen": 0.13499784469604492,
      "logits/rejected": 0.6166836619377136,
      "logps/chosen": -1.251449704170227,
      "logps/rejected": -10.169729232788086,
      "loss": 2.6754,
      "nll_loss": 2.658219814300537,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12514497339725494,
      "rewards/margins": 0.8918280005455017,
      "rewards/rejected": -1.0169728994369507,
      "step": 912
    },
    {
      "epoch": 0.5679626749611197,
      "grad_norm": 0.47506648302078247,
      "learning_rate": 2.7175e-05,
      "log_odds_chosen": 10.130144119262695,
      "log_odds_ratio": -0.04994361102581024,
      "logits/chosen": 0.17378772795200348,
      "logits/rejected": 0.629872739315033,
      "logps/chosen": -0.772241473197937,
      "logps/rejected": -10.029999732971191,
      "loss": 2.8842,
      "nll_loss": 2.8792366981506348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07722415030002594,
      "rewards/margins": 0.9257758855819702,
      "rewards/rejected": -1.003000020980835,
      "step": 913
    },
    {
      "epoch": 0.5685847589424572,
      "grad_norm": 0.48764681816101074,
      "learning_rate": 2.7150000000000003e-05,
      "log_odds_chosen": 5.916272163391113,
      "log_odds_ratio": -0.25303128361701965,
      "logits/chosen": 0.1644158661365509,
      "logits/rejected": 0.30059051513671875,
      "logps/chosen": -1.1256078481674194,
      "logps/rejected": -6.709453105926514,
      "loss": 3.0249,
      "nll_loss": 2.9995951652526855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11256079375743866,
      "rewards/margins": 0.5583845376968384,
      "rewards/rejected": -0.6709453463554382,
      "step": 914
    },
    {
      "epoch": 0.5692068429237948,
      "grad_norm": 0.5090453028678894,
      "learning_rate": 2.7125000000000002e-05,
      "log_odds_chosen": 9.453378677368164,
      "log_odds_ratio": -0.11234141886234283,
      "logits/chosen": 0.2241164743900299,
      "logits/rejected": 0.9943174719810486,
      "logps/chosen": -1.0483677387237549,
      "logps/rejected": -9.902132987976074,
      "loss": 2.8647,
      "nll_loss": 2.8534181118011475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10483676195144653,
      "rewards/margins": 0.8853765726089478,
      "rewards/rejected": -0.9902133941650391,
      "step": 915
    },
    {
      "epoch": 0.5698289269051322,
      "grad_norm": 0.3839872181415558,
      "learning_rate": 2.7100000000000005e-05,
      "log_odds_chosen": 12.735099792480469,
      "log_odds_ratio": -0.07314764708280563,
      "logits/chosen": 0.3284153938293457,
      "logits/rejected": 1.1309449672698975,
      "logps/chosen": -0.8413907289505005,
      "logps/rejected": -12.652066230773926,
      "loss": 3.1403,
      "nll_loss": 3.132990837097168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08413907140493393,
      "rewards/margins": 1.1810675859451294,
      "rewards/rejected": -1.2652066946029663,
      "step": 916
    },
    {
      "epoch": 0.5704510108864697,
      "grad_norm": 0.7262453436851501,
      "learning_rate": 2.7075e-05,
      "log_odds_chosen": 10.773283958435059,
      "log_odds_ratio": -0.17366476356983185,
      "logits/chosen": 0.27300554513931274,
      "logits/rejected": 0.6795448064804077,
      "logps/chosen": -1.1927194595336914,
      "logps/rejected": -11.570327758789062,
      "loss": 2.9818,
      "nll_loss": 2.964430570602417,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11927194148302078,
      "rewards/margins": 1.037760853767395,
      "rewards/rejected": -1.15703284740448,
      "step": 917
    },
    {
      "epoch": 0.5710730948678071,
      "grad_norm": 0.4268437922000885,
      "learning_rate": 2.7050000000000004e-05,
      "log_odds_chosen": 12.669416427612305,
      "log_odds_ratio": -0.18929001688957214,
      "logits/chosen": 0.23210585117340088,
      "logits/rejected": 0.7328687906265259,
      "logps/chosen": -0.9498953223228455,
      "logps/rejected": -13.114092826843262,
      "loss": 3.6412,
      "nll_loss": 3.622225761413574,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09498953074216843,
      "rewards/margins": 1.2164198160171509,
      "rewards/rejected": -1.3114093542099,
      "step": 918
    },
    {
      "epoch": 0.5716951788491447,
      "grad_norm": 0.4433814585208893,
      "learning_rate": 2.7025e-05,
      "log_odds_chosen": 10.184727668762207,
      "log_odds_ratio": -0.025781212374567986,
      "logits/chosen": 0.17551684379577637,
      "logits/rejected": 1.0827550888061523,
      "logps/chosen": -1.2214159965515137,
      "logps/rejected": -11.030315399169922,
      "loss": 2.7723,
      "nll_loss": 2.769714832305908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12214161455631256,
      "rewards/margins": 0.9808899760246277,
      "rewards/rejected": -1.1030315160751343,
      "step": 919
    },
    {
      "epoch": 0.5723172628304821,
      "grad_norm": 0.48769259452819824,
      "learning_rate": 2.7000000000000002e-05,
      "log_odds_chosen": 9.125656127929688,
      "log_odds_ratio": -0.26961836218833923,
      "logits/chosen": 0.1965629905462265,
      "logits/rejected": 0.3319164514541626,
      "logps/chosen": -1.3702173233032227,
      "logps/rejected": -10.158577919006348,
      "loss": 3.5997,
      "nll_loss": 3.5727407932281494,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1370217353105545,
      "rewards/margins": 0.8788360357284546,
      "rewards/rejected": -1.0158576965332031,
      "step": 920
    },
    {
      "epoch": 0.5729393468118196,
      "grad_norm": 0.48434844613075256,
      "learning_rate": 2.6975000000000002e-05,
      "log_odds_chosen": 13.56881332397461,
      "log_odds_ratio": -0.0026917767245322466,
      "logits/chosen": 0.2141491025686264,
      "logits/rejected": 1.3020198345184326,
      "logps/chosen": -1.3336665630340576,
      "logps/rejected": -14.552571296691895,
      "loss": 2.7108,
      "nll_loss": 2.7105112075805664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1333666741847992,
      "rewards/margins": 1.3218905925750732,
      "rewards/rejected": -1.4552571773529053,
      "step": 921
    },
    {
      "epoch": 0.573561430793157,
      "grad_norm": 0.5728157162666321,
      "learning_rate": 2.6950000000000005e-05,
      "log_odds_chosen": 10.488691329956055,
      "log_odds_ratio": -0.23412871360778809,
      "logits/chosen": 0.2155309021472931,
      "logits/rejected": 0.6558471322059631,
      "logps/chosen": -1.1757211685180664,
      "logps/rejected": -11.354267120361328,
      "loss": 3.3212,
      "nll_loss": 3.2977969646453857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.117572121322155,
      "rewards/margins": 1.0178545713424683,
      "rewards/rejected": -1.135426640510559,
      "step": 922
    },
    {
      "epoch": 0.5741835147744946,
      "grad_norm": 0.472119003534317,
      "learning_rate": 2.6925e-05,
      "log_odds_chosen": 8.171377182006836,
      "log_odds_ratio": -0.22905880212783813,
      "logits/chosen": 0.1736389398574829,
      "logits/rejected": 0.8877742290496826,
      "logps/chosen": -1.1164681911468506,
      "logps/rejected": -8.963059425354004,
      "loss": 2.8084,
      "nll_loss": 2.785543203353882,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11164682358503342,
      "rewards/margins": 0.7846590876579285,
      "rewards/rejected": -0.8963059186935425,
      "step": 923
    },
    {
      "epoch": 0.574805598755832,
      "grad_norm": 1.098946213722229,
      "learning_rate": 2.6900000000000003e-05,
      "log_odds_chosen": 8.532747268676758,
      "log_odds_ratio": -0.16291804611682892,
      "logits/chosen": 0.26898065209388733,
      "logits/rejected": 0.7751161456108093,
      "logps/chosen": -1.2293330430984497,
      "logps/rejected": -9.393357276916504,
      "loss": 3.3784,
      "nll_loss": 3.3621580600738525,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12293331325054169,
      "rewards/margins": 0.8164024353027344,
      "rewards/rejected": -0.9393357634544373,
      "step": 924
    },
    {
      "epoch": 0.5754276827371695,
      "grad_norm": 0.661605715751648,
      "learning_rate": 2.6875e-05,
      "log_odds_chosen": 16.35256576538086,
      "log_odds_ratio": -0.1963188201189041,
      "logits/chosen": 0.1324017345905304,
      "logits/rejected": 0.8213398456573486,
      "logps/chosen": -1.7915053367614746,
      "logps/rejected": -17.91004180908203,
      "loss": 3.0766,
      "nll_loss": 3.0570147037506104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1791505366563797,
      "rewards/margins": 1.611853837966919,
      "rewards/rejected": -1.7910041809082031,
      "step": 925
    },
    {
      "epoch": 0.576049766718507,
      "grad_norm": 0.3944947123527527,
      "learning_rate": 2.6850000000000002e-05,
      "log_odds_chosen": 19.438589096069336,
      "log_odds_ratio": -0.09257218986749649,
      "logits/chosen": 0.11398239433765411,
      "logits/rejected": 0.9922657608985901,
      "logps/chosen": -0.9387992024421692,
      "logps/rejected": -19.83526039123535,
      "loss": 2.8709,
      "nll_loss": 2.8616538047790527,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09387992322444916,
      "rewards/margins": 1.889646291732788,
      "rewards/rejected": -1.9835262298583984,
      "step": 926
    },
    {
      "epoch": 0.5766718506998445,
      "grad_norm": 0.3849738538265228,
      "learning_rate": 2.6825e-05,
      "log_odds_chosen": 16.61699867248535,
      "log_odds_ratio": -0.03179116174578667,
      "logits/chosen": 0.2330043613910675,
      "logits/rejected": 1.0248134136199951,
      "logps/chosen": -1.3797637224197388,
      "logps/rejected": -17.69917106628418,
      "loss": 3.4806,
      "nll_loss": 3.4774537086486816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13797636330127716,
      "rewards/margins": 1.6319406032562256,
      "rewards/rejected": -1.7699170112609863,
      "step": 927
    },
    {
      "epoch": 0.577293934681182,
      "grad_norm": 0.5158620476722717,
      "learning_rate": 2.6800000000000004e-05,
      "log_odds_chosen": 20.5090274810791,
      "log_odds_ratio": -0.0825430378317833,
      "logits/chosen": 0.11498251557350159,
      "logits/rejected": 0.9717357754707336,
      "logps/chosen": -0.903712272644043,
      "logps/rejected": -20.81587028503418,
      "loss": 2.7439,
      "nll_loss": 2.735637664794922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09037123620510101,
      "rewards/margins": 1.9912158250808716,
      "rewards/rejected": -2.081587314605713,
      "step": 928
    },
    {
      "epoch": 0.5779160186625194,
      "grad_norm": 0.4215962886810303,
      "learning_rate": 2.6775e-05,
      "log_odds_chosen": 22.170780181884766,
      "log_odds_ratio": 0.0,
      "logits/chosen": 0.3199015259742737,
      "logits/rejected": 1.854710340499878,
      "logps/chosen": -1.3474838733673096,
      "logps/rejected": -23.19628143310547,
      "loss": 3.4444,
      "nll_loss": 3.444409132003784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13474836945533752,
      "rewards/margins": 2.184879779815674,
      "rewards/rejected": -2.3196280002593994,
      "step": 929
    },
    {
      "epoch": 0.578538102643857,
      "grad_norm": 0.7233343720436096,
      "learning_rate": 2.6750000000000003e-05,
      "log_odds_chosen": 13.190216064453125,
      "log_odds_ratio": -0.1239352896809578,
      "logits/chosen": -0.007729051634669304,
      "logits/rejected": 0.6695624589920044,
      "logps/chosen": -1.3063056468963623,
      "logps/rejected": -14.166091918945312,
      "loss": 2.7662,
      "nll_loss": 2.7538347244262695,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13063055276870728,
      "rewards/margins": 1.2859786748886108,
      "rewards/rejected": -1.416609287261963,
      "step": 930
    },
    {
      "epoch": 0.5791601866251944,
      "grad_norm": 0.5069626569747925,
      "learning_rate": 2.6725e-05,
      "log_odds_chosen": 10.079081535339355,
      "log_odds_ratio": -0.2044445276260376,
      "logits/chosen": 0.21780771017074585,
      "logits/rejected": 0.6870952844619751,
      "logps/chosen": -1.3345181941986084,
      "logps/rejected": -11.053346633911133,
      "loss": 3.3541,
      "nll_loss": 3.333698272705078,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13345181941986084,
      "rewards/margins": 0.9718827605247498,
      "rewards/rejected": -1.1053345203399658,
      "step": 931
    },
    {
      "epoch": 0.5797822706065319,
      "grad_norm": 0.7521727681159973,
      "learning_rate": 2.6700000000000002e-05,
      "log_odds_chosen": 10.476853370666504,
      "log_odds_ratio": -0.2420932799577713,
      "logits/chosen": 0.17987507581710815,
      "logits/rejected": 0.8727846741676331,
      "logps/chosen": -1.2537364959716797,
      "logps/rejected": -11.50350570678711,
      "loss": 3.1104,
      "nll_loss": 3.0861494541168213,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12537366151809692,
      "rewards/margins": 1.0249770879745483,
      "rewards/rejected": -1.1503506898880005,
      "step": 932
    },
    {
      "epoch": 0.5804043545878693,
      "grad_norm": 0.5356715321540833,
      "learning_rate": 2.6675e-05,
      "log_odds_chosen": 14.187935829162598,
      "log_odds_ratio": -0.24278056621551514,
      "logits/chosen": 0.17469918727874756,
      "logits/rejected": 1.1708053350448608,
      "logps/chosen": -1.0131219625473022,
      "logps/rejected": -14.83353328704834,
      "loss": 2.2884,
      "nll_loss": 2.26407527923584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10131219774484634,
      "rewards/margins": 1.3820412158966064,
      "rewards/rejected": -1.4833533763885498,
      "step": 933
    },
    {
      "epoch": 0.5810264385692069,
      "grad_norm": 0.5571019649505615,
      "learning_rate": 2.6650000000000004e-05,
      "log_odds_chosen": 9.943033218383789,
      "log_odds_ratio": -0.26776123046875,
      "logits/chosen": 0.265704870223999,
      "logits/rejected": 0.9763750433921814,
      "logps/chosen": -1.1288625001907349,
      "logps/rejected": -10.797094345092773,
      "loss": 3.5617,
      "nll_loss": 3.534961700439453,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11288625001907349,
      "rewards/margins": 0.966823160648346,
      "rewards/rejected": -1.0797094106674194,
      "step": 934
    },
    {
      "epoch": 0.5816485225505443,
      "grad_norm": 0.45613187551498413,
      "learning_rate": 2.6625e-05,
      "log_odds_chosen": 8.923871040344238,
      "log_odds_ratio": -0.25769421458244324,
      "logits/chosen": 0.07586418092250824,
      "logits/rejected": 0.34221017360687256,
      "logps/chosen": -1.0070983171463013,
      "logps/rejected": -9.35234260559082,
      "loss": 3.1174,
      "nll_loss": 3.0916061401367188,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10070984065532684,
      "rewards/margins": 0.8345244526863098,
      "rewards/rejected": -0.9352343082427979,
      "step": 935
    },
    {
      "epoch": 0.5822706065318818,
      "grad_norm": 0.44240835309028625,
      "learning_rate": 2.6600000000000003e-05,
      "log_odds_chosen": 20.093387603759766,
      "log_odds_ratio": -0.11705049872398376,
      "logits/chosen": 0.2263740599155426,
      "logits/rejected": 1.405846357345581,
      "logps/chosen": -1.076794147491455,
      "logps/rejected": -20.625396728515625,
      "loss": 2.8483,
      "nll_loss": 2.8366036415100098,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10767941921949387,
      "rewards/margins": 1.9548602104187012,
      "rewards/rejected": -2.062539577484131,
      "step": 936
    },
    {
      "epoch": 0.5828926905132192,
      "grad_norm": 0.39377760887145996,
      "learning_rate": 2.6575e-05,
      "log_odds_chosen": 14.986661911010742,
      "log_odds_ratio": -0.27674102783203125,
      "logits/chosen": 0.1652100682258606,
      "logits/rejected": 0.9994176626205444,
      "logps/chosen": -1.0255934000015259,
      "logps/rejected": -15.587301254272461,
      "loss": 2.8296,
      "nll_loss": 2.801974296569824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10255934298038483,
      "rewards/margins": 1.4561707973480225,
      "rewards/rejected": -1.558730125427246,
      "step": 937
    },
    {
      "epoch": 0.5835147744945568,
      "grad_norm": 4.184680938720703,
      "learning_rate": 2.655e-05,
      "log_odds_chosen": 11.676606178283691,
      "log_odds_ratio": -0.12282869964838028,
      "logits/chosen": 0.21672847867012024,
      "logits/rejected": 0.7535300254821777,
      "logps/chosen": -1.5664196014404297,
      "logps/rejected": -12.69096851348877,
      "loss": 3.4853,
      "nll_loss": 3.4730653762817383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15664197504520416,
      "rewards/margins": 1.112454891204834,
      "rewards/rejected": -1.269096851348877,
      "step": 938
    },
    {
      "epoch": 0.5841368584758942,
      "grad_norm": 0.5880439877510071,
      "learning_rate": 2.6525e-05,
      "log_odds_chosen": 10.81521224975586,
      "log_odds_ratio": -0.47919604182243347,
      "logits/chosen": 0.1771206557750702,
      "logits/rejected": 0.8535869121551514,
      "logps/chosen": -1.2247461080551147,
      "logps/rejected": -11.720307350158691,
      "loss": 2.8229,
      "nll_loss": 2.774963855743408,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12247461825609207,
      "rewards/margins": 1.0495561361312866,
      "rewards/rejected": -1.1720308065414429,
      "step": 939
    },
    {
      "epoch": 0.5847589424572317,
      "grad_norm": 2.263173818588257,
      "learning_rate": 2.6500000000000004e-05,
      "log_odds_chosen": 14.880687713623047,
      "log_odds_ratio": -0.04369397461414337,
      "logits/chosen": 0.17089928686618805,
      "logits/rejected": 0.7093451023101807,
      "logps/chosen": -1.288264274597168,
      "logps/rejected": -15.811230659484863,
      "loss": 3.2519,
      "nll_loss": 3.247570276260376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12882643938064575,
      "rewards/margins": 1.4522968530654907,
      "rewards/rejected": -1.5811231136322021,
      "step": 940
    },
    {
      "epoch": 0.5853810264385692,
      "grad_norm": 0.6679919362068176,
      "learning_rate": 2.6475e-05,
      "log_odds_chosen": 13.88052749633789,
      "log_odds_ratio": -0.17339672148227692,
      "logits/chosen": 0.2502378821372986,
      "logits/rejected": 1.128913402557373,
      "logps/chosen": -1.2840638160705566,
      "logps/rejected": -14.904745101928711,
      "loss": 2.6963,
      "nll_loss": 2.67893648147583,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1284063756465912,
      "rewards/margins": 1.3620681762695312,
      "rewards/rejected": -1.4904745817184448,
      "step": 941
    },
    {
      "epoch": 0.5860031104199067,
      "grad_norm": 1.2314687967300415,
      "learning_rate": 2.6450000000000003e-05,
      "log_odds_chosen": 18.171859741210938,
      "log_odds_ratio": -0.1047465056180954,
      "logits/chosen": 0.36809712648391724,
      "logits/rejected": 1.2636301517486572,
      "logps/chosen": -1.0214661359786987,
      "logps/rejected": -18.68871307373047,
      "loss": 3.5026,
      "nll_loss": 3.492130756378174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10214661061763763,
      "rewards/margins": 1.7667248249053955,
      "rewards/rejected": -1.8688714504241943,
      "step": 942
    },
    {
      "epoch": 0.5866251944012442,
      "grad_norm": 0.5520533919334412,
      "learning_rate": 2.6425e-05,
      "log_odds_chosen": 9.532363891601562,
      "log_odds_ratio": -0.033491168171167374,
      "logits/chosen": 0.3422502279281616,
      "logits/rejected": 0.8718456029891968,
      "logps/chosen": -1.18587327003479,
      "logps/rejected": -10.302650451660156,
      "loss": 3.2041,
      "nll_loss": 3.20070743560791,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11858732998371124,
      "rewards/margins": 0.9116777181625366,
      "rewards/rejected": -1.0302650928497314,
      "step": 943
    },
    {
      "epoch": 0.5872472783825816,
      "grad_norm": 0.5288044810295105,
      "learning_rate": 2.64e-05,
      "log_odds_chosen": 15.668525695800781,
      "log_odds_ratio": -0.27726009488105774,
      "logits/chosen": 0.2326730489730835,
      "logits/rejected": 0.898745059967041,
      "logps/chosen": -1.0137486457824707,
      "logps/rejected": -16.33362579345703,
      "loss": 2.6635,
      "nll_loss": 2.6358118057250977,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10137484967708588,
      "rewards/margins": 1.5319877862930298,
      "rewards/rejected": -1.6333625316619873,
      "step": 944
    },
    {
      "epoch": 0.5878693623639192,
      "grad_norm": 0.46301695704460144,
      "learning_rate": 2.6375e-05,
      "log_odds_chosen": 17.152551651000977,
      "log_odds_ratio": -0.07121972739696503,
      "logits/chosen": 0.4616335928440094,
      "logits/rejected": 1.5514743328094482,
      "logps/chosen": -1.638116478919983,
      "logps/rejected": -18.526134490966797,
      "loss": 3.2308,
      "nll_loss": 3.22369647026062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16381165385246277,
      "rewards/margins": 1.6888020038604736,
      "rewards/rejected": -1.8526135683059692,
      "step": 945
    },
    {
      "epoch": 0.5884914463452566,
      "grad_norm": 0.4805068075656891,
      "learning_rate": 2.6350000000000004e-05,
      "log_odds_chosen": 10.249034881591797,
      "log_odds_ratio": -0.19067715108394623,
      "logits/chosen": 0.2742692232131958,
      "logits/rejected": 0.5140730142593384,
      "logps/chosen": -1.0531798601150513,
      "logps/rejected": -10.836760520935059,
      "loss": 2.7838,
      "nll_loss": 2.764707565307617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10531798750162125,
      "rewards/margins": 0.9783580899238586,
      "rewards/rejected": -1.0836760997772217,
      "step": 946
    },
    {
      "epoch": 0.5891135303265941,
      "grad_norm": 0.4442595839500427,
      "learning_rate": 2.6325e-05,
      "log_odds_chosen": 16.763885498046875,
      "log_odds_ratio": -0.14290139079093933,
      "logits/chosen": 0.3244515061378479,
      "logits/rejected": 1.0541630983352661,
      "logps/chosen": -0.9952157735824585,
      "logps/rejected": -17.375211715698242,
      "loss": 3.0868,
      "nll_loss": 3.072525978088379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09952157735824585,
      "rewards/margins": 1.6379995346069336,
      "rewards/rejected": -1.7375211715698242,
      "step": 947
    },
    {
      "epoch": 0.5897356143079315,
      "grad_norm": 0.3361284136772156,
      "learning_rate": 2.6300000000000002e-05,
      "log_odds_chosen": 19.245708465576172,
      "log_odds_ratio": -0.07403115928173065,
      "logits/chosen": 0.31128478050231934,
      "logits/rejected": 0.9080635905265808,
      "logps/chosen": -0.8605557680130005,
      "logps/rejected": -19.487812042236328,
      "loss": 3.7087,
      "nll_loss": 3.7012686729431152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08605558425188065,
      "rewards/margins": 1.8627257347106934,
      "rewards/rejected": -1.9487813711166382,
      "step": 948
    },
    {
      "epoch": 0.5903576982892691,
      "grad_norm": 0.5872551798820496,
      "learning_rate": 2.6275e-05,
      "log_odds_chosen": 8.34841251373291,
      "log_odds_ratio": -0.3960355222225189,
      "logits/chosen": 0.1396239548921585,
      "logits/rejected": 0.6412820219993591,
      "logps/chosen": -1.0153343677520752,
      "logps/rejected": -9.043679237365723,
      "loss": 2.4457,
      "nll_loss": 2.4060676097869873,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.101533442735672,
      "rewards/margins": 0.8028345108032227,
      "rewards/rejected": -0.9043679237365723,
      "step": 949
    },
    {
      "epoch": 0.5909797822706065,
      "grad_norm": 0.5597155094146729,
      "learning_rate": 2.625e-05,
      "log_odds_chosen": 17.4451847076416,
      "log_odds_ratio": -0.0725177526473999,
      "logits/chosen": 0.110418900847435,
      "logits/rejected": 0.8833154439926147,
      "logps/chosen": -0.8898366093635559,
      "logps/rejected": -17.72234535217285,
      "loss": 2.631,
      "nll_loss": 2.623771905899048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08898366987705231,
      "rewards/margins": 1.683250904083252,
      "rewards/rejected": -1.7722344398498535,
      "step": 950
    },
    {
      "epoch": 0.591601866251944,
      "grad_norm": 0.7085988521575928,
      "learning_rate": 2.6225e-05,
      "log_odds_chosen": 8.395122528076172,
      "log_odds_ratio": -0.36476755142211914,
      "logits/chosen": 0.26892557740211487,
      "logits/rejected": 0.6090400218963623,
      "logps/chosen": -1.258937954902649,
      "logps/rejected": -9.453425407409668,
      "loss": 3.015,
      "nll_loss": 2.9785232543945312,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12589380145072937,
      "rewards/margins": 0.8194488286972046,
      "rewards/rejected": -0.9453425407409668,
      "step": 951
    },
    {
      "epoch": 0.5922239502332814,
      "grad_norm": 0.7648904323577881,
      "learning_rate": 2.6200000000000003e-05,
      "log_odds_chosen": 8.385675430297852,
      "log_odds_ratio": -0.21214787662029266,
      "logits/chosen": 0.18494370579719543,
      "logits/rejected": 0.8852918148040771,
      "logps/chosen": -1.3512593507766724,
      "logps/rejected": -9.508600234985352,
      "loss": 2.5537,
      "nll_loss": 2.532467842102051,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13512593507766724,
      "rewards/margins": 0.8157341480255127,
      "rewards/rejected": -0.9508601427078247,
      "step": 952
    },
    {
      "epoch": 0.592846034214619,
      "grad_norm": 0.4409056305885315,
      "learning_rate": 2.6175e-05,
      "log_odds_chosen": 19.01350212097168,
      "log_odds_ratio": -7.912763976491988e-06,
      "logits/chosen": 0.24696475267410278,
      "logits/rejected": 1.9544768333435059,
      "logps/chosen": -1.246018648147583,
      "logps/rejected": -19.90366554260254,
      "loss": 2.4758,
      "nll_loss": 2.4758312702178955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12460187077522278,
      "rewards/margins": 1.8657647371292114,
      "rewards/rejected": -1.9903665781021118,
      "step": 953
    },
    {
      "epoch": 0.5934681181959565,
      "grad_norm": 0.5151073336601257,
      "learning_rate": 2.6150000000000002e-05,
      "log_odds_chosen": 10.778078079223633,
      "log_odds_ratio": -0.29022905230522156,
      "logits/chosen": 0.13743232190608978,
      "logits/rejected": 0.6315884590148926,
      "logps/chosen": -1.1233041286468506,
      "logps/rejected": -11.37158203125,
      "loss": 2.9005,
      "nll_loss": 2.871509313583374,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11233042180538177,
      "rewards/margins": 1.0248278379440308,
      "rewards/rejected": -1.1371582746505737,
      "step": 954
    },
    {
      "epoch": 0.5940902021772939,
      "grad_norm": 0.3522619307041168,
      "learning_rate": 2.6124999999999998e-05,
      "log_odds_chosen": 11.523155212402344,
      "log_odds_ratio": -0.2107788622379303,
      "logits/chosen": 0.3925516903400421,
      "logits/rejected": 1.0518916845321655,
      "logps/chosen": -1.0370612144470215,
      "logps/rejected": -12.201189994812012,
      "loss": 3.6713,
      "nll_loss": 3.650195360183716,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10370612144470215,
      "rewards/margins": 1.116412878036499,
      "rewards/rejected": -1.2201189994812012,
      "step": 955
    },
    {
      "epoch": 0.5947122861586314,
      "grad_norm": 0.5082216262817383,
      "learning_rate": 2.61e-05,
      "log_odds_chosen": 16.058555603027344,
      "log_odds_ratio": -0.06356058269739151,
      "logits/chosen": 0.17834536731243134,
      "logits/rejected": 0.8007286190986633,
      "logps/chosen": -1.3070101737976074,
      "logps/rejected": -17.062992095947266,
      "loss": 3.2413,
      "nll_loss": 3.234963893890381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13070102035999298,
      "rewards/margins": 1.5755982398986816,
      "rewards/rejected": -1.7062993049621582,
      "step": 956
    },
    {
      "epoch": 0.5953343701399689,
      "grad_norm": 0.5702728629112244,
      "learning_rate": 2.6075e-05,
      "log_odds_chosen": 24.80580711364746,
      "log_odds_ratio": -0.021674897521734238,
      "logits/chosen": 0.2285652458667755,
      "logits/rejected": 1.1431223154067993,
      "logps/chosen": -1.0820637941360474,
      "logps/rejected": -25.419315338134766,
      "loss": 2.9305,
      "nll_loss": 2.9282896518707275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1082063838839531,
      "rewards/margins": 2.433724880218506,
      "rewards/rejected": -2.541931390762329,
      "step": 957
    },
    {
      "epoch": 0.5959564541213064,
      "grad_norm": 1.0141100883483887,
      "learning_rate": 2.6050000000000003e-05,
      "log_odds_chosen": 16.26883316040039,
      "log_odds_ratio": -0.3170120120048523,
      "logits/chosen": 0.3057654798030853,
      "logits/rejected": 1.2227212190628052,
      "logps/chosen": -1.0155651569366455,
      "logps/rejected": -16.994293212890625,
      "loss": 2.2965,
      "nll_loss": 2.264829158782959,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10155650973320007,
      "rewards/margins": 1.5978727340698242,
      "rewards/rejected": -1.6994292736053467,
      "step": 958
    },
    {
      "epoch": 0.5965785381026438,
      "grad_norm": 0.489462673664093,
      "learning_rate": 2.6025e-05,
      "log_odds_chosen": 19.681560516357422,
      "log_odds_ratio": -0.023856129497289658,
      "logits/chosen": 0.2779539227485657,
      "logits/rejected": 1.4224390983581543,
      "logps/chosen": -1.0721073150634766,
      "logps/rejected": -20.223628997802734,
      "loss": 2.8642,
      "nll_loss": 2.861823558807373,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10721074789762497,
      "rewards/margins": 1.9151520729064941,
      "rewards/rejected": -2.0223629474639893,
      "step": 959
    },
    {
      "epoch": 0.5972006220839814,
      "grad_norm": 0.5690145492553711,
      "learning_rate": 2.6000000000000002e-05,
      "log_odds_chosen": 16.70014190673828,
      "log_odds_ratio": -0.10952571779489517,
      "logits/chosen": 0.26871487498283386,
      "logits/rejected": 1.045303463935852,
      "logps/chosen": -1.7916289567947388,
      "logps/rejected": -18.297130584716797,
      "loss": 2.8808,
      "nll_loss": 2.869889736175537,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1791629046201706,
      "rewards/margins": 1.650550127029419,
      "rewards/rejected": -1.8297131061553955,
      "step": 960
    },
    {
      "epoch": 0.5978227060653188,
      "grad_norm": 0.37688279151916504,
      "learning_rate": 2.5974999999999998e-05,
      "log_odds_chosen": 11.515750885009766,
      "log_odds_ratio": -0.12808285653591156,
      "logits/chosen": 0.4044106602668762,
      "logits/rejected": 1.100311517715454,
      "logps/chosen": -1.228320837020874,
      "logps/rejected": -12.13768482208252,
      "loss": 3.5135,
      "nll_loss": 3.500669002532959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12283208966255188,
      "rewards/margins": 1.0909364223480225,
      "rewards/rejected": -1.213768482208252,
      "step": 961
    },
    {
      "epoch": 0.5984447900466563,
      "grad_norm": 0.5325865745544434,
      "learning_rate": 2.595e-05,
      "log_odds_chosen": 21.426223754882812,
      "log_odds_ratio": -0.13188225030899048,
      "logits/chosen": 0.30785199999809265,
      "logits/rejected": 1.3725229501724243,
      "logps/chosen": -1.0754218101501465,
      "logps/rejected": -21.913288116455078,
      "loss": 2.9752,
      "nll_loss": 2.9620540142059326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10754218697547913,
      "rewards/margins": 2.0837864875793457,
      "rewards/rejected": -2.191328525543213,
      "step": 962
    },
    {
      "epoch": 0.5990668740279937,
      "grad_norm": 0.5490668416023254,
      "learning_rate": 2.5925e-05,
      "log_odds_chosen": 9.045169830322266,
      "log_odds_ratio": -0.29519692063331604,
      "logits/chosen": 0.2381560206413269,
      "logits/rejected": 0.8757392764091492,
      "logps/chosen": -1.1203395128250122,
      "logps/rejected": -9.771276473999023,
      "loss": 2.7598,
      "nll_loss": 2.730231761932373,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11203394830226898,
      "rewards/margins": 0.8650937676429749,
      "rewards/rejected": -0.977127730846405,
      "step": 963
    },
    {
      "epoch": 0.5996889580093313,
      "grad_norm": 0.5232042074203491,
      "learning_rate": 2.5900000000000003e-05,
      "log_odds_chosen": 16.675994873046875,
      "log_odds_ratio": -0.07509205490350723,
      "logits/chosen": 0.24147437512874603,
      "logits/rejected": 1.0761547088623047,
      "logps/chosen": -0.8001296520233154,
      "logps/rejected": -16.835155487060547,
      "loss": 3.0022,
      "nll_loss": 2.9946541786193848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0800129622220993,
      "rewards/margins": 1.6035027503967285,
      "rewards/rejected": -1.6835157871246338,
      "step": 964
    },
    {
      "epoch": 0.6003110419906688,
      "grad_norm": 0.36291661858558655,
      "learning_rate": 2.5875e-05,
      "log_odds_chosen": 19.8973445892334,
      "log_odds_ratio": -0.022971786558628082,
      "logits/chosen": 0.07175733894109726,
      "logits/rejected": 0.8581687211990356,
      "logps/chosen": -0.9344284534454346,
      "logps/rejected": -20.3033390045166,
      "loss": 2.6604,
      "nll_loss": 2.658095359802246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09344284236431122,
      "rewards/margins": 1.9368910789489746,
      "rewards/rejected": -2.030333995819092,
      "step": 965
    },
    {
      "epoch": 0.6009331259720062,
      "grad_norm": 0.8470042943954468,
      "learning_rate": 2.585e-05,
      "log_odds_chosen": 16.980083465576172,
      "log_odds_ratio": -0.2519052028656006,
      "logits/chosen": 0.22023792564868927,
      "logits/rejected": 1.116905689239502,
      "logps/chosen": -1.0762019157409668,
      "logps/rejected": -17.686002731323242,
      "loss": 3.0433,
      "nll_loss": 3.0181312561035156,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10762019455432892,
      "rewards/margins": 1.6609801054000854,
      "rewards/rejected": -1.7686002254486084,
      "step": 966
    },
    {
      "epoch": 0.6015552099533437,
      "grad_norm": 0.6046644449234009,
      "learning_rate": 2.5824999999999998e-05,
      "log_odds_chosen": 12.115790367126465,
      "log_odds_ratio": -0.22894859313964844,
      "logits/chosen": 0.21607916057109833,
      "logits/rejected": 0.9050865173339844,
      "logps/chosen": -1.0518709421157837,
      "logps/rejected": -12.8179292678833,
      "loss": 2.7021,
      "nll_loss": 2.6792078018188477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10518710315227509,
      "rewards/margins": 1.1766059398651123,
      "rewards/rejected": -1.2817929983139038,
      "step": 967
    },
    {
      "epoch": 0.6021772939346812,
      "grad_norm": 0.5602324604988098,
      "learning_rate": 2.58e-05,
      "log_odds_chosen": 16.60848617553711,
      "log_odds_ratio": -0.16328565776348114,
      "logits/chosen": 0.2715965807437897,
      "logits/rejected": 1.2223488092422485,
      "logps/chosen": -1.2314372062683105,
      "logps/rejected": -17.56035804748535,
      "loss": 3.1499,
      "nll_loss": 3.1335926055908203,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12314371764659882,
      "rewards/margins": 1.63289213180542,
      "rewards/rejected": -1.7560360431671143,
      "step": 968
    },
    {
      "epoch": 0.6027993779160187,
      "grad_norm": 0.499213308095932,
      "learning_rate": 2.5775e-05,
      "log_odds_chosen": 9.463824272155762,
      "log_odds_ratio": -0.2594992518424988,
      "logits/chosen": 0.13768236339092255,
      "logits/rejected": 0.3776164650917053,
      "logps/chosen": -1.2077715396881104,
      "logps/rejected": -10.380818367004395,
      "loss": 3.2398,
      "nll_loss": 3.2138986587524414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12077715992927551,
      "rewards/margins": 0.9173046350479126,
      "rewards/rejected": -1.0380817651748657,
      "step": 969
    },
    {
      "epoch": 0.6034214618973561,
      "grad_norm": 0.44655492901802063,
      "learning_rate": 2.5750000000000002e-05,
      "log_odds_chosen": 19.190210342407227,
      "log_odds_ratio": -0.10930891335010529,
      "logits/chosen": 0.1695220023393631,
      "logits/rejected": 1.1123127937316895,
      "logps/chosen": -1.1357357501983643,
      "logps/rejected": -19.973918914794922,
      "loss": 2.8969,
      "nll_loss": 2.885981321334839,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1135735735297203,
      "rewards/margins": 1.8838183879852295,
      "rewards/rejected": -1.997391939163208,
      "step": 970
    },
    {
      "epoch": 0.6040435458786936,
      "grad_norm": 0.539529025554657,
      "learning_rate": 2.5725e-05,
      "log_odds_chosen": 18.83353042602539,
      "log_odds_ratio": -0.09447011351585388,
      "logits/chosen": 0.15006932616233826,
      "logits/rejected": 0.7248711585998535,
      "logps/chosen": -1.026084303855896,
      "logps/rejected": -19.30118179321289,
      "loss": 3.159,
      "nll_loss": 3.1495561599731445,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10260843485593796,
      "rewards/margins": 1.827509880065918,
      "rewards/rejected": -1.9301183223724365,
      "step": 971
    },
    {
      "epoch": 0.6046656298600311,
      "grad_norm": 5.969109058380127,
      "learning_rate": 2.57e-05,
      "log_odds_chosen": 14.402713775634766,
      "log_odds_ratio": -0.3919801414012909,
      "logits/chosen": 0.18785786628723145,
      "logits/rejected": 0.6937410235404968,
      "logps/chosen": -1.815807819366455,
      "logps/rejected": -15.869197845458984,
      "loss": 3.7518,
      "nll_loss": 3.712613821029663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1815807819366455,
      "rewards/margins": 1.4053387641906738,
      "rewards/rejected": -1.5869196653366089,
      "step": 972
    },
    {
      "epoch": 0.6052877138413686,
      "grad_norm": 0.48145902156829834,
      "learning_rate": 2.5675e-05,
      "log_odds_chosen": 18.73621940612793,
      "log_odds_ratio": -0.11193342506885529,
      "logits/chosen": 0.20852616429328918,
      "logits/rejected": 1.1668469905853271,
      "logps/chosen": -1.0225214958190918,
      "logps/rejected": -19.24210548400879,
      "loss": 2.9252,
      "nll_loss": 2.9140021800994873,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10225215554237366,
      "rewards/margins": 1.8219584226608276,
      "rewards/rejected": -1.924210548400879,
      "step": 973
    },
    {
      "epoch": 0.605909797822706,
      "grad_norm": 0.3980204463005066,
      "learning_rate": 2.5650000000000003e-05,
      "log_odds_chosen": 11.081676483154297,
      "log_odds_ratio": -0.08179325610399246,
      "logits/chosen": 0.26339149475097656,
      "logits/rejected": 0.9369683861732483,
      "logps/chosen": -0.9269077777862549,
      "logps/rejected": -11.361970901489258,
      "loss": 3.488,
      "nll_loss": 3.479806900024414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09269078075885773,
      "rewards/margins": 1.043506383895874,
      "rewards/rejected": -1.1361970901489258,
      "step": 974
    },
    {
      "epoch": 0.6065318818040435,
      "grad_norm": 0.5839635729789734,
      "learning_rate": 2.5625e-05,
      "log_odds_chosen": 13.500631332397461,
      "log_odds_ratio": -0.0822688415646553,
      "logits/chosen": 0.28204748034477234,
      "logits/rejected": 1.2430696487426758,
      "logps/chosen": -1.311071753501892,
      "logps/rejected": -14.472478866577148,
      "loss": 3.6005,
      "nll_loss": 3.59232234954834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1311071664094925,
      "rewards/margins": 1.3161406517028809,
      "rewards/rejected": -1.4472477436065674,
      "step": 975
    },
    {
      "epoch": 0.6071539657853811,
      "grad_norm": 0.44382327795028687,
      "learning_rate": 2.5600000000000002e-05,
      "log_odds_chosen": 18.092775344848633,
      "log_odds_ratio": -0.11017350852489471,
      "logits/chosen": 0.06789172440767288,
      "logits/rejected": 0.7348443865776062,
      "logps/chosen": -0.9244815111160278,
      "logps/rejected": -18.366724014282227,
      "loss": 3.2554,
      "nll_loss": 3.2444264888763428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0924481526017189,
      "rewards/margins": 1.744224190711975,
      "rewards/rejected": -1.8366724252700806,
      "step": 976
    },
    {
      "epoch": 0.6077760497667185,
      "grad_norm": 0.6069784164428711,
      "learning_rate": 2.5574999999999998e-05,
      "log_odds_chosen": 10.627729415893555,
      "log_odds_ratio": -0.12660586833953857,
      "logits/chosen": 0.0689058005809784,
      "logits/rejected": 0.5917812585830688,
      "logps/chosen": -0.9631859064102173,
      "logps/rejected": -11.07681655883789,
      "loss": 2.7384,
      "nll_loss": 2.7257091999053955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09631859511137009,
      "rewards/margins": 1.0113630294799805,
      "rewards/rejected": -1.1076816320419312,
      "step": 977
    },
    {
      "epoch": 0.608398133748056,
      "grad_norm": 0.45217904448509216,
      "learning_rate": 2.555e-05,
      "log_odds_chosen": 15.359528541564941,
      "log_odds_ratio": -0.21051090955734253,
      "logits/chosen": 0.065369613468647,
      "logits/rejected": 0.8507799506187439,
      "logps/chosen": -1.0331388711929321,
      "logps/rejected": -15.877238273620605,
      "loss": 2.6868,
      "nll_loss": 2.6657238006591797,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10331389307975769,
      "rewards/margins": 1.4844098091125488,
      "rewards/rejected": -1.587723731994629,
      "step": 978
    },
    {
      "epoch": 0.6090202177293935,
      "grad_norm": 0.5865066051483154,
      "learning_rate": 2.5525e-05,
      "log_odds_chosen": 17.618770599365234,
      "log_odds_ratio": -0.10912881791591644,
      "logits/chosen": 0.110966756939888,
      "logits/rejected": 1.0559561252593994,
      "logps/chosen": -1.1738271713256836,
      "logps/rejected": -18.374874114990234,
      "loss": 2.8958,
      "nll_loss": 2.8849360942840576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1173827201128006,
      "rewards/margins": 1.7201048135757446,
      "rewards/rejected": -1.8374874591827393,
      "step": 979
    },
    {
      "epoch": 0.609642301710731,
      "grad_norm": 0.4005695879459381,
      "learning_rate": 2.5500000000000003e-05,
      "log_odds_chosen": 15.089963912963867,
      "log_odds_ratio": -0.10706076771020889,
      "logits/chosen": 0.19660604000091553,
      "logits/rejected": 0.7421777844429016,
      "logps/chosen": -1.6507558822631836,
      "logps/rejected": -16.388992309570312,
      "loss": 3.6645,
      "nll_loss": 3.6537725925445557,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16507560014724731,
      "rewards/margins": 1.4738237857818604,
      "rewards/rejected": -1.638899326324463,
      "step": 980
    },
    {
      "epoch": 0.6102643856920684,
      "grad_norm": 0.49785563349723816,
      "learning_rate": 2.5475e-05,
      "log_odds_chosen": 11.623516082763672,
      "log_odds_ratio": -0.26292070746421814,
      "logits/chosen": 0.11350183188915253,
      "logits/rejected": 0.8784139752388,
      "logps/chosen": -1.347922682762146,
      "logps/rejected": -12.797706604003906,
      "loss": 2.894,
      "nll_loss": 2.867668390274048,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1347922682762146,
      "rewards/margins": 1.144978404045105,
      "rewards/rejected": -1.2797706127166748,
      "step": 981
    },
    {
      "epoch": 0.6108864696734059,
      "grad_norm": 0.4649457037448883,
      "learning_rate": 2.5450000000000002e-05,
      "log_odds_chosen": 20.90005874633789,
      "log_odds_ratio": -8.940698137394065e-08,
      "logits/chosen": 0.21456483006477356,
      "logits/rejected": 1.568105697631836,
      "logps/chosen": -1.1345700025558472,
      "logps/rejected": -21.52691650390625,
      "loss": 3.2944,
      "nll_loss": 3.2943665981292725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11345699429512024,
      "rewards/margins": 2.0392348766326904,
      "rewards/rejected": -2.1526918411254883,
      "step": 982
    },
    {
      "epoch": 0.6115085536547434,
      "grad_norm": 0.5988860130310059,
      "learning_rate": 2.5424999999999998e-05,
      "log_odds_chosen": 10.66419792175293,
      "log_odds_ratio": -0.3058769106864929,
      "logits/chosen": 0.05050152167677879,
      "logits/rejected": 0.2465941309928894,
      "logps/chosen": -0.8942004442214966,
      "logps/rejected": -11.015356063842773,
      "loss": 3.0743,
      "nll_loss": 3.0436720848083496,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08942003548145294,
      "rewards/margins": 1.012115716934204,
      "rewards/rejected": -1.101535677909851,
      "step": 983
    },
    {
      "epoch": 0.6121306376360809,
      "grad_norm": 0.6072388887405396,
      "learning_rate": 2.54e-05,
      "log_odds_chosen": 16.32722282409668,
      "log_odds_ratio": -0.11603593826293945,
      "logits/chosen": 0.12268135696649551,
      "logits/rejected": 1.0790374279022217,
      "logps/chosen": -1.3971768617630005,
      "logps/rejected": -17.381330490112305,
      "loss": 2.7476,
      "nll_loss": 2.735957384109497,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1397176831960678,
      "rewards/margins": 1.5984153747558594,
      "rewards/rejected": -1.7381330728530884,
      "step": 984
    },
    {
      "epoch": 0.6127527216174183,
      "grad_norm": 0.4394105076789856,
      "learning_rate": 2.5375e-05,
      "log_odds_chosen": 20.815372467041016,
      "log_odds_ratio": -0.11913128942251205,
      "logits/chosen": 0.2849823832511902,
      "logits/rejected": 1.7651139497756958,
      "logps/chosen": -1.3435211181640625,
      "logps/rejected": -21.902217864990234,
      "loss": 3.1595,
      "nll_loss": 3.1476199626922607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13435211777687073,
      "rewards/margins": 2.0558695793151855,
      "rewards/rejected": -2.1902217864990234,
      "step": 985
    },
    {
      "epoch": 0.6133748055987558,
      "grad_norm": 0.4176173210144043,
      "learning_rate": 2.5350000000000003e-05,
      "log_odds_chosen": 20.118194580078125,
      "log_odds_ratio": -0.041579604148864746,
      "logits/chosen": 0.2213955968618393,
      "logits/rejected": 1.2255231142044067,
      "logps/chosen": -1.022838830947876,
      "logps/rejected": -20.671480178833008,
      "loss": 3.4104,
      "nll_loss": 3.406257390975952,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10228388756513596,
      "rewards/margins": 1.9648641347885132,
      "rewards/rejected": -2.067148208618164,
      "step": 986
    },
    {
      "epoch": 0.6139968895800934,
      "grad_norm": 0.5290238261222839,
      "learning_rate": 2.5325e-05,
      "log_odds_chosen": 15.454238891601562,
      "log_odds_ratio": -0.09638696908950806,
      "logits/chosen": 0.2696585953235626,
      "logits/rejected": 1.1312285661697388,
      "logps/chosen": -1.2690268754959106,
      "logps/rejected": -16.407495498657227,
      "loss": 3.1839,
      "nll_loss": 3.174257755279541,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12690268456935883,
      "rewards/margins": 1.51384699344635,
      "rewards/rejected": -1.6407495737075806,
      "step": 987
    },
    {
      "epoch": 0.6146189735614308,
      "grad_norm": 0.44811832904815674,
      "learning_rate": 2.5300000000000002e-05,
      "log_odds_chosen": 18.196022033691406,
      "log_odds_ratio": -0.0027530835941433907,
      "logits/chosen": 0.306151807308197,
      "logits/rejected": 1.4259248971939087,
      "logps/chosen": -1.3465137481689453,
      "logps/rejected": -19.213729858398438,
      "loss": 3.4642,
      "nll_loss": 3.463940143585205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13465136289596558,
      "rewards/margins": 1.7867215871810913,
      "rewards/rejected": -1.9213731288909912,
      "step": 988
    },
    {
      "epoch": 0.6152410575427683,
      "grad_norm": 0.5820187330245972,
      "learning_rate": 2.5274999999999998e-05,
      "log_odds_chosen": 13.781078338623047,
      "log_odds_ratio": -0.19668269157409668,
      "logits/chosen": -0.0342557318508625,
      "logits/rejected": 0.30851006507873535,
      "logps/chosen": -0.8840237855911255,
      "logps/rejected": -14.013053894042969,
      "loss": 2.5317,
      "nll_loss": 2.5119993686676025,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08840238302946091,
      "rewards/margins": 1.3129030466079712,
      "rewards/rejected": -1.4013054370880127,
      "step": 989
    },
    {
      "epoch": 0.6158631415241057,
      "grad_norm": 0.5693361163139343,
      "learning_rate": 2.525e-05,
      "log_odds_chosen": 9.703099250793457,
      "log_odds_ratio": -0.2681022882461548,
      "logits/chosen": 0.00022461498156189919,
      "logits/rejected": 0.3866625428199768,
      "logps/chosen": -1.0230138301849365,
      "logps/rejected": -10.349815368652344,
      "loss": 2.8994,
      "nll_loss": 2.8725709915161133,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10230138897895813,
      "rewards/margins": 0.9326801300048828,
      "rewards/rejected": -1.0349814891815186,
      "step": 990
    },
    {
      "epoch": 0.6164852255054433,
      "grad_norm": 0.5749667882919312,
      "learning_rate": 2.5225e-05,
      "log_odds_chosen": 10.241193771362305,
      "log_odds_ratio": -0.3188455104827881,
      "logits/chosen": 0.07506673783063889,
      "logits/rejected": 0.46898865699768066,
      "logps/chosen": -1.2021185159683228,
      "logps/rejected": -11.231522560119629,
      "loss": 2.8334,
      "nll_loss": 2.8015153408050537,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12021185457706451,
      "rewards/margins": 1.0029405355453491,
      "rewards/rejected": -1.1231523752212524,
      "step": 991
    },
    {
      "epoch": 0.6171073094867807,
      "grad_norm": 0.6073411107063293,
      "learning_rate": 2.5200000000000003e-05,
      "log_odds_chosen": 11.625310897827148,
      "log_odds_ratio": -0.19626720249652863,
      "logits/chosen": 0.2134128212928772,
      "logits/rejected": 0.8630800247192383,
      "logps/chosen": -0.7396443486213684,
      "logps/rejected": -11.635083198547363,
      "loss": 3.168,
      "nll_loss": 3.148395538330078,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0739644318819046,
      "rewards/margins": 1.0895440578460693,
      "rewards/rejected": -1.163508415222168,
      "step": 992
    },
    {
      "epoch": 0.6177293934681182,
      "grad_norm": 0.5637009143829346,
      "learning_rate": 2.5175e-05,
      "log_odds_chosen": 8.45470142364502,
      "log_odds_ratio": -0.38158631324768066,
      "logits/chosen": 0.02333865687251091,
      "logits/rejected": 0.4010867476463318,
      "logps/chosen": -1.1170642375946045,
      "logps/rejected": -9.288888931274414,
      "loss": 2.8206,
      "nll_loss": 2.7824807167053223,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11170642077922821,
      "rewards/margins": 0.8171824216842651,
      "rewards/rejected": -0.9288888573646545,
      "step": 993
    },
    {
      "epoch": 0.6183514774494556,
      "grad_norm": 0.5053258538246155,
      "learning_rate": 2.515e-05,
      "log_odds_chosen": 21.61244773864746,
      "log_odds_ratio": -0.11922389268875122,
      "logits/chosen": 0.281334787607193,
      "logits/rejected": 1.7507303953170776,
      "logps/chosen": -1.2235077619552612,
      "logps/rejected": -22.507673263549805,
      "loss": 2.9003,
      "nll_loss": 2.88834810256958,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1223507821559906,
      "rewards/margins": 2.1284167766571045,
      "rewards/rejected": -2.250767469406128,
      "step": 994
    },
    {
      "epoch": 0.6189735614307932,
      "grad_norm": 0.4218065142631531,
      "learning_rate": 2.5124999999999997e-05,
      "log_odds_chosen": 13.609810829162598,
      "log_odds_ratio": -0.17529192566871643,
      "logits/chosen": 0.20690587162971497,
      "logits/rejected": 0.9557357430458069,
      "logps/chosen": -1.2656549215316772,
      "logps/rejected": -14.553084373474121,
      "loss": 3.604,
      "nll_loss": 3.5864503383636475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12656548619270325,
      "rewards/margins": 1.3287428617477417,
      "rewards/rejected": -1.455308437347412,
      "step": 995
    },
    {
      "epoch": 0.6195956454121306,
      "grad_norm": 0.6388038396835327,
      "learning_rate": 2.51e-05,
      "log_odds_chosen": 13.46223258972168,
      "log_odds_ratio": -0.28370407223701477,
      "logits/chosen": 0.19488823413848877,
      "logits/rejected": 0.7138241529464722,
      "logps/chosen": -1.0961161851882935,
      "logps/rejected": -14.22940444946289,
      "loss": 3.1086,
      "nll_loss": 3.0802266597747803,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1096116155385971,
      "rewards/margins": 1.3133288621902466,
      "rewards/rejected": -1.4229404926300049,
      "step": 996
    },
    {
      "epoch": 0.6202177293934681,
      "grad_norm": 0.49505603313446045,
      "learning_rate": 2.5075e-05,
      "log_odds_chosen": 9.735605239868164,
      "log_odds_ratio": -0.21508948504924774,
      "logits/chosen": 0.14490671455860138,
      "logits/rejected": 0.45840179920196533,
      "logps/chosen": -1.259823203086853,
      "logps/rejected": -10.751209259033203,
      "loss": 3.2765,
      "nll_loss": 3.254956007003784,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12598231434822083,
      "rewards/margins": 0.9491385221481323,
      "rewards/rejected": -1.0751209259033203,
      "step": 997
    },
    {
      "epoch": 0.6208398133748056,
      "grad_norm": 0.5221970677375793,
      "learning_rate": 2.5050000000000002e-05,
      "log_odds_chosen": 17.351388931274414,
      "log_odds_ratio": -0.0899883359670639,
      "logits/chosen": 0.22275736927986145,
      "logits/rejected": 0.9023792743682861,
      "logps/chosen": -1.104635238647461,
      "logps/rejected": -18.011194229125977,
      "loss": 3.4768,
      "nll_loss": 3.467817783355713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11046352982521057,
      "rewards/margins": 1.690656065940857,
      "rewards/rejected": -1.8011195659637451,
      "step": 998
    },
    {
      "epoch": 0.6214618973561431,
      "grad_norm": 0.5278105735778809,
      "learning_rate": 2.5025e-05,
      "log_odds_chosen": 19.447450637817383,
      "log_odds_ratio": -0.1714281141757965,
      "logits/chosen": 0.2100244164466858,
      "logits/rejected": 1.0468909740447998,
      "logps/chosen": -0.9524563550949097,
      "logps/rejected": -19.773094177246094,
      "loss": 3.0782,
      "nll_loss": 3.06109619140625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09524564445018768,
      "rewards/margins": 1.882063865661621,
      "rewards/rejected": -1.9773094654083252,
      "step": 999
    },
    {
      "epoch": 0.6220839813374806,
      "grad_norm": 0.7394985556602478,
      "learning_rate": 2.5e-05,
      "log_odds_chosen": 24.217151641845703,
      "log_odds_ratio": -0.11016383767127991,
      "logits/chosen": 0.09682357311248779,
      "logits/rejected": 1.188527226448059,
      "logps/chosen": -1.3372222185134888,
      "logps/rejected": -25.23487091064453,
      "loss": 2.2574,
      "nll_loss": 2.2463905811309814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1337222158908844,
      "rewards/margins": 2.3897650241851807,
      "rewards/rejected": -2.5234873294830322,
      "step": 1000
    },
    {
      "epoch": 0.622706065318818,
      "grad_norm": 10.445813179016113,
      "learning_rate": 2.4975e-05,
      "log_odds_chosen": 25.370601654052734,
      "log_odds_ratio": -0.0863175019621849,
      "logits/chosen": 0.23290672898292542,
      "logits/rejected": 1.5314041376113892,
      "logps/chosen": -1.8342373371124268,
      "logps/rejected": -26.98197364807129,
      "loss": 2.9581,
      "nll_loss": 2.949495315551758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1834237426519394,
      "rewards/margins": 2.514773368835449,
      "rewards/rejected": -2.69819712638855,
      "step": 1001
    },
    {
      "epoch": 0.6233281493001556,
      "grad_norm": 0.9837327599525452,
      "learning_rate": 2.495e-05,
      "log_odds_chosen": 26.120695114135742,
      "log_odds_ratio": -8.239331509685144e-05,
      "logits/chosen": 0.21516145765781403,
      "logits/rejected": 1.3835011720657349,
      "logps/chosen": -1.1667652130126953,
      "logps/rejected": -26.76409912109375,
      "loss": 3.2298,
      "nll_loss": 3.2298390865325928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11667650938034058,
      "rewards/margins": 2.5597333908081055,
      "rewards/rejected": -2.6764097213745117,
      "step": 1002
    },
    {
      "epoch": 0.623950233281493,
      "grad_norm": 0.5452992916107178,
      "learning_rate": 2.4925000000000003e-05,
      "log_odds_chosen": 11.288724899291992,
      "log_odds_ratio": -0.3280940353870392,
      "logits/chosen": 0.02780914306640625,
      "logits/rejected": 0.698295533657074,
      "logps/chosen": -1.4289721250534058,
      "logps/rejected": -12.558980941772461,
      "loss": 2.5772,
      "nll_loss": 2.54435658454895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14289721846580505,
      "rewards/margins": 1.1130008697509766,
      "rewards/rejected": -1.2558979988098145,
      "step": 1003
    },
    {
      "epoch": 0.6245723172628305,
      "grad_norm": 1.2315927743911743,
      "learning_rate": 2.4900000000000002e-05,
      "log_odds_chosen": 21.15532684326172,
      "log_odds_ratio": -0.2190973460674286,
      "logits/chosen": 0.11318109184503555,
      "logits/rejected": 1.183733582496643,
      "logps/chosen": -1.5021014213562012,
      "logps/rejected": -22.3739070892334,
      "loss": 2.6072,
      "nll_loss": 2.5852890014648438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1502101570367813,
      "rewards/margins": 2.0871806144714355,
      "rewards/rejected": -2.2373907566070557,
      "step": 1004
    },
    {
      "epoch": 0.6251944012441679,
      "grad_norm": 0.5702396631240845,
      "learning_rate": 2.4875e-05,
      "log_odds_chosen": 18.833229064941406,
      "log_odds_ratio": -0.07586246728897095,
      "logits/chosen": -0.06201568990945816,
      "logits/rejected": 1.269275426864624,
      "logps/chosen": -1.2848140001296997,
      "logps/rejected": -19.835512161254883,
      "loss": 2.0326,
      "nll_loss": 2.0250072479248047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1284814029932022,
      "rewards/margins": 1.855069875717163,
      "rewards/rejected": -1.9835513830184937,
      "step": 1005
    },
    {
      "epoch": 0.6258164852255055,
      "grad_norm": 0.5604910850524902,
      "learning_rate": 2.485e-05,
      "log_odds_chosen": 12.637853622436523,
      "log_odds_ratio": -0.08352010697126389,
      "logits/chosen": 0.0846133753657341,
      "logits/rejected": 0.6342355608940125,
      "logps/chosen": -0.9381386637687683,
      "logps/rejected": -12.998655319213867,
      "loss": 2.849,
      "nll_loss": 2.8406927585601807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09381386637687683,
      "rewards/margins": 1.2060517072677612,
      "rewards/rejected": -1.299865484237671,
      "step": 1006
    },
    {
      "epoch": 0.6264385692068429,
      "grad_norm": 0.42290839552879333,
      "learning_rate": 2.4825e-05,
      "log_odds_chosen": 11.835851669311523,
      "log_odds_ratio": -0.1617979258298874,
      "logits/chosen": 0.10653581470251083,
      "logits/rejected": 0.5529526472091675,
      "logps/chosen": -1.1142702102661133,
      "logps/rejected": -12.558614730834961,
      "loss": 3.1891,
      "nll_loss": 3.1729464530944824,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11142702400684357,
      "rewards/margins": 1.1444344520568848,
      "rewards/rejected": -1.255861520767212,
      "step": 1007
    },
    {
      "epoch": 0.6270606531881804,
      "grad_norm": 0.48082074522972107,
      "learning_rate": 2.48e-05,
      "log_odds_chosen": 20.665428161621094,
      "log_odds_ratio": -0.0005173269310034811,
      "logits/chosen": 0.2348843216896057,
      "logits/rejected": 1.3875151872634888,
      "logps/chosen": -1.0887088775634766,
      "logps/rejected": -21.299148559570312,
      "loss": 3.3537,
      "nll_loss": 3.3536720275878906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10887089371681213,
      "rewards/margins": 2.0210437774658203,
      "rewards/rejected": -2.1299147605895996,
      "step": 1008
    },
    {
      "epoch": 0.6276827371695178,
      "grad_norm": 0.589474618434906,
      "learning_rate": 2.4775000000000003e-05,
      "log_odds_chosen": 8.409334182739258,
      "log_odds_ratio": -0.2995333969593048,
      "logits/chosen": 0.09103868901729584,
      "logits/rejected": 0.5081114768981934,
      "logps/chosen": -1.3712034225463867,
      "logps/rejected": -9.455653190612793,
      "loss": 3.2591,
      "nll_loss": 3.2291390895843506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1371203362941742,
      "rewards/margins": 0.8084449768066406,
      "rewards/rejected": -0.9455653429031372,
      "step": 1009
    },
    {
      "epoch": 0.6283048211508554,
      "grad_norm": 1.1940956115722656,
      "learning_rate": 2.4750000000000002e-05,
      "log_odds_chosen": 11.112287521362305,
      "log_odds_ratio": -0.17824390530586243,
      "logits/chosen": 0.030925815925002098,
      "logits/rejected": 0.5977746248245239,
      "logps/chosen": -1.2858790159225464,
      "logps/rejected": -12.050931930541992,
      "loss": 3.1955,
      "nll_loss": 3.177677869796753,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12858790159225464,
      "rewards/margins": 1.0765053033828735,
      "rewards/rejected": -1.2050931453704834,
      "step": 1010
    },
    {
      "epoch": 0.6289269051321928,
      "grad_norm": 0.40679919719696045,
      "learning_rate": 2.4725e-05,
      "log_odds_chosen": 19.506404876708984,
      "log_odds_ratio": -0.21070463955402374,
      "logits/chosen": 0.08191069960594177,
      "logits/rejected": 0.7757222652435303,
      "logps/chosen": -0.983881950378418,
      "logps/rejected": -20.12225341796875,
      "loss": 2.9743,
      "nll_loss": 2.953184127807617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0983881950378418,
      "rewards/margins": 1.9138373136520386,
      "rewards/rejected": -2.012225389480591,
      "step": 1011
    },
    {
      "epoch": 0.6295489891135303,
      "grad_norm": 1.9736859798431396,
      "learning_rate": 2.47e-05,
      "log_odds_chosen": 12.312723159790039,
      "log_odds_ratio": -0.2725464701652527,
      "logits/chosen": 0.09264761954545975,
      "logits/rejected": 0.7018003463745117,
      "logps/chosen": -1.1806060075759888,
      "logps/rejected": -13.192741394042969,
      "loss": 2.6436,
      "nll_loss": 2.6163501739501953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11806060373783112,
      "rewards/margins": 1.2012135982513428,
      "rewards/rejected": -1.3192741870880127,
      "step": 1012
    },
    {
      "epoch": 0.6301710730948679,
      "grad_norm": 0.5497341156005859,
      "learning_rate": 2.4675e-05,
      "log_odds_chosen": 12.800125122070312,
      "log_odds_ratio": -0.133247509598732,
      "logits/chosen": 0.07118834555149078,
      "logits/rejected": 1.0009067058563232,
      "logps/chosen": -1.0593130588531494,
      "logps/rejected": -13.403684616088867,
      "loss": 2.8056,
      "nll_loss": 2.792227029800415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10593131929636002,
      "rewards/margins": 1.2344372272491455,
      "rewards/rejected": -1.3403685092926025,
      "step": 1013
    },
    {
      "epoch": 0.6307931570762053,
      "grad_norm": 0.4955105185508728,
      "learning_rate": 2.465e-05,
      "log_odds_chosen": 15.762319564819336,
      "log_odds_ratio": -0.16057822108268738,
      "logits/chosen": 0.27589911222457886,
      "logits/rejected": 1.0252506732940674,
      "logps/chosen": -1.2522268295288086,
      "logps/rejected": -16.704978942871094,
      "loss": 3.5937,
      "nll_loss": 3.5776491165161133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12522268295288086,
      "rewards/margins": 1.545275092124939,
      "rewards/rejected": -1.6704977750778198,
      "step": 1014
    },
    {
      "epoch": 0.6314152410575428,
      "grad_norm": 0.4815879464149475,
      "learning_rate": 2.4625000000000002e-05,
      "log_odds_chosen": 16.962005615234375,
      "log_odds_ratio": -0.012883543968200684,
      "logits/chosen": 0.05764898285269737,
      "logits/rejected": 0.8884307742118835,
      "logps/chosen": -1.1525784730911255,
      "logps/rejected": -17.727672576904297,
      "loss": 2.7473,
      "nll_loss": 2.7459635734558105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11525784432888031,
      "rewards/margins": 1.6575093269348145,
      "rewards/rejected": -1.7727673053741455,
      "step": 1015
    },
    {
      "epoch": 0.6320373250388802,
      "grad_norm": 0.48324769735336304,
      "learning_rate": 2.46e-05,
      "log_odds_chosen": 16.998523712158203,
      "log_odds_ratio": -0.06444693356752396,
      "logits/chosen": 0.09633009135723114,
      "logits/rejected": 0.7852844595909119,
      "logps/chosen": -1.0850187540054321,
      "logps/rejected": -17.69282341003418,
      "loss": 2.8924,
      "nll_loss": 2.88600492477417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10850188136100769,
      "rewards/margins": 1.660780429840088,
      "rewards/rejected": -1.769282341003418,
      "step": 1016
    },
    {
      "epoch": 0.6326594090202178,
      "grad_norm": 0.7034952044487,
      "learning_rate": 2.4575e-05,
      "log_odds_chosen": 10.237931251525879,
      "log_odds_ratio": -0.2111145257949829,
      "logits/chosen": 0.10851834714412689,
      "logits/rejected": 0.8848839998245239,
      "logps/chosen": -1.2556936740875244,
      "logps/rejected": -11.173830032348633,
      "loss": 2.6084,
      "nll_loss": 2.5872819423675537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12556937336921692,
      "rewards/margins": 0.9918137192726135,
      "rewards/rejected": -1.1173830032348633,
      "step": 1017
    },
    {
      "epoch": 0.6332814930015552,
      "grad_norm": 0.4607456922531128,
      "learning_rate": 2.455e-05,
      "log_odds_chosen": 18.348682403564453,
      "log_odds_ratio": -0.001183602144010365,
      "logits/chosen": 0.18806810677051544,
      "logits/rejected": 1.8798189163208008,
      "logps/chosen": -1.3022570610046387,
      "logps/rejected": -19.317102432250977,
      "loss": 3.0541,
      "nll_loss": 3.0539772510528564,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13022570312023163,
      "rewards/margins": 1.8014845848083496,
      "rewards/rejected": -1.9317102432250977,
      "step": 1018
    },
    {
      "epoch": 0.6339035769828927,
      "grad_norm": 0.5187017917633057,
      "learning_rate": 2.4525e-05,
      "log_odds_chosen": 22.93252944946289,
      "log_odds_ratio": -4.4703490686970326e-08,
      "logits/chosen": 0.1605634093284607,
      "logits/rejected": 1.8251597881317139,
      "logps/chosen": -1.1265020370483398,
      "logps/rejected": -23.65814971923828,
      "loss": 2.4123,
      "nll_loss": 2.4122743606567383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11265021562576294,
      "rewards/margins": 2.2531650066375732,
      "rewards/rejected": -2.3658151626586914,
      "step": 1019
    },
    {
      "epoch": 0.6345256609642301,
      "grad_norm": 0.5116434097290039,
      "learning_rate": 2.45e-05,
      "log_odds_chosen": 17.80671501159668,
      "log_odds_ratio": -0.2849065065383911,
      "logits/chosen": 0.10839949548244476,
      "logits/rejected": 1.3892898559570312,
      "logps/chosen": -1.1734644174575806,
      "logps/rejected": -18.635705947875977,
      "loss": 3.038,
      "nll_loss": 3.009539842605591,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11734645068645477,
      "rewards/margins": 1.7462241649627686,
      "rewards/rejected": -1.8635706901550293,
      "step": 1020
    },
    {
      "epoch": 0.6351477449455677,
      "grad_norm": 0.5411016345024109,
      "learning_rate": 2.4475000000000002e-05,
      "log_odds_chosen": 14.560041427612305,
      "log_odds_ratio": -0.23231032490730286,
      "logits/chosen": 0.20895469188690186,
      "logits/rejected": 1.1282305717468262,
      "logps/chosen": -0.9110753536224365,
      "logps/rejected": -15.089509963989258,
      "loss": 3.0198,
      "nll_loss": 2.9965720176696777,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09110753983259201,
      "rewards/margins": 1.4178435802459717,
      "rewards/rejected": -1.50895094871521,
      "step": 1021
    },
    {
      "epoch": 0.6357698289269051,
      "grad_norm": 0.6156372427940369,
      "learning_rate": 2.445e-05,
      "log_odds_chosen": 16.500022888183594,
      "log_odds_ratio": -0.12201125919818878,
      "logits/chosen": 0.19160394370555878,
      "logits/rejected": 0.9838314652442932,
      "logps/chosen": -1.152182936668396,
      "logps/rejected": -17.30032730102539,
      "loss": 2.4539,
      "nll_loss": 2.441667318344116,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11521830409765244,
      "rewards/margins": 1.6148146390914917,
      "rewards/rejected": -1.7300329208374023,
      "step": 1022
    },
    {
      "epoch": 0.6363919129082426,
      "grad_norm": 0.437686949968338,
      "learning_rate": 2.4425e-05,
      "log_odds_chosen": 9.71164608001709,
      "log_odds_ratio": -0.1439571976661682,
      "logits/chosen": 0.051395781338214874,
      "logits/rejected": 0.7666381001472473,
      "logps/chosen": -1.4583333730697632,
      "logps/rejected": -10.898263931274414,
      "loss": 2.44,
      "nll_loss": 2.4255826473236084,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1458333432674408,
      "rewards/margins": 0.9439931511878967,
      "rewards/rejected": -1.0898264646530151,
      "step": 1023
    },
    {
      "epoch": 0.63701399688958,
      "grad_norm": 0.42805302143096924,
      "learning_rate": 2.44e-05,
      "log_odds_chosen": 16.866641998291016,
      "log_odds_ratio": -0.11649006605148315,
      "logits/chosen": 0.17741787433624268,
      "logits/rejected": 1.0520137548446655,
      "logps/chosen": -1.4884114265441895,
      "logps/rejected": -18.07642364501953,
      "loss": 3.1272,
      "nll_loss": 3.1155271530151367,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14884114265441895,
      "rewards/margins": 1.6588011980056763,
      "rewards/rejected": -1.8076422214508057,
      "step": 1024
    },
    {
      "epoch": 0.6376360808709176,
      "grad_norm": 0.4942615330219269,
      "learning_rate": 2.4375e-05,
      "log_odds_chosen": 20.236421585083008,
      "log_odds_ratio": -0.0003126816882286221,
      "logits/chosen": 0.3794230818748474,
      "logits/rejected": 1.705965518951416,
      "logps/chosen": -1.119978427886963,
      "logps/rejected": -20.884624481201172,
      "loss": 3.1617,
      "nll_loss": 3.1616899967193604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11199785023927689,
      "rewards/margins": 1.9764647483825684,
      "rewards/rejected": -2.0884625911712646,
      "step": 1025
    },
    {
      "epoch": 0.6382581648522551,
      "grad_norm": 0.5315221548080444,
      "learning_rate": 2.435e-05,
      "log_odds_chosen": 17.677621841430664,
      "log_odds_ratio": -0.1995067596435547,
      "logits/chosen": 0.1513439416885376,
      "logits/rejected": 1.3360419273376465,
      "logps/chosen": -1.2765010595321655,
      "logps/rejected": -18.678483963012695,
      "loss": 2.5653,
      "nll_loss": 2.545377016067505,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12765009701251984,
      "rewards/margins": 1.7401981353759766,
      "rewards/rejected": -1.8678483963012695,
      "step": 1026
    },
    {
      "epoch": 0.6388802488335925,
      "grad_norm": 1.1557024717330933,
      "learning_rate": 2.4325000000000002e-05,
      "log_odds_chosen": 19.117094039916992,
      "log_odds_ratio": -0.08928176760673523,
      "logits/chosen": 0.2427656352519989,
      "logits/rejected": 1.4793113470077515,
      "logps/chosen": -1.402268409729004,
      "logps/rejected": -20.209617614746094,
      "loss": 2.912,
      "nll_loss": 2.9031004905700684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1402268409729004,
      "rewards/margins": 1.880735158920288,
      "rewards/rejected": -2.0209617614746094,
      "step": 1027
    },
    {
      "epoch": 0.63950233281493,
      "grad_norm": 0.6319509744644165,
      "learning_rate": 2.43e-05,
      "log_odds_chosen": 21.61825942993164,
      "log_odds_ratio": -0.09945891797542572,
      "logits/chosen": 0.10278205573558807,
      "logits/rejected": 0.8583885431289673,
      "logps/chosen": -1.3808908462524414,
      "logps/rejected": -22.713088989257812,
      "loss": 2.7508,
      "nll_loss": 2.7408299446105957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13808909058570862,
      "rewards/margins": 2.1332197189331055,
      "rewards/rejected": -2.2713088989257812,
      "step": 1028
    },
    {
      "epoch": 0.6401244167962675,
      "grad_norm": 0.4444740116596222,
      "learning_rate": 2.4275e-05,
      "log_odds_chosen": 20.923830032348633,
      "log_odds_ratio": -0.1124124825000763,
      "logits/chosen": 0.09198861569166183,
      "logits/rejected": 0.6280720829963684,
      "logps/chosen": -1.0244150161743164,
      "logps/rejected": -21.289560317993164,
      "loss": 2.8599,
      "nll_loss": 2.848630905151367,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10244148969650269,
      "rewards/margins": 2.0265142917633057,
      "rewards/rejected": -2.1289560794830322,
      "step": 1029
    },
    {
      "epoch": 0.640746500777605,
      "grad_norm": 0.4391319751739502,
      "learning_rate": 2.425e-05,
      "log_odds_chosen": 17.82830810546875,
      "log_odds_ratio": -0.20137743651866913,
      "logits/chosen": 0.4636791944503784,
      "logits/rejected": 1.4730623960494995,
      "logps/chosen": -1.35006582736969,
      "logps/rejected": -18.960269927978516,
      "loss": 3.6925,
      "nll_loss": 3.672400951385498,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13500657677650452,
      "rewards/margins": 1.761020541191101,
      "rewards/rejected": -1.8960270881652832,
      "step": 1030
    },
    {
      "epoch": 0.6413685847589424,
      "grad_norm": 5.313420295715332,
      "learning_rate": 2.4225e-05,
      "log_odds_chosen": 16.11229705810547,
      "log_odds_ratio": -0.06506342440843582,
      "logits/chosen": 0.2195087969303131,
      "logits/rejected": 0.9584240317344666,
      "logps/chosen": -2.420592784881592,
      "logps/rejected": -18.16111183166504,
      "loss": 3.0274,
      "nll_loss": 3.020932912826538,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.24205929040908813,
      "rewards/margins": 1.574051856994629,
      "rewards/rejected": -1.8161113262176514,
      "step": 1031
    },
    {
      "epoch": 0.64199066874028,
      "grad_norm": 0.40007975697517395,
      "learning_rate": 2.4200000000000002e-05,
      "log_odds_chosen": 15.192806243896484,
      "log_odds_ratio": -0.3525593876838684,
      "logits/chosen": 0.2354370653629303,
      "logits/rejected": 1.27947199344635,
      "logps/chosen": -1.2368344068527222,
      "logps/rejected": -16.266971588134766,
      "loss": 3.1313,
      "nll_loss": 3.096068859100342,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12368344515562057,
      "rewards/margins": 1.5030137300491333,
      "rewards/rejected": -1.6266971826553345,
      "step": 1032
    },
    {
      "epoch": 0.6426127527216174,
      "grad_norm": 0.5552942156791687,
      "learning_rate": 2.4175e-05,
      "log_odds_chosen": 15.988584518432617,
      "log_odds_ratio": -0.07936682552099228,
      "logits/chosen": 0.10833962261676788,
      "logits/rejected": 0.8231009244918823,
      "logps/chosen": -0.965819239616394,
      "logps/rejected": -16.373872756958008,
      "loss": 2.7753,
      "nll_loss": 2.7673282623291016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09658192843198776,
      "rewards/margins": 1.5408053398132324,
      "rewards/rejected": -1.6373872756958008,
      "step": 1033
    },
    {
      "epoch": 0.6432348367029549,
      "grad_norm": 0.4293906092643738,
      "learning_rate": 2.415e-05,
      "log_odds_chosen": 26.196657180786133,
      "log_odds_ratio": -0.0685395896434784,
      "logits/chosen": 0.19044487178325653,
      "logits/rejected": 1.0367000102996826,
      "logps/chosen": -0.9686414003372192,
      "logps/rejected": -26.55857276916504,
      "loss": 3.0572,
      "nll_loss": 3.050372838973999,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09686414152383804,
      "rewards/margins": 2.558993101119995,
      "rewards/rejected": -2.6558570861816406,
      "step": 1034
    },
    {
      "epoch": 0.6438569206842923,
      "grad_norm": 0.5134528279304504,
      "learning_rate": 2.4125e-05,
      "log_odds_chosen": 22.512725830078125,
      "log_odds_ratio": -1.673509723332245e-05,
      "logits/chosen": 0.03580861911177635,
      "logits/rejected": 0.7297523021697998,
      "logps/chosen": -1.1147419214248657,
      "logps/rejected": -23.143142700195312,
      "loss": 2.7376,
      "nll_loss": 2.7376210689544678,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11147420108318329,
      "rewards/margins": 2.2028400897979736,
      "rewards/rejected": -2.314314365386963,
      "step": 1035
    },
    {
      "epoch": 0.6444790046656299,
      "grad_norm": 3.430917263031006,
      "learning_rate": 2.41e-05,
      "log_odds_chosen": 16.318185806274414,
      "log_odds_ratio": -0.0678626224398613,
      "logits/chosen": 0.272255003452301,
      "logits/rejected": 1.316011905670166,
      "logps/chosen": -1.4429469108581543,
      "logps/rejected": -17.469158172607422,
      "loss": 3.3379,
      "nll_loss": 3.3311076164245605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14429470896720886,
      "rewards/margins": 1.6026211977005005,
      "rewards/rejected": -1.7469159364700317,
      "step": 1036
    },
    {
      "epoch": 0.6451010886469674,
      "grad_norm": 0.4014487564563751,
      "learning_rate": 2.4075e-05,
      "log_odds_chosen": 10.295687675476074,
      "log_odds_ratio": -0.055723220109939575,
      "logits/chosen": 0.09633732587099075,
      "logits/rejected": 0.6330247521400452,
      "logps/chosen": -1.3109146356582642,
      "logps/rejected": -11.317893981933594,
      "loss": 3.0171,
      "nll_loss": 3.0115628242492676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13109146058559418,
      "rewards/margins": 1.0006979703903198,
      "rewards/rejected": -1.1317894458770752,
      "step": 1037
    },
    {
      "epoch": 0.6457231726283048,
      "grad_norm": 0.5607584118843079,
      "learning_rate": 2.4050000000000002e-05,
      "log_odds_chosen": 18.80658721923828,
      "log_odds_ratio": -0.023404287174344063,
      "logits/chosen": 0.39442214369773865,
      "logits/rejected": 1.445420265197754,
      "logps/chosen": -1.073487639427185,
      "logps/rejected": -19.431781768798828,
      "loss": 2.8651,
      "nll_loss": 2.8627424240112305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10734876245260239,
      "rewards/margins": 1.8358293771743774,
      "rewards/rejected": -1.9431781768798828,
      "step": 1038
    },
    {
      "epoch": 0.6463452566096423,
      "grad_norm": 0.41965922713279724,
      "learning_rate": 2.4025e-05,
      "log_odds_chosen": 15.010441780090332,
      "log_odds_ratio": -0.049356162548065186,
      "logits/chosen": 0.258677214384079,
      "logits/rejected": 1.031904935836792,
      "logps/chosen": -1.0338720083236694,
      "logps/rejected": -15.611568450927734,
      "loss": 3.118,
      "nll_loss": 3.1130309104919434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10338720679283142,
      "rewards/margins": 1.4577698707580566,
      "rewards/rejected": -1.561156988143921,
      "step": 1039
    },
    {
      "epoch": 0.6469673405909798,
      "grad_norm": 0.36550575494766235,
      "learning_rate": 2.4e-05,
      "log_odds_chosen": 14.48094367980957,
      "log_odds_ratio": -0.1260167956352234,
      "logits/chosen": 0.1283513307571411,
      "logits/rejected": 0.8361437320709229,
      "logps/chosen": -1.0468213558197021,
      "logps/rejected": -14.914956092834473,
      "loss": 3.0304,
      "nll_loss": 3.0178399085998535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10468213260173798,
      "rewards/margins": 1.3868136405944824,
      "rewards/rejected": -1.4914956092834473,
      "step": 1040
    },
    {
      "epoch": 0.6475894245723173,
      "grad_norm": 1.547513484954834,
      "learning_rate": 2.3975e-05,
      "log_odds_chosen": 12.045820236206055,
      "log_odds_ratio": -0.22973600029945374,
      "logits/chosen": 0.1489129662513733,
      "logits/rejected": 0.9729058146476746,
      "logps/chosen": -1.2251245975494385,
      "logps/rejected": -12.936586380004883,
      "loss": 2.5435,
      "nll_loss": 2.5204789638519287,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12251245975494385,
      "rewards/margins": 1.1711461544036865,
      "rewards/rejected": -1.2936586141586304,
      "step": 1041
    },
    {
      "epoch": 0.6482115085536547,
      "grad_norm": 0.3640393912792206,
      "learning_rate": 2.395e-05,
      "log_odds_chosen": 13.0018310546875,
      "log_odds_ratio": -0.23523327708244324,
      "logits/chosen": 0.10855109244585037,
      "logits/rejected": 0.6623581647872925,
      "logps/chosen": -1.285058617591858,
      "logps/rejected": -14.037286758422852,
      "loss": 3.0383,
      "nll_loss": 3.014784812927246,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1285058706998825,
      "rewards/margins": 1.2752227783203125,
      "rewards/rejected": -1.403728723526001,
      "step": 1042
    },
    {
      "epoch": 0.6488335925349922,
      "grad_norm": 0.4505966007709503,
      "learning_rate": 2.3925e-05,
      "log_odds_chosen": 7.586543083190918,
      "log_odds_ratio": -0.278242290019989,
      "logits/chosen": 0.1186991035938263,
      "logits/rejected": 0.504472017288208,
      "logps/chosen": -1.3483988046646118,
      "logps/rejected": -8.706100463867188,
      "loss": 3.0296,
      "nll_loss": 3.0018014907836914,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13483989238739014,
      "rewards/margins": 0.7357701659202576,
      "rewards/rejected": -0.8706101179122925,
      "step": 1043
    },
    {
      "epoch": 0.6494556765163297,
      "grad_norm": 0.5245290994644165,
      "learning_rate": 2.39e-05,
      "log_odds_chosen": 8.640314102172852,
      "log_odds_ratio": -0.21840831637382507,
      "logits/chosen": 0.23793521523475647,
      "logits/rejected": 0.5504704713821411,
      "logps/chosen": -1.2520349025726318,
      "logps/rejected": -9.354804992675781,
      "loss": 3.5628,
      "nll_loss": 3.540966749191284,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12520349025726318,
      "rewards/margins": 0.8102771043777466,
      "rewards/rejected": -0.9354805946350098,
      "step": 1044
    },
    {
      "epoch": 0.6500777604976672,
      "grad_norm": 0.5833095908164978,
      "learning_rate": 2.3875e-05,
      "log_odds_chosen": 10.210502624511719,
      "log_odds_ratio": -0.17007872462272644,
      "logits/chosen": 0.12730933725833893,
      "logits/rejected": 0.5066798329353333,
      "logps/chosen": -0.9844697713851929,
      "logps/rejected": -10.70468807220459,
      "loss": 2.9488,
      "nll_loss": 2.9318201541900635,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09844697266817093,
      "rewards/margins": 0.9720218777656555,
      "rewards/rejected": -1.0704689025878906,
      "step": 1045
    },
    {
      "epoch": 0.6506998444790046,
      "grad_norm": 0.47387048602104187,
      "learning_rate": 2.385e-05,
      "log_odds_chosen": 8.573151588439941,
      "log_odds_ratio": -0.2872176766395569,
      "logits/chosen": 0.2543594539165497,
      "logits/rejected": 0.5832099914550781,
      "logps/chosen": -1.0247888565063477,
      "logps/rejected": -9.087888717651367,
      "loss": 3.2127,
      "nll_loss": 3.184014081954956,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10247889161109924,
      "rewards/margins": 0.8063100576400757,
      "rewards/rejected": -0.9087889194488525,
      "step": 1046
    },
    {
      "epoch": 0.6513219284603421,
      "grad_norm": 0.4664608836174011,
      "learning_rate": 2.3825e-05,
      "log_odds_chosen": 7.170677661895752,
      "log_odds_ratio": -0.2467111349105835,
      "logits/chosen": -0.025047319009900093,
      "logits/rejected": 0.4170008897781372,
      "logps/chosen": -1.1048283576965332,
      "logps/rejected": -7.730550765991211,
      "loss": 2.3306,
      "nll_loss": 2.305936098098755,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1104828268289566,
      "rewards/margins": 0.6625722646713257,
      "rewards/rejected": -0.7730551362037659,
      "step": 1047
    },
    {
      "epoch": 0.6519440124416797,
      "grad_norm": 0.5314801931381226,
      "learning_rate": 2.38e-05,
      "log_odds_chosen": 6.302850246429443,
      "log_odds_ratio": -0.18075837194919586,
      "logits/chosen": 0.0019352678209543228,
      "logits/rejected": 0.21269835531711578,
      "logps/chosen": -0.8562182784080505,
      "logps/rejected": -6.463325023651123,
      "loss": 2.9775,
      "nll_loss": 2.959415912628174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08562182635068893,
      "rewards/margins": 0.5607106685638428,
      "rewards/rejected": -0.6463325023651123,
      "step": 1048
    },
    {
      "epoch": 0.6525660964230171,
      "grad_norm": 0.5723354816436768,
      "learning_rate": 2.3775e-05,
      "log_odds_chosen": 6.959724426269531,
      "log_odds_ratio": -0.1822098344564438,
      "logits/chosen": 0.12127923220396042,
      "logits/rejected": 0.36715346574783325,
      "logps/chosen": -1.1644635200500488,
      "logps/rejected": -7.830971717834473,
      "loss": 3.0659,
      "nll_loss": 3.0477237701416016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.116446353495121,
      "rewards/margins": 0.6666508913040161,
      "rewards/rejected": -0.7830972075462341,
      "step": 1049
    },
    {
      "epoch": 0.6531881804043546,
      "grad_norm": 0.48077571392059326,
      "learning_rate": 2.375e-05,
      "log_odds_chosen": 10.392155647277832,
      "log_odds_ratio": -0.18274815380573273,
      "logits/chosen": 0.2417854368686676,
      "logits/rejected": 0.8186101913452148,
      "logps/chosen": -1.1042566299438477,
      "logps/rejected": -11.141618728637695,
      "loss": 3.3063,
      "nll_loss": 3.288038492202759,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11042565852403641,
      "rewards/margins": 1.0037362575531006,
      "rewards/rejected": -1.1141619682312012,
      "step": 1050
    },
    {
      "epoch": 0.6538102643856921,
      "grad_norm": 0.46275290846824646,
      "learning_rate": 2.3725e-05,
      "log_odds_chosen": 8.993810653686523,
      "log_odds_ratio": -0.13959504663944244,
      "logits/chosen": 0.1534716784954071,
      "logits/rejected": 0.7214460372924805,
      "logps/chosen": -1.1377089023590088,
      "logps/rejected": -9.824823379516602,
      "loss": 3.2004,
      "nll_loss": 3.186446189880371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11377088725566864,
      "rewards/margins": 0.8687114119529724,
      "rewards/rejected": -0.9824823141098022,
      "step": 1051
    },
    {
      "epoch": 0.6544323483670296,
      "grad_norm": 0.4433314800262451,
      "learning_rate": 2.37e-05,
      "log_odds_chosen": 10.695637702941895,
      "log_odds_ratio": -0.15089482069015503,
      "logits/chosen": 0.24725496768951416,
      "logits/rejected": 0.9210346341133118,
      "logps/chosen": -1.2222862243652344,
      "logps/rejected": -11.55354118347168,
      "loss": 3.217,
      "nll_loss": 3.201942205429077,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12222862243652344,
      "rewards/margins": 1.0331255197525024,
      "rewards/rejected": -1.1553540229797363,
      "step": 1052
    },
    {
      "epoch": 0.655054432348367,
      "grad_norm": 0.41843631863594055,
      "learning_rate": 2.3675e-05,
      "log_odds_chosen": 15.736959457397461,
      "log_odds_ratio": -0.0017072423361241817,
      "logits/chosen": 0.06725797802209854,
      "logits/rejected": 0.6552459001541138,
      "logps/chosen": -1.5394830703735352,
      "logps/rejected": -16.988080978393555,
      "loss": 3.3547,
      "nll_loss": 3.3545265197753906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15394830703735352,
      "rewards/margins": 1.5448598861694336,
      "rewards/rejected": -1.698808193206787,
      "step": 1053
    },
    {
      "epoch": 0.6556765163297045,
      "grad_norm": 0.506420373916626,
      "learning_rate": 2.365e-05,
      "log_odds_chosen": 9.227112770080566,
      "log_odds_ratio": -0.047787003219127655,
      "logits/chosen": 0.0533127635717392,
      "logits/rejected": 0.6818662285804749,
      "logps/chosen": -1.3065093755722046,
      "logps/rejected": -10.112043380737305,
      "loss": 2.7755,
      "nll_loss": 2.7706892490386963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13065093755722046,
      "rewards/margins": 0.8805534839630127,
      "rewards/rejected": -1.0112043619155884,
      "step": 1054
    },
    {
      "epoch": 0.656298600311042,
      "grad_norm": 0.4978622794151306,
      "learning_rate": 2.3624999999999998e-05,
      "log_odds_chosen": 12.25826358795166,
      "log_odds_ratio": -0.1236209124326706,
      "logits/chosen": 0.18020248413085938,
      "logits/rejected": 1.1149808168411255,
      "logps/chosen": -1.2221086025238037,
      "logps/rejected": -13.194238662719727,
      "loss": 2.4563,
      "nll_loss": 2.443939447402954,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12221086025238037,
      "rewards/margins": 1.197213053703308,
      "rewards/rejected": -1.3194239139556885,
      "step": 1055
    },
    {
      "epoch": 0.6569206842923795,
      "grad_norm": 0.6258676648139954,
      "learning_rate": 2.36e-05,
      "log_odds_chosen": 10.127604484558105,
      "log_odds_ratio": -0.15316568315029144,
      "logits/chosen": 0.2081230878829956,
      "logits/rejected": 0.5622198581695557,
      "logps/chosen": -1.128497838973999,
      "logps/rejected": -10.898340225219727,
      "loss": 3.3251,
      "nll_loss": 3.309760093688965,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11284978687763214,
      "rewards/margins": 0.9769842624664307,
      "rewards/rejected": -1.0898339748382568,
      "step": 1056
    },
    {
      "epoch": 0.6575427682737169,
      "grad_norm": 0.5358809232711792,
      "learning_rate": 2.3575e-05,
      "log_odds_chosen": 14.776838302612305,
      "log_odds_ratio": -0.12466295063495636,
      "logits/chosen": 0.3303581774234772,
      "logits/rejected": 0.9545040130615234,
      "logps/chosen": -0.9283601641654968,
      "logps/rejected": -15.194375991821289,
      "loss": 3.6474,
      "nll_loss": 3.6349401473999023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09283602982759476,
      "rewards/margins": 1.4266016483306885,
      "rewards/rejected": -1.519437551498413,
      "step": 1057
    },
    {
      "epoch": 0.6581648522550544,
      "grad_norm": 0.5612006783485413,
      "learning_rate": 2.355e-05,
      "log_odds_chosen": 13.543533325195312,
      "log_odds_ratio": -0.1741914451122284,
      "logits/chosen": 0.3145029842853546,
      "logits/rejected": 1.0952863693237305,
      "logps/chosen": -0.9952583909034729,
      "logps/rejected": -14.09577465057373,
      "loss": 3.4287,
      "nll_loss": 3.4113099575042725,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09952583909034729,
      "rewards/margins": 1.310051679611206,
      "rewards/rejected": -1.409577488899231,
      "step": 1058
    },
    {
      "epoch": 0.658786936236392,
      "grad_norm": 0.5212019085884094,
      "learning_rate": 2.3525e-05,
      "log_odds_chosen": 11.775590896606445,
      "log_odds_ratio": -0.08714871853590012,
      "logits/chosen": 0.17237700521945953,
      "logits/rejected": 1.0523440837860107,
      "logps/chosen": -0.8339164853096008,
      "logps/rejected": -11.932096481323242,
      "loss": 2.8133,
      "nll_loss": 2.8046300411224365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08339165151119232,
      "rewards/margins": 1.1098179817199707,
      "rewards/rejected": -1.1932096481323242,
      "step": 1059
    },
    {
      "epoch": 0.6594090202177294,
      "grad_norm": 0.6565141677856445,
      "learning_rate": 2.35e-05,
      "log_odds_chosen": 11.25783920288086,
      "log_odds_ratio": -0.26100558042526245,
      "logits/chosen": 0.18183985352516174,
      "logits/rejected": 0.8650422096252441,
      "logps/chosen": -0.9057870507240295,
      "logps/rejected": -11.680928230285645,
      "loss": 2.5247,
      "nll_loss": 2.498648166656494,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09057871252298355,
      "rewards/margins": 1.0775140523910522,
      "rewards/rejected": -1.1680928468704224,
      "step": 1060
    },
    {
      "epoch": 0.6600311041990669,
      "grad_norm": 0.4263257384300232,
      "learning_rate": 2.3475e-05,
      "log_odds_chosen": 17.359153747558594,
      "log_odds_ratio": -0.09745100885629654,
      "logits/chosen": 0.18176640570163727,
      "logits/rejected": 1.2584339380264282,
      "logps/chosen": -1.2209086418151855,
      "logps/rejected": -18.21963119506836,
      "loss": 3.1567,
      "nll_loss": 3.146958112716675,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12209086120128632,
      "rewards/margins": 1.6998722553253174,
      "rewards/rejected": -1.8219630718231201,
      "step": 1061
    },
    {
      "epoch": 0.6606531881804043,
      "grad_norm": 0.41249606013298035,
      "learning_rate": 2.345e-05,
      "log_odds_chosen": 18.928997039794922,
      "log_odds_ratio": -0.001390795805491507,
      "logits/chosen": 0.19376930594444275,
      "logits/rejected": 1.2592663764953613,
      "logps/chosen": -0.9024685621261597,
      "logps/rejected": -19.096473693847656,
      "loss": 2.7631,
      "nll_loss": 2.7629570960998535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09024685621261597,
      "rewards/margins": 1.8194003105163574,
      "rewards/rejected": -1.9096472263336182,
      "step": 1062
    },
    {
      "epoch": 0.6612752721617419,
      "grad_norm": 0.4502386152744293,
      "learning_rate": 2.3425000000000004e-05,
      "log_odds_chosen": 12.114904403686523,
      "log_odds_ratio": -0.1213536337018013,
      "logits/chosen": 0.24823647737503052,
      "logits/rejected": 1.1324775218963623,
      "logps/chosen": -1.1689949035644531,
      "logps/rejected": -12.937753677368164,
      "loss": 3.2664,
      "nll_loss": 3.254295587539673,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1168995052576065,
      "rewards/margins": 1.1768758296966553,
      "rewards/rejected": -1.2937753200531006,
      "step": 1063
    },
    {
      "epoch": 0.6618973561430793,
      "grad_norm": 0.5948436856269836,
      "learning_rate": 2.3400000000000003e-05,
      "log_odds_chosen": 9.675779342651367,
      "log_odds_ratio": -0.15184316039085388,
      "logits/chosen": 0.07025956362485886,
      "logits/rejected": 0.536703884601593,
      "logps/chosen": -1.4402323961257935,
      "logps/rejected": -10.905509948730469,
      "loss": 2.9341,
      "nll_loss": 2.9189584255218506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14402325451374054,
      "rewards/margins": 0.946527898311615,
      "rewards/rejected": -1.0905512571334839,
      "step": 1064
    },
    {
      "epoch": 0.6625194401244168,
      "grad_norm": 1.291369080543518,
      "learning_rate": 2.3375000000000002e-05,
      "log_odds_chosen": 15.299062728881836,
      "log_odds_ratio": -0.10372968018054962,
      "logits/chosen": 0.14694362878799438,
      "logits/rejected": 1.0618005990982056,
      "logps/chosen": -1.1700985431671143,
      "logps/rejected": -16.039615631103516,
      "loss": 2.9016,
      "nll_loss": 2.8912243843078613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11700985580682755,
      "rewards/margins": 1.4869518280029297,
      "rewards/rejected": -1.603961706161499,
      "step": 1065
    },
    {
      "epoch": 0.6631415241057543,
      "grad_norm": 0.41034582257270813,
      "learning_rate": 2.3350000000000002e-05,
      "log_odds_chosen": 10.85059642791748,
      "log_odds_ratio": -0.2492515593767166,
      "logits/chosen": 0.3745499849319458,
      "logits/rejected": 0.9561654329299927,
      "logps/chosen": -1.2551486492156982,
      "logps/rejected": -11.893285751342773,
      "loss": 3.7877,
      "nll_loss": 3.762781858444214,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12551486492156982,
      "rewards/margins": 1.0638136863708496,
      "rewards/rejected": -1.1893285512924194,
      "step": 1066
    },
    {
      "epoch": 0.6637636080870918,
      "grad_norm": 0.42354220151901245,
      "learning_rate": 2.3325e-05,
      "log_odds_chosen": 17.378068923950195,
      "log_odds_ratio": -0.13644790649414062,
      "logits/chosen": 0.24174179136753082,
      "logits/rejected": 1.284676432609558,
      "logps/chosen": -1.274251937866211,
      "logps/rejected": -18.352718353271484,
      "loss": 3.082,
      "nll_loss": 3.0684027671813965,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1274251937866211,
      "rewards/margins": 1.7078466415405273,
      "rewards/rejected": -1.8352718353271484,
      "step": 1067
    },
    {
      "epoch": 0.6643856920684292,
      "grad_norm": 1.2665324211120605,
      "learning_rate": 2.3300000000000004e-05,
      "log_odds_chosen": 11.014419555664062,
      "log_odds_ratio": -0.06745558232069016,
      "logits/chosen": 0.19637992978096008,
      "logits/rejected": 0.9851882457733154,
      "logps/chosen": -1.110511302947998,
      "logps/rejected": -11.721153259277344,
      "loss": 3.3051,
      "nll_loss": 3.298313617706299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11105112731456757,
      "rewards/margins": 1.0610642433166504,
      "rewards/rejected": -1.1721153259277344,
      "step": 1068
    },
    {
      "epoch": 0.6650077760497667,
      "grad_norm": 0.42871132493019104,
      "learning_rate": 2.3275000000000003e-05,
      "log_odds_chosen": 19.26378631591797,
      "log_odds_ratio": -0.00020241702441126108,
      "logits/chosen": 0.28503745794296265,
      "logits/rejected": 1.4887564182281494,
      "logps/chosen": -1.0176734924316406,
      "logps/rejected": -19.808406829833984,
      "loss": 3.1755,
      "nll_loss": 3.175529718399048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10176734626293182,
      "rewards/margins": 1.8790735006332397,
      "rewards/rejected": -1.980840802192688,
      "step": 1069
    },
    {
      "epoch": 0.6656298600311042,
      "grad_norm": 1.2943010330200195,
      "learning_rate": 2.3250000000000003e-05,
      "log_odds_chosen": 11.460484504699707,
      "log_odds_ratio": -0.30275437235832214,
      "logits/chosen": 0.11974619328975677,
      "logits/rejected": 0.6861406564712524,
      "logps/chosen": -0.9381018280982971,
      "logps/rejected": -11.976990699768066,
      "loss": 2.6535,
      "nll_loss": 2.623176097869873,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09381018579006195,
      "rewards/margins": 1.103888988494873,
      "rewards/rejected": -1.1976991891860962,
      "step": 1070
    },
    {
      "epoch": 0.6662519440124417,
      "grad_norm": 4.658538818359375,
      "learning_rate": 2.3225000000000002e-05,
      "log_odds_chosen": 14.916189193725586,
      "log_odds_ratio": -0.1716422736644745,
      "logits/chosen": 0.19798047840595245,
      "logits/rejected": 0.8838070631027222,
      "logps/chosen": -1.6107474565505981,
      "logps/rejected": -15.790814399719238,
      "loss": 3.1172,
      "nll_loss": 3.100071907043457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16107475757598877,
      "rewards/margins": 1.4180067777633667,
      "rewards/rejected": -1.5790815353393555,
      "step": 1071
    },
    {
      "epoch": 0.6668740279937792,
      "grad_norm": 0.5105116963386536,
      "learning_rate": 2.32e-05,
      "log_odds_chosen": 14.650772094726562,
      "log_odds_ratio": -0.007171155884861946,
      "logits/chosen": 0.15524886548519135,
      "logits/rejected": 1.0009796619415283,
      "logps/chosen": -1.2726460695266724,
      "logps/rejected": -15.550126075744629,
      "loss": 2.8389,
      "nll_loss": 2.838151454925537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1272645890712738,
      "rewards/margins": 1.4277479648590088,
      "rewards/rejected": -1.5550127029418945,
      "step": 1072
    },
    {
      "epoch": 0.6674961119751166,
      "grad_norm": 0.6901617050170898,
      "learning_rate": 2.3175e-05,
      "log_odds_chosen": 7.709420680999756,
      "log_odds_ratio": -0.05205727741122246,
      "logits/chosen": 0.14403298497200012,
      "logits/rejected": 0.5285943746566772,
      "logps/chosen": -1.155900001525879,
      "logps/rejected": -8.388169288635254,
      "loss": 3.0981,
      "nll_loss": 3.092885971069336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11559000611305237,
      "rewards/margins": 0.7232269048690796,
      "rewards/rejected": -0.8388169407844543,
      "step": 1073
    },
    {
      "epoch": 0.6681181959564542,
      "grad_norm": 0.5226415991783142,
      "learning_rate": 2.3150000000000004e-05,
      "log_odds_chosen": 14.16978645324707,
      "log_odds_ratio": -0.10162307322025299,
      "logits/chosen": 0.1284743696451187,
      "logits/rejected": 0.8283305168151855,
      "logps/chosen": -1.5021108388900757,
      "logps/rejected": -15.423286437988281,
      "loss": 3.2585,
      "nll_loss": 3.248311996459961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15021109580993652,
      "rewards/margins": 1.3921175003051758,
      "rewards/rejected": -1.5423285961151123,
      "step": 1074
    },
    {
      "epoch": 0.6687402799377916,
      "grad_norm": 1.8207690715789795,
      "learning_rate": 2.3125000000000003e-05,
      "log_odds_chosen": 16.029922485351562,
      "log_odds_ratio": -5.97071266383864e-05,
      "logits/chosen": 0.06656906008720398,
      "logits/rejected": 0.8557884097099304,
      "logps/chosen": -1.1395089626312256,
      "logps/rejected": -16.5898494720459,
      "loss": 2.7598,
      "nll_loss": 2.7598206996917725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11395090073347092,
      "rewards/margins": 1.5450341701507568,
      "rewards/rejected": -1.6589850187301636,
      "step": 1075
    },
    {
      "epoch": 0.6693623639191291,
      "grad_norm": 0.6128720045089722,
      "learning_rate": 2.3100000000000002e-05,
      "log_odds_chosen": 7.803811550140381,
      "log_odds_ratio": -0.16541123390197754,
      "logits/chosen": 0.04209950566291809,
      "logits/rejected": 0.5215015411376953,
      "logps/chosen": -1.3678196668624878,
      "logps/rejected": -8.899198532104492,
      "loss": 2.797,
      "nll_loss": 2.780494213104248,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1367819756269455,
      "rewards/margins": 0.7531379461288452,
      "rewards/rejected": -0.8899198174476624,
      "step": 1076
    },
    {
      "epoch": 0.6699844479004665,
      "grad_norm": 0.533178448677063,
      "learning_rate": 2.3075000000000002e-05,
      "log_odds_chosen": 5.934687614440918,
      "log_odds_ratio": -0.3675815463066101,
      "logits/chosen": 0.07074688374996185,
      "logits/rejected": 0.4116901159286499,
      "logps/chosen": -1.1007202863693237,
      "logps/rejected": -6.782163619995117,
      "loss": 2.5115,
      "nll_loss": 2.4747588634490967,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11007203161716461,
      "rewards/margins": 0.5681442618370056,
      "rewards/rejected": -0.6782163381576538,
      "step": 1077
    },
    {
      "epoch": 0.6706065318818041,
      "grad_norm": 0.5132092833518982,
      "learning_rate": 2.305e-05,
      "log_odds_chosen": 11.052836418151855,
      "log_odds_ratio": -0.2220580130815506,
      "logits/chosen": 0.11277395486831665,
      "logits/rejected": 0.8609932661056519,
      "logps/chosen": -1.1415894031524658,
      "logps/rejected": -11.936271667480469,
      "loss": 2.4863,
      "nll_loss": 2.4641189575195312,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11415895819664001,
      "rewards/margins": 1.0794682502746582,
      "rewards/rejected": -1.1936272382736206,
      "step": 1078
    },
    {
      "epoch": 0.6712286158631415,
      "grad_norm": 0.5379019379615784,
      "learning_rate": 2.3025e-05,
      "log_odds_chosen": 12.296998977661133,
      "log_odds_ratio": -0.10660477727651596,
      "logits/chosen": 0.14019262790679932,
      "logits/rejected": 0.8425007462501526,
      "logps/chosen": -1.435052514076233,
      "logps/rejected": -13.453621864318848,
      "loss": 3.4839,
      "nll_loss": 3.473271369934082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14350526034832,
      "rewards/margins": 1.2018568515777588,
      "rewards/rejected": -1.3453621864318848,
      "step": 1079
    },
    {
      "epoch": 0.671850699844479,
      "grad_norm": 0.46937939524650574,
      "learning_rate": 2.3000000000000003e-05,
      "log_odds_chosen": 12.263298034667969,
      "log_odds_ratio": -0.24854573607444763,
      "logits/chosen": 0.14811545610427856,
      "logits/rejected": 0.7356073260307312,
      "logps/chosen": -1.1790632009506226,
      "logps/rejected": -13.17032527923584,
      "loss": 2.9886,
      "nll_loss": 2.963712215423584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11790631711483002,
      "rewards/margins": 1.1991263628005981,
      "rewards/rejected": -1.3170325756072998,
      "step": 1080
    },
    {
      "epoch": 0.6724727838258164,
      "grad_norm": 0.48714011907577515,
      "learning_rate": 2.2975000000000003e-05,
      "log_odds_chosen": 5.6596174240112305,
      "log_odds_ratio": -0.5009754300117493,
      "logits/chosen": 0.10506222397089005,
      "logits/rejected": 0.18131209909915924,
      "logps/chosen": -1.3187544345855713,
      "logps/rejected": -6.899771690368652,
      "loss": 3.4112,
      "nll_loss": 3.361135959625244,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1318754404783249,
      "rewards/margins": 0.5581017732620239,
      "rewards/rejected": -0.6899771690368652,
      "step": 1081
    },
    {
      "epoch": 0.673094867807154,
      "grad_norm": 0.5086039900779724,
      "learning_rate": 2.2950000000000002e-05,
      "log_odds_chosen": 10.907814979553223,
      "log_odds_ratio": -0.08033996820449829,
      "logits/chosen": 0.12853284180164337,
      "logits/rejected": 0.823364794254303,
      "logps/chosen": -1.0794905424118042,
      "logps/rejected": -11.466302871704102,
      "loss": 2.8303,
      "nll_loss": 2.8222546577453613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10794906318187714,
      "rewards/margins": 1.038681149482727,
      "rewards/rejected": -1.1466301679611206,
      "step": 1082
    },
    {
      "epoch": 0.6737169517884914,
      "grad_norm": 0.4580300748348236,
      "learning_rate": 2.2925e-05,
      "log_odds_chosen": 16.086458206176758,
      "log_odds_ratio": -0.00020257926371414214,
      "logits/chosen": 0.13432970643043518,
      "logits/rejected": 1.132068395614624,
      "logps/chosen": -1.136250376701355,
      "logps/rejected": -16.81014633178711,
      "loss": 2.7228,
      "nll_loss": 2.722776412963867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11362504959106445,
      "rewards/margins": 1.567389726638794,
      "rewards/rejected": -1.6810145378112793,
      "step": 1083
    },
    {
      "epoch": 0.6743390357698289,
      "grad_norm": 1.2523407936096191,
      "learning_rate": 2.29e-05,
      "log_odds_chosen": 10.811274528503418,
      "log_odds_ratio": -0.3882692754268646,
      "logits/chosen": 0.09717198461294174,
      "logits/rejected": 0.5859137773513794,
      "logps/chosen": -1.5438910722732544,
      "logps/rejected": -12.124390602111816,
      "loss": 2.8007,
      "nll_loss": 2.7619214057922363,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15438911318778992,
      "rewards/margins": 1.0580499172210693,
      "rewards/rejected": -1.2124391794204712,
      "step": 1084
    },
    {
      "epoch": 0.6749611197511665,
      "grad_norm": 0.5664082765579224,
      "learning_rate": 2.2875e-05,
      "log_odds_chosen": 12.99592113494873,
      "log_odds_ratio": -0.15935920178890228,
      "logits/chosen": 0.15744078159332275,
      "logits/rejected": 0.6450719237327576,
      "logps/chosen": -0.8572717905044556,
      "logps/rejected": -13.253767013549805,
      "loss": 3.2876,
      "nll_loss": 3.2716193199157715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08572718501091003,
      "rewards/margins": 1.2396494150161743,
      "rewards/rejected": -1.3253766298294067,
      "step": 1085
    },
    {
      "epoch": 0.6755832037325039,
      "grad_norm": 0.8332890272140503,
      "learning_rate": 2.2850000000000003e-05,
      "log_odds_chosen": 9.230619430541992,
      "log_odds_ratio": -0.4146743416786194,
      "logits/chosen": 0.136587455868721,
      "logits/rejected": 0.7445818781852722,
      "logps/chosen": -1.5004432201385498,
      "logps/rejected": -10.546527862548828,
      "loss": 3.4476,
      "nll_loss": 3.4061646461486816,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15004432201385498,
      "rewards/margins": 0.9046083688735962,
      "rewards/rejected": -1.0546526908874512,
      "step": 1086
    },
    {
      "epoch": 0.6762052877138414,
      "grad_norm": 0.5611684322357178,
      "learning_rate": 2.2825000000000003e-05,
      "log_odds_chosen": 7.212390899658203,
      "log_odds_ratio": -0.253339022397995,
      "logits/chosen": 0.09394458681344986,
      "logits/rejected": 0.35125231742858887,
      "logps/chosen": -0.6430901288986206,
      "logps/rejected": -7.047637939453125,
      "loss": 2.771,
      "nll_loss": 2.745687961578369,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0643090084195137,
      "rewards/margins": 0.6404548287391663,
      "rewards/rejected": -0.7047637701034546,
      "step": 1087
    },
    {
      "epoch": 0.6768273716951788,
      "grad_norm": 0.5705518126487732,
      "learning_rate": 2.2800000000000002e-05,
      "log_odds_chosen": 10.540750503540039,
      "log_odds_ratio": -0.1235579401254654,
      "logits/chosen": 0.221440851688385,
      "logits/rejected": 0.7182592153549194,
      "logps/chosen": -1.1119794845581055,
      "logps/rejected": -11.2672119140625,
      "loss": 3.233,
      "nll_loss": 3.2206497192382812,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11119795590639114,
      "rewards/margins": 1.0155231952667236,
      "rewards/rejected": -1.1267211437225342,
      "step": 1088
    },
    {
      "epoch": 0.6774494556765164,
      "grad_norm": 0.4467851221561432,
      "learning_rate": 2.2775e-05,
      "log_odds_chosen": 16.156049728393555,
      "log_odds_ratio": -0.2083779126405716,
      "logits/chosen": 0.13699352741241455,
      "logits/rejected": 0.7858900427818298,
      "logps/chosen": -1.272729754447937,
      "logps/rejected": -17.19986343383789,
      "loss": 2.9677,
      "nll_loss": 2.946847915649414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12727296352386475,
      "rewards/margins": 1.592713475227356,
      "rewards/rejected": -1.7199864387512207,
      "step": 1089
    },
    {
      "epoch": 0.6780715396578538,
      "grad_norm": 0.5542477369308472,
      "learning_rate": 2.275e-05,
      "log_odds_chosen": 5.15977144241333,
      "log_odds_ratio": -0.3942872881889343,
      "logits/chosen": 0.15950456261634827,
      "logits/rejected": 0.2518439292907715,
      "logps/chosen": -1.4990637302398682,
      "logps/rejected": -6.542140483856201,
      "loss": 3.1346,
      "nll_loss": 3.0951952934265137,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14990636706352234,
      "rewards/margins": 0.5043076872825623,
      "rewards/rejected": -0.6542140245437622,
      "step": 1090
    },
    {
      "epoch": 0.6786936236391913,
      "grad_norm": 0.5533297061920166,
      "learning_rate": 2.2725000000000003e-05,
      "log_odds_chosen": 10.516497611999512,
      "log_odds_ratio": -0.18116550147533417,
      "logits/chosen": 0.15582889318466187,
      "logits/rejected": 0.7493836879730225,
      "logps/chosen": -1.2090470790863037,
      "logps/rejected": -11.461071014404297,
      "loss": 3.2506,
      "nll_loss": 3.232510805130005,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12090469896793365,
      "rewards/margins": 1.0252025127410889,
      "rewards/rejected": -1.1461071968078613,
      "step": 1091
    },
    {
      "epoch": 0.6793157076205287,
      "grad_norm": 0.44804054498672485,
      "learning_rate": 2.2700000000000003e-05,
      "log_odds_chosen": 9.345958709716797,
      "log_odds_ratio": -0.0998968705534935,
      "logits/chosen": 0.10394829511642456,
      "logits/rejected": 0.22612932324409485,
      "logps/chosen": -0.6652541160583496,
      "logps/rejected": -9.150422096252441,
      "loss": 3.0124,
      "nll_loss": 3.0023632049560547,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0665254145860672,
      "rewards/margins": 0.8485168218612671,
      "rewards/rejected": -0.9150421619415283,
      "step": 1092
    },
    {
      "epoch": 0.6799377916018663,
      "grad_norm": 0.45218420028686523,
      "learning_rate": 2.2675000000000002e-05,
      "log_odds_chosen": 11.6668062210083,
      "log_odds_ratio": -0.08176952600479126,
      "logits/chosen": 0.3411112129688263,
      "logits/rejected": 1.1545166969299316,
      "logps/chosen": -0.9080636501312256,
      "logps/rejected": -11.78917407989502,
      "loss": 3.5003,
      "nll_loss": 3.492164134979248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09080636501312256,
      "rewards/margins": 1.088111162185669,
      "rewards/rejected": -1.1789175271987915,
      "step": 1093
    },
    {
      "epoch": 0.6805598755832037,
      "grad_norm": 0.6241004467010498,
      "learning_rate": 2.265e-05,
      "log_odds_chosen": 6.734440803527832,
      "log_odds_ratio": -0.2348792403936386,
      "logits/chosen": 0.11519190669059753,
      "logits/rejected": 0.5573137998580933,
      "logps/chosen": -1.2351096868515015,
      "logps/rejected": -7.5556793212890625,
      "loss": 2.7045,
      "nll_loss": 2.681022882461548,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12351097166538239,
      "rewards/margins": 0.6320569515228271,
      "rewards/rejected": -0.7555679082870483,
      "step": 1094
    },
    {
      "epoch": 0.6811819595645412,
      "grad_norm": 2.355020761489868,
      "learning_rate": 2.2625e-05,
      "log_odds_chosen": 13.27316665649414,
      "log_odds_ratio": -0.13957035541534424,
      "logits/chosen": 0.24990928173065186,
      "logits/rejected": 0.8357008099555969,
      "logps/chosen": -1.2236608266830444,
      "logps/rejected": -14.109601020812988,
      "loss": 3.6854,
      "nll_loss": 3.6714515686035156,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12236608564853668,
      "rewards/margins": 1.2885940074920654,
      "rewards/rejected": -1.410960078239441,
      "step": 1095
    },
    {
      "epoch": 0.6818040435458786,
      "grad_norm": 0.43016043305397034,
      "learning_rate": 2.26e-05,
      "log_odds_chosen": 13.364396095275879,
      "log_odds_ratio": -0.10089391469955444,
      "logits/chosen": 0.12420229613780975,
      "logits/rejected": 0.6156033277511597,
      "logps/chosen": -1.0751311779022217,
      "logps/rejected": -13.820520401000977,
      "loss": 3.2673,
      "nll_loss": 3.2571640014648438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10751312971115112,
      "rewards/margins": 1.2745389938354492,
      "rewards/rejected": -1.3820520639419556,
      "step": 1096
    },
    {
      "epoch": 0.6824261275272162,
      "grad_norm": 0.7112170457839966,
      "learning_rate": 2.2575000000000003e-05,
      "log_odds_chosen": 15.50126838684082,
      "log_odds_ratio": -0.17970341444015503,
      "logits/chosen": 0.13599060475826263,
      "logits/rejected": 0.8270803689956665,
      "logps/chosen": -1.0619562864303589,
      "logps/rejected": -16.088138580322266,
      "loss": 2.6784,
      "nll_loss": 2.6603970527648926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10619563609361649,
      "rewards/margins": 1.5026183128356934,
      "rewards/rejected": -1.6088138818740845,
      "step": 1097
    },
    {
      "epoch": 0.6830482115085537,
      "grad_norm": 0.5061715841293335,
      "learning_rate": 2.2550000000000003e-05,
      "log_odds_chosen": 13.113398551940918,
      "log_odds_ratio": -0.12759830057621002,
      "logits/chosen": 0.16755160689353943,
      "logits/rejected": 0.8419879674911499,
      "logps/chosen": -1.0674017667770386,
      "logps/rejected": -13.659807205200195,
      "loss": 2.6941,
      "nll_loss": 2.681312322616577,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10674017667770386,
      "rewards/margins": 1.259240746498108,
      "rewards/rejected": -1.365980863571167,
      "step": 1098
    },
    {
      "epoch": 0.6836702954898911,
      "grad_norm": 0.4506334960460663,
      "learning_rate": 2.2525000000000002e-05,
      "log_odds_chosen": 11.249850273132324,
      "log_odds_ratio": -0.1481650024652481,
      "logits/chosen": 0.16708631813526154,
      "logits/rejected": 0.6881266832351685,
      "logps/chosen": -1.1782821416854858,
      "logps/rejected": -12.08417797088623,
      "loss": 3.056,
      "nll_loss": 3.041147232055664,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11782821267843246,
      "rewards/margins": 1.0905896425247192,
      "rewards/rejected": -1.2084178924560547,
      "step": 1099
    },
    {
      "epoch": 0.6842923794712286,
      "grad_norm": 0.47914910316467285,
      "learning_rate": 2.25e-05,
      "log_odds_chosen": 13.498701095581055,
      "log_odds_ratio": -0.05041101947426796,
      "logits/chosen": 0.2943468689918518,
      "logits/rejected": 1.0841618776321411,
      "logps/chosen": -1.1848316192626953,
      "logps/rejected": -14.254444122314453,
      "loss": 3.3055,
      "nll_loss": 3.3004813194274902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11848315596580505,
      "rewards/margins": 1.3069612979888916,
      "rewards/rejected": -1.4254443645477295,
      "step": 1100
    },
    {
      "epoch": 0.6849144634525661,
      "grad_norm": 0.46730777621269226,
      "learning_rate": 2.2475e-05,
      "log_odds_chosen": 13.177154541015625,
      "log_odds_ratio": -0.20132030546665192,
      "logits/chosen": 0.19598224759101868,
      "logits/rejected": 0.6804916262626648,
      "logps/chosen": -1.008966088294983,
      "logps/rejected": -13.777807235717773,
      "loss": 3.1359,
      "nll_loss": 3.1157898902893066,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10089661180973053,
      "rewards/margins": 1.2768840789794922,
      "rewards/rejected": -1.3777806758880615,
      "step": 1101
    },
    {
      "epoch": 0.6855365474339036,
      "grad_norm": 0.5543920397758484,
      "learning_rate": 2.245e-05,
      "log_odds_chosen": 13.08626651763916,
      "log_odds_ratio": -0.08596739172935486,
      "logits/chosen": 0.12830127775669098,
      "logits/rejected": 0.7522597312927246,
      "logps/chosen": -1.0769314765930176,
      "logps/rejected": -13.698116302490234,
      "loss": 2.7265,
      "nll_loss": 2.7178595066070557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1076931580901146,
      "rewards/margins": 1.2621186971664429,
      "rewards/rejected": -1.369811773300171,
      "step": 1102
    },
    {
      "epoch": 0.686158631415241,
      "grad_norm": 0.5758435130119324,
      "learning_rate": 2.2425000000000003e-05,
      "log_odds_chosen": 14.17343521118164,
      "log_odds_ratio": -0.10952825099229813,
      "logits/chosen": 0.2742041349411011,
      "logits/rejected": 1.0324076414108276,
      "logps/chosen": -1.4365853071212769,
      "logps/rejected": -15.37224006652832,
      "loss": 2.8356,
      "nll_loss": 2.8246562480926514,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14365853369235992,
      "rewards/margins": 1.3935655355453491,
      "rewards/rejected": -1.5372240543365479,
      "step": 1103
    },
    {
      "epoch": 0.6867807153965786,
      "grad_norm": 0.36500388383865356,
      "learning_rate": 2.2400000000000002e-05,
      "log_odds_chosen": 16.786182403564453,
      "log_odds_ratio": -0.06503584235906601,
      "logits/chosen": 0.3020685911178589,
      "logits/rejected": 1.4873595237731934,
      "logps/chosen": -1.0615830421447754,
      "logps/rejected": -17.41973876953125,
      "loss": 3.3075,
      "nll_loss": 3.3009724617004395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1061583086848259,
      "rewards/margins": 1.6358156204223633,
      "rewards/rejected": -1.741973876953125,
      "step": 1104
    },
    {
      "epoch": 0.687402799377916,
      "grad_norm": 0.49695879220962524,
      "learning_rate": 2.2375000000000002e-05,
      "log_odds_chosen": 20.125638961791992,
      "log_odds_ratio": -6.066138666938059e-05,
      "logits/chosen": 0.2225799411535263,
      "logits/rejected": 1.438215970993042,
      "logps/chosen": -1.3540451526641846,
      "logps/rejected": -21.168962478637695,
      "loss": 3.1225,
      "nll_loss": 3.1224632263183594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1354045271873474,
      "rewards/margins": 1.9814918041229248,
      "rewards/rejected": -2.116896390914917,
      "step": 1105
    },
    {
      "epoch": 0.6880248833592535,
      "grad_norm": 0.6216310262680054,
      "learning_rate": 2.235e-05,
      "log_odds_chosen": 15.467144012451172,
      "log_odds_ratio": -0.0264641921967268,
      "logits/chosen": 0.22354130446910858,
      "logits/rejected": 0.8883647918701172,
      "logps/chosen": -0.950381338596344,
      "logps/rejected": -15.762426376342773,
      "loss": 3.0147,
      "nll_loss": 3.012061834335327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09503813087940216,
      "rewards/margins": 1.481204628944397,
      "rewards/rejected": -1.5762428045272827,
      "step": 1106
    },
    {
      "epoch": 0.6886469673405909,
      "grad_norm": 0.6510394811630249,
      "learning_rate": 2.2325e-05,
      "log_odds_chosen": 15.790122032165527,
      "log_odds_ratio": -0.0470377579331398,
      "logits/chosen": 0.07415720820426941,
      "logits/rejected": 0.7188281416893005,
      "logps/chosen": -1.2903048992156982,
      "logps/rejected": -16.766942977905273,
      "loss": 2.9234,
      "nll_loss": 2.918691635131836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1290304958820343,
      "rewards/margins": 1.547663927078247,
      "rewards/rejected": -1.676694393157959,
      "step": 1107
    },
    {
      "epoch": 0.6892690513219285,
      "grad_norm": 0.7006060481071472,
      "learning_rate": 2.23e-05,
      "log_odds_chosen": 14.18627643585205,
      "log_odds_ratio": -0.14766886830329895,
      "logits/chosen": 0.2061242163181305,
      "logits/rejected": 0.6048423051834106,
      "logps/chosen": -1.2005126476287842,
      "logps/rejected": -15.043365478515625,
      "loss": 3.36,
      "nll_loss": 3.345226526260376,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12005127221345901,
      "rewards/margins": 1.3842853307724,
      "rewards/rejected": -1.5043364763259888,
      "step": 1108
    },
    {
      "epoch": 0.689891135303266,
      "grad_norm": 11.92190933227539,
      "learning_rate": 2.2275000000000003e-05,
      "log_odds_chosen": 17.27753448486328,
      "log_odds_ratio": -0.09403269737958908,
      "logits/chosen": 0.3899608850479126,
      "logits/rejected": 1.604209303855896,
      "logps/chosen": -1.9780341386795044,
      "logps/rejected": -18.90896224975586,
      "loss": 3.4092,
      "nll_loss": 3.3998172283172607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19780340790748596,
      "rewards/margins": 1.6930928230285645,
      "rewards/rejected": -1.890896201133728,
      "step": 1109
    },
    {
      "epoch": 0.6905132192846034,
      "grad_norm": 0.5367807745933533,
      "learning_rate": 2.2250000000000002e-05,
      "log_odds_chosen": 11.975065231323242,
      "log_odds_ratio": -0.1697680503129959,
      "logits/chosen": 0.1529502123594284,
      "logits/rejected": 1.010935664176941,
      "logps/chosen": -1.0631258487701416,
      "logps/rejected": -12.682013511657715,
      "loss": 2.5578,
      "nll_loss": 2.5407814979553223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1063125804066658,
      "rewards/margins": 1.1618887186050415,
      "rewards/rejected": -1.2682013511657715,
      "step": 1110
    },
    {
      "epoch": 0.6911353032659409,
      "grad_norm": 0.5301330089569092,
      "learning_rate": 2.2225e-05,
      "log_odds_chosen": 10.99447250366211,
      "log_odds_ratio": -0.19437550008296967,
      "logits/chosen": 0.015375887975096703,
      "logits/rejected": 0.5180283188819885,
      "logps/chosen": -0.95285564661026,
      "logps/rejected": -11.372751235961914,
      "loss": 2.3729,
      "nll_loss": 2.3534395694732666,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.095285564661026,
      "rewards/margins": 1.0419895648956299,
      "rewards/rejected": -1.1372750997543335,
      "step": 1111
    },
    {
      "epoch": 0.6917573872472784,
      "grad_norm": 0.5872764587402344,
      "learning_rate": 2.22e-05,
      "log_odds_chosen": 17.98602294921875,
      "log_odds_ratio": -0.09409792721271515,
      "logits/chosen": 0.24407103657722473,
      "logits/rejected": 1.2107287645339966,
      "logps/chosen": -0.9928297996520996,
      "logps/rejected": -18.3500919342041,
      "loss": 2.6225,
      "nll_loss": 2.613097667694092,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09928297251462936,
      "rewards/margins": 1.735726237297058,
      "rewards/rejected": -1.8350093364715576,
      "step": 1112
    },
    {
      "epoch": 0.6923794712286159,
      "grad_norm": 0.5280458331108093,
      "learning_rate": 2.2175e-05,
      "log_odds_chosen": 9.216509819030762,
      "log_odds_ratio": -0.2553660571575165,
      "logits/chosen": 0.21463128924369812,
      "logits/rejected": 0.5493931174278259,
      "logps/chosen": -1.0679031610488892,
      "logps/rejected": -9.879371643066406,
      "loss": 3.2027,
      "nll_loss": 3.1772048473358154,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10679031163454056,
      "rewards/margins": 0.8811469078063965,
      "rewards/rejected": -0.9879372119903564,
      "step": 1113
    },
    {
      "epoch": 0.6930015552099533,
      "grad_norm": 0.44100382924079895,
      "learning_rate": 2.215e-05,
      "log_odds_chosen": 14.408587455749512,
      "log_odds_ratio": -0.0010577059583738446,
      "logits/chosen": 0.05236963927745819,
      "logits/rejected": 0.5131943225860596,
      "logps/chosen": -1.3580536842346191,
      "logps/rejected": -15.343432426452637,
      "loss": 3.184,
      "nll_loss": 3.1839027404785156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1358053833246231,
      "rewards/margins": 1.3985378742218018,
      "rewards/rejected": -1.5343432426452637,
      "step": 1114
    },
    {
      "epoch": 0.6936236391912908,
      "grad_norm": 0.5382852554321289,
      "learning_rate": 2.2125000000000002e-05,
      "log_odds_chosen": 9.10023307800293,
      "log_odds_ratio": -0.30452102422714233,
      "logits/chosen": 0.3505043387413025,
      "logits/rejected": 1.0762572288513184,
      "logps/chosen": -1.0695630311965942,
      "logps/rejected": -9.810070037841797,
      "loss": 3.3941,
      "nll_loss": 3.3636703491210938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10695630311965942,
      "rewards/margins": 0.8740507364273071,
      "rewards/rejected": -0.9810069799423218,
      "step": 1115
    },
    {
      "epoch": 0.6942457231726283,
      "grad_norm": 0.44020068645477295,
      "learning_rate": 2.2100000000000002e-05,
      "log_odds_chosen": 14.810802459716797,
      "log_odds_ratio": -0.10710399597883224,
      "logits/chosen": 0.11335402727127075,
      "logits/rejected": 0.7006421089172363,
      "logps/chosen": -1.4732582569122314,
      "logps/rejected": -15.886306762695312,
      "loss": 2.9517,
      "nll_loss": 2.9409916400909424,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1473258137702942,
      "rewards/margins": 1.4413048028945923,
      "rewards/rejected": -1.5886306762695312,
      "step": 1116
    },
    {
      "epoch": 0.6948678071539658,
      "grad_norm": 0.6491237282752991,
      "learning_rate": 2.2075e-05,
      "log_odds_chosen": 8.861434936523438,
      "log_odds_ratio": -0.09095767885446548,
      "logits/chosen": 0.22493115067481995,
      "logits/rejected": 0.7060920000076294,
      "logps/chosen": -1.0282622575759888,
      "logps/rejected": -9.425237655639648,
      "loss": 3.1569,
      "nll_loss": 3.1477737426757812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10282622277736664,
      "rewards/margins": 0.8396974802017212,
      "rewards/rejected": -0.942523717880249,
      "step": 1117
    },
    {
      "epoch": 0.6954898911353032,
      "grad_norm": 0.5764707326889038,
      "learning_rate": 2.205e-05,
      "log_odds_chosen": 7.087107181549072,
      "log_odds_ratio": -0.33189916610717773,
      "logits/chosen": 0.14153975248336792,
      "logits/rejected": 0.653849720954895,
      "logps/chosen": -1.2045460939407349,
      "logps/rejected": -8.065446853637695,
      "loss": 3.0227,
      "nll_loss": 2.9894912242889404,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12045460939407349,
      "rewards/margins": 0.6860900521278381,
      "rewards/rejected": -0.8065447211265564,
      "step": 1118
    },
    {
      "epoch": 0.6961119751166407,
      "grad_norm": 0.6903092265129089,
      "learning_rate": 2.2025e-05,
      "log_odds_chosen": 9.988282203674316,
      "log_odds_ratio": -0.2689623534679413,
      "logits/chosen": 0.18848875164985657,
      "logits/rejected": 0.3909173309803009,
      "logps/chosen": -1.037498950958252,
      "logps/rejected": -10.61701774597168,
      "loss": 3.3393,
      "nll_loss": 3.3123836517333984,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10374990105628967,
      "rewards/margins": 0.9579518437385559,
      "rewards/rejected": -1.061701774597168,
      "step": 1119
    },
    {
      "epoch": 0.6967340590979783,
      "grad_norm": 0.5290844440460205,
      "learning_rate": 2.2000000000000003e-05,
      "log_odds_chosen": 12.99549674987793,
      "log_odds_ratio": -0.04361700266599655,
      "logits/chosen": 0.2747880816459656,
      "logits/rejected": 0.681348443031311,
      "logps/chosen": -1.3809425830841064,
      "logps/rejected": -13.821148872375488,
      "loss": 3.3427,
      "nll_loss": 3.3383641242980957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13809427618980408,
      "rewards/margins": 1.244020700454712,
      "rewards/rejected": -1.3821148872375488,
      "step": 1120
    },
    {
      "epoch": 0.6973561430793157,
      "grad_norm": 0.4989076256752014,
      "learning_rate": 2.1975000000000002e-05,
      "log_odds_chosen": 10.855270385742188,
      "log_odds_ratio": -0.038061439990997314,
      "logits/chosen": 0.10623833537101746,
      "logits/rejected": 0.9076218605041504,
      "logps/chosen": -1.3005287647247314,
      "logps/rejected": -11.753395080566406,
      "loss": 2.8766,
      "nll_loss": 2.8727827072143555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1300528645515442,
      "rewards/margins": 1.0452866554260254,
      "rewards/rejected": -1.1753395795822144,
      "step": 1121
    },
    {
      "epoch": 0.6979782270606532,
      "grad_norm": 0.7603822946548462,
      "learning_rate": 2.195e-05,
      "log_odds_chosen": 11.2942476272583,
      "log_odds_ratio": -0.06391564011573792,
      "logits/chosen": 0.22070272266864777,
      "logits/rejected": 0.8907159566879272,
      "logps/chosen": -1.1149401664733887,
      "logps/rejected": -12.02164363861084,
      "loss": 2.6533,
      "nll_loss": 2.646906852722168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11149401217699051,
      "rewards/margins": 1.0906702280044556,
      "rewards/rejected": -1.2021644115447998,
      "step": 1122
    },
    {
      "epoch": 0.6986003110419907,
      "grad_norm": 0.46358850598335266,
      "learning_rate": 2.1925e-05,
      "log_odds_chosen": 16.277067184448242,
      "log_odds_ratio": -0.106183260679245,
      "logits/chosen": 0.12363559007644653,
      "logits/rejected": 0.7034668326377869,
      "logps/chosen": -1.018523931503296,
      "logps/rejected": -16.79071807861328,
      "loss": 2.6021,
      "nll_loss": 2.591519594192505,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1018524020910263,
      "rewards/margins": 1.5772193670272827,
      "rewards/rejected": -1.6790719032287598,
      "step": 1123
    },
    {
      "epoch": 0.6992223950233282,
      "grad_norm": 0.6551591753959656,
      "learning_rate": 2.19e-05,
      "log_odds_chosen": 6.512795448303223,
      "log_odds_ratio": -0.046365853399038315,
      "logits/chosen": 0.25244051218032837,
      "logits/rejected": 0.5576794147491455,
      "logps/chosen": -1.0957938432693481,
      "logps/rejected": -7.111298084259033,
      "loss": 2.7942,
      "nll_loss": 2.7895545959472656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10957939177751541,
      "rewards/margins": 0.6015503406524658,
      "rewards/rejected": -0.7111297249794006,
      "step": 1124
    },
    {
      "epoch": 0.6998444790046656,
      "grad_norm": 0.662030041217804,
      "learning_rate": 2.1875e-05,
      "log_odds_chosen": 13.827706336975098,
      "log_odds_ratio": -0.19535228610038757,
      "logits/chosen": 0.3746417462825775,
      "logits/rejected": 1.1256225109100342,
      "logps/chosen": -1.1726419925689697,
      "logps/rejected": -14.544219970703125,
      "loss": 2.8034,
      "nll_loss": 2.783874273300171,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11726421117782593,
      "rewards/margins": 1.3371577262878418,
      "rewards/rejected": -1.4544219970703125,
      "step": 1125
    },
    {
      "epoch": 0.7004665629860031,
      "grad_norm": 4.096120357513428,
      "learning_rate": 2.1850000000000003e-05,
      "log_odds_chosen": 20.036169052124023,
      "log_odds_ratio": -0.013790621422231197,
      "logits/chosen": 0.20078325271606445,
      "logits/rejected": 0.9888613820075989,
      "logps/chosen": -0.9814634919166565,
      "logps/rejected": -20.45991325378418,
      "loss": 1.9901,
      "nll_loss": 1.9886776208877563,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09814634919166565,
      "rewards/margins": 1.9478448629379272,
      "rewards/rejected": -2.0459911823272705,
      "step": 1126
    },
    {
      "epoch": 0.7010886469673406,
      "grad_norm": 0.5734096169471741,
      "learning_rate": 2.1825000000000002e-05,
      "log_odds_chosen": 13.67259693145752,
      "log_odds_ratio": -0.04071996361017227,
      "logits/chosen": 0.17778745293617249,
      "logits/rejected": 0.9131073951721191,
      "logps/chosen": -1.2617847919464111,
      "logps/rejected": -14.583654403686523,
      "loss": 2.9613,
      "nll_loss": 2.9572203159332275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12617850303649902,
      "rewards/margins": 1.3321870565414429,
      "rewards/rejected": -1.4583654403686523,
      "step": 1127
    },
    {
      "epoch": 0.7017107309486781,
      "grad_norm": 0.4533398747444153,
      "learning_rate": 2.18e-05,
      "log_odds_chosen": 10.825969696044922,
      "log_odds_ratio": -0.026933560147881508,
      "logits/chosen": 0.24595864117145538,
      "logits/rejected": 0.6292278170585632,
      "logps/chosen": -1.0519578456878662,
      "logps/rejected": -11.445229530334473,
      "loss": 3.5149,
      "nll_loss": 3.512205123901367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1051957830786705,
      "rewards/margins": 1.0393271446228027,
      "rewards/rejected": -1.1445229053497314,
      "step": 1128
    },
    {
      "epoch": 0.7023328149300155,
      "grad_norm": 4.797630310058594,
      "learning_rate": 2.1775e-05,
      "log_odds_chosen": 14.788459777832031,
      "log_odds_ratio": -0.03414743393659592,
      "logits/chosen": 0.15490153431892395,
      "logits/rejected": 0.9039937257766724,
      "logps/chosen": -2.220851421356201,
      "logps/rejected": -16.61078453063965,
      "loss": 2.912,
      "nll_loss": 2.908588409423828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2220851480960846,
      "rewards/margins": 1.4389934539794922,
      "rewards/rejected": -1.6610784530639648,
      "step": 1129
    },
    {
      "epoch": 0.702954898911353,
      "grad_norm": 2.169843912124634,
      "learning_rate": 2.175e-05,
      "log_odds_chosen": 10.93464183807373,
      "log_odds_ratio": -0.214813232421875,
      "logits/chosen": 0.16185230016708374,
      "logits/rejected": 0.7838496565818787,
      "logps/chosen": -1.3578927516937256,
      "logps/rejected": -12.01943588256836,
      "loss": 2.6853,
      "nll_loss": 2.6637721061706543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13578927516937256,
      "rewards/margins": 1.0661543607711792,
      "rewards/rejected": -1.2019436359405518,
      "step": 1130
    },
    {
      "epoch": 0.7035769828926906,
      "grad_norm": 0.8374118804931641,
      "learning_rate": 2.1725e-05,
      "log_odds_chosen": 13.784649848937988,
      "log_odds_ratio": -0.0661187693476677,
      "logits/chosen": 0.20649418234825134,
      "logits/rejected": 0.9395582675933838,
      "logps/chosen": -1.278693437576294,
      "logps/rejected": -14.600447654724121,
      "loss": 3.1107,
      "nll_loss": 3.1040682792663574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1278693526983261,
      "rewards/margins": 1.3321754932403564,
      "rewards/rejected": -1.4600448608398438,
      "step": 1131
    },
    {
      "epoch": 0.704199066874028,
      "grad_norm": 0.378974974155426,
      "learning_rate": 2.1700000000000002e-05,
      "log_odds_chosen": 14.10969352722168,
      "log_odds_ratio": -0.10490639507770538,
      "logits/chosen": 0.17890948057174683,
      "logits/rejected": 0.7284139394760132,
      "logps/chosen": -1.0358339548110962,
      "logps/rejected": -14.461109161376953,
      "loss": 3.1646,
      "nll_loss": 3.1540842056274414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10358339548110962,
      "rewards/margins": 1.3425275087356567,
      "rewards/rejected": -1.4461108446121216,
      "step": 1132
    },
    {
      "epoch": 0.7048211508553655,
      "grad_norm": 0.41824260354042053,
      "learning_rate": 2.1675e-05,
      "log_odds_chosen": 11.248109817504883,
      "log_odds_ratio": -0.08439905941486359,
      "logits/chosen": 0.22522927820682526,
      "logits/rejected": 0.8612341284751892,
      "logps/chosen": -0.9619340896606445,
      "logps/rejected": -11.490693092346191,
      "loss": 3.2554,
      "nll_loss": 3.2469184398651123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09619341045618057,
      "rewards/margins": 1.0528758764266968,
      "rewards/rejected": -1.1490693092346191,
      "step": 1133
    },
    {
      "epoch": 0.7054432348367029,
      "grad_norm": 0.5656015872955322,
      "learning_rate": 2.165e-05,
      "log_odds_chosen": 9.454839706420898,
      "log_odds_ratio": -0.18580016493797302,
      "logits/chosen": 0.1890837401151657,
      "logits/rejected": 0.6705374717712402,
      "logps/chosen": -1.2932785749435425,
      "logps/rejected": -10.466346740722656,
      "loss": 3.2662,
      "nll_loss": 3.2475852966308594,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12932786345481873,
      "rewards/margins": 0.9173068404197693,
      "rewards/rejected": -1.0466346740722656,
      "step": 1134
    },
    {
      "epoch": 0.7060653188180405,
      "grad_norm": 0.48979511857032776,
      "learning_rate": 2.1625e-05,
      "log_odds_chosen": 8.302387237548828,
      "log_odds_ratio": -0.10199929028749466,
      "logits/chosen": 0.2716232240200043,
      "logits/rejected": 0.7033010125160217,
      "logps/chosen": -1.0194844007492065,
      "logps/rejected": -8.780854225158691,
      "loss": 2.9983,
      "nll_loss": 2.9880552291870117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10194844752550125,
      "rewards/margins": 0.7761369943618774,
      "rewards/rejected": -0.8780854940414429,
      "step": 1135
    },
    {
      "epoch": 0.7066874027993779,
      "grad_norm": 0.7489378452301025,
      "learning_rate": 2.16e-05,
      "log_odds_chosen": 10.226913452148438,
      "log_odds_ratio": -0.23562005162239075,
      "logits/chosen": 0.32799530029296875,
      "logits/rejected": 0.7691959142684937,
      "logps/chosen": -1.3166583776474,
      "logps/rejected": -11.2990083694458,
      "loss": 3.5254,
      "nll_loss": 3.501816511154175,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13166584074497223,
      "rewards/margins": 0.9982350468635559,
      "rewards/rejected": -1.1299008131027222,
      "step": 1136
    },
    {
      "epoch": 0.7073094867807154,
      "grad_norm": 0.4850473701953888,
      "learning_rate": 2.1575e-05,
      "log_odds_chosen": 7.751132488250732,
      "log_odds_ratio": -0.12384605407714844,
      "logits/chosen": 0.0676664263010025,
      "logits/rejected": 0.3730185627937317,
      "logps/chosen": -1.1014970541000366,
      "logps/rejected": -8.484350204467773,
      "loss": 2.7592,
      "nll_loss": 2.746859312057495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11014970391988754,
      "rewards/margins": 0.7382854223251343,
      "rewards/rejected": -0.8484350442886353,
      "step": 1137
    },
    {
      "epoch": 0.7079315707620529,
      "grad_norm": 0.5743958950042725,
      "learning_rate": 2.1550000000000002e-05,
      "log_odds_chosen": 11.897217750549316,
      "log_odds_ratio": -0.22582495212554932,
      "logits/chosen": 0.14629925787448883,
      "logits/rejected": 0.6454797983169556,
      "logps/chosen": -1.0044411420822144,
      "logps/rejected": -12.506041526794434,
      "loss": 3.048,
      "nll_loss": 3.025416135787964,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10044410824775696,
      "rewards/margins": 1.1501600742340088,
      "rewards/rejected": -1.2506040334701538,
      "step": 1138
    },
    {
      "epoch": 0.7085536547433904,
      "grad_norm": 0.38661685585975647,
      "learning_rate": 2.1525e-05,
      "log_odds_chosen": 8.775971412658691,
      "log_odds_ratio": -0.05596862733364105,
      "logits/chosen": 0.31263870000839233,
      "logits/rejected": 1.1504080295562744,
      "logps/chosen": -1.1882120370864868,
      "logps/rejected": -9.621572494506836,
      "loss": 3.3544,
      "nll_loss": 3.3488073348999023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11882121115922928,
      "rewards/margins": 0.8433359861373901,
      "rewards/rejected": -0.9621572494506836,
      "step": 1139
    },
    {
      "epoch": 0.7091757387247278,
      "grad_norm": 0.4194515645503998,
      "learning_rate": 2.15e-05,
      "log_odds_chosen": 7.772745609283447,
      "log_odds_ratio": -0.15363647043704987,
      "logits/chosen": 0.05815325677394867,
      "logits/rejected": 0.4343949854373932,
      "logps/chosen": -1.258744478225708,
      "logps/rejected": -8.714080810546875,
      "loss": 2.8419,
      "nll_loss": 2.8265483379364014,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12587445974349976,
      "rewards/margins": 0.7455336451530457,
      "rewards/rejected": -0.8714081048965454,
      "step": 1140
    },
    {
      "epoch": 0.7097978227060653,
      "grad_norm": 0.5058637261390686,
      "learning_rate": 2.1475e-05,
      "log_odds_chosen": 8.859088897705078,
      "log_odds_ratio": -0.19793754816055298,
      "logits/chosen": 0.10581560432910919,
      "logits/rejected": 0.7795206904411316,
      "logps/chosen": -1.165850043296814,
      "logps/rejected": -9.689474105834961,
      "loss": 2.6781,
      "nll_loss": 2.658310651779175,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11658500134944916,
      "rewards/margins": 0.8523623943328857,
      "rewards/rejected": -0.9689474105834961,
      "step": 1141
    },
    {
      "epoch": 0.7104199066874028,
      "grad_norm": 0.5065990090370178,
      "learning_rate": 2.145e-05,
      "log_odds_chosen": 7.223630905151367,
      "log_odds_ratio": -0.19884894788265228,
      "logits/chosen": 0.01418336108326912,
      "logits/rejected": 0.1823025494813919,
      "logps/chosen": -1.134019136428833,
      "logps/rejected": -7.986211776733398,
      "loss": 2.8221,
      "nll_loss": 2.802203416824341,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11340190470218658,
      "rewards/margins": 0.6852192878723145,
      "rewards/rejected": -0.7986211776733398,
      "step": 1142
    },
    {
      "epoch": 0.7110419906687403,
      "grad_norm": 1.9043158292770386,
      "learning_rate": 2.1425e-05,
      "log_odds_chosen": 7.798857688903809,
      "log_odds_ratio": -0.3148168623447418,
      "logits/chosen": -0.015639889985322952,
      "logits/rejected": 0.4448607861995697,
      "logps/chosen": -1.3772170543670654,
      "logps/rejected": -8.906472206115723,
      "loss": 2.6039,
      "nll_loss": 2.5723979473114014,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1377217024564743,
      "rewards/margins": 0.7529255747795105,
      "rewards/rejected": -0.8906472325325012,
      "step": 1143
    },
    {
      "epoch": 0.7116640746500777,
      "grad_norm": 0.4393058717250824,
      "learning_rate": 2.1400000000000002e-05,
      "log_odds_chosen": 9.942191123962402,
      "log_odds_ratio": -0.08139564841985703,
      "logits/chosen": 0.32947584986686707,
      "logits/rejected": 0.6434736251831055,
      "logps/chosen": -1.153541088104248,
      "logps/rejected": -10.689937591552734,
      "loss": 3.8071,
      "nll_loss": 3.7989487648010254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11535410583019257,
      "rewards/margins": 0.953639566898346,
      "rewards/rejected": -1.0689938068389893,
      "step": 1144
    },
    {
      "epoch": 0.7122861586314152,
      "grad_norm": 0.6094390749931335,
      "learning_rate": 2.1375e-05,
      "log_odds_chosen": 8.202652931213379,
      "log_odds_ratio": -0.24067726731300354,
      "logits/chosen": 0.2636764943599701,
      "logits/rejected": 0.8318098783493042,
      "logps/chosen": -1.185968041419983,
      "logps/rejected": -9.15146255493164,
      "loss": 2.7167,
      "nll_loss": 2.6925878524780273,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11859681457281113,
      "rewards/margins": 0.7965494394302368,
      "rewards/rejected": -0.9151462316513062,
      "step": 1145
    },
    {
      "epoch": 0.7129082426127528,
      "grad_norm": 0.4632388651371002,
      "learning_rate": 2.135e-05,
      "log_odds_chosen": 12.712577819824219,
      "log_odds_ratio": -0.0024499651044607162,
      "logits/chosen": 0.21387840807437897,
      "logits/rejected": 0.5963307023048401,
      "logps/chosen": -0.9470509886741638,
      "logps/rejected": -12.899580955505371,
      "loss": 3.5905,
      "nll_loss": 3.590237617492676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09470509737730026,
      "rewards/margins": 1.1952530145645142,
      "rewards/rejected": -1.289958119392395,
      "step": 1146
    },
    {
      "epoch": 0.7135303265940902,
      "grad_norm": 0.6410781741142273,
      "learning_rate": 2.1325e-05,
      "log_odds_chosen": 9.504288673400879,
      "log_odds_ratio": -0.2655012309551239,
      "logits/chosen": 0.016082381829619408,
      "logits/rejected": 0.46460607647895813,
      "logps/chosen": -1.0009565353393555,
      "logps/rejected": -10.186278343200684,
      "loss": 2.1953,
      "nll_loss": 2.16878080368042,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10009565949440002,
      "rewards/margins": 0.9185321927070618,
      "rewards/rejected": -1.0186278820037842,
      "step": 1147
    },
    {
      "epoch": 0.7141524105754277,
      "grad_norm": 2.453840494155884,
      "learning_rate": 2.13e-05,
      "log_odds_chosen": 15.4404935836792,
      "log_odds_ratio": -0.07841021567583084,
      "logits/chosen": 0.31317567825317383,
      "logits/rejected": 1.2708203792572021,
      "logps/chosen": -1.4197622537612915,
      "logps/rejected": -16.562503814697266,
      "loss": 3.4137,
      "nll_loss": 3.405905246734619,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1419762372970581,
      "rewards/margins": 1.5142741203308105,
      "rewards/rejected": -1.6562504768371582,
      "step": 1148
    },
    {
      "epoch": 0.7147744945567651,
      "grad_norm": 0.601822018623352,
      "learning_rate": 2.1275000000000002e-05,
      "log_odds_chosen": 6.593123435974121,
      "log_odds_ratio": -0.3577158451080322,
      "logits/chosen": 0.14801143109798431,
      "logits/rejected": 0.33144474029541016,
      "logps/chosen": -1.1508957147598267,
      "logps/rejected": -7.38616418838501,
      "loss": 3.1124,
      "nll_loss": 3.0766658782958984,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11508957296609879,
      "rewards/margins": 0.623526930809021,
      "rewards/rejected": -0.738616406917572,
      "step": 1149
    },
    {
      "epoch": 0.7153965785381027,
      "grad_norm": 0.49013224244117737,
      "learning_rate": 2.125e-05,
      "log_odds_chosen": 9.185380935668945,
      "log_odds_ratio": -0.061306171119213104,
      "logits/chosen": 0.13947425782680511,
      "logits/rejected": 0.6735214591026306,
      "logps/chosen": -1.1954495906829834,
      "logps/rejected": -9.900875091552734,
      "loss": 3.3637,
      "nll_loss": 3.357612133026123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11954495310783386,
      "rewards/margins": 0.870542585849762,
      "rewards/rejected": -0.9900875091552734,
      "step": 1150
    },
    {
      "epoch": 0.7160186625194401,
      "grad_norm": 0.7656285166740417,
      "learning_rate": 2.1225e-05,
      "log_odds_chosen": 5.765224933624268,
      "log_odds_ratio": -0.44179147481918335,
      "logits/chosen": 0.28343236446380615,
      "logits/rejected": 0.5950878858566284,
      "logps/chosen": -1.435044765472412,
      "logps/rejected": -7.039450645446777,
      "loss": 3.4903,
      "nll_loss": 3.446166515350342,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14350447058677673,
      "rewards/margins": 0.5604405999183655,
      "rewards/rejected": -0.7039451003074646,
      "step": 1151
    },
    {
      "epoch": 0.7166407465007776,
      "grad_norm": 0.5079005360603333,
      "learning_rate": 2.12e-05,
      "log_odds_chosen": 11.404075622558594,
      "log_odds_ratio": -0.010933519341051579,
      "logits/chosen": 0.22485215961933136,
      "logits/rejected": 1.0881459712982178,
      "logps/chosen": -0.9811384677886963,
      "logps/rejected": -11.846966743469238,
      "loss": 3.0934,
      "nll_loss": 3.092313289642334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09811384975910187,
      "rewards/margins": 1.086582899093628,
      "rewards/rejected": -1.1846967935562134,
      "step": 1152
    },
    {
      "epoch": 0.717262830482115,
      "grad_norm": 0.5223401784896851,
      "learning_rate": 2.1175e-05,
      "log_odds_chosen": 11.468758583068848,
      "log_odds_ratio": -0.06769321858882904,
      "logits/chosen": 0.16616982221603394,
      "logits/rejected": 0.9403862953186035,
      "logps/chosen": -1.4131141901016235,
      "logps/rejected": -12.616225242614746,
      "loss": 2.7869,
      "nll_loss": 2.7801098823547363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1413114219903946,
      "rewards/margins": 1.1203110218048096,
      "rewards/rejected": -1.261622428894043,
      "step": 1153
    },
    {
      "epoch": 0.7178849144634526,
      "grad_norm": 0.6829721331596375,
      "learning_rate": 2.115e-05,
      "log_odds_chosen": 12.060152053833008,
      "log_odds_ratio": -0.08659256994724274,
      "logits/chosen": 0.30466094613075256,
      "logits/rejected": 1.010999321937561,
      "logps/chosen": -1.0162991285324097,
      "logps/rejected": -12.572675704956055,
      "loss": 2.9428,
      "nll_loss": 2.9341888427734375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10162991285324097,
      "rewards/margins": 1.1556376218795776,
      "rewards/rejected": -1.2572674751281738,
      "step": 1154
    },
    {
      "epoch": 0.71850699844479,
      "grad_norm": 0.4815412163734436,
      "learning_rate": 2.1125000000000002e-05,
      "log_odds_chosen": 15.830819129943848,
      "log_odds_ratio": -1.4529315194522496e-05,
      "logits/chosen": 0.10796906799077988,
      "logits/rejected": 1.0258997678756714,
      "logps/chosen": -1.0770153999328613,
      "logps/rejected": -16.436660766601562,
      "loss": 2.605,
      "nll_loss": 2.6049692630767822,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10770154744386673,
      "rewards/margins": 1.5359647274017334,
      "rewards/rejected": -1.64366614818573,
      "step": 1155
    },
    {
      "epoch": 0.7191290824261275,
      "grad_norm": 0.5743563175201416,
      "learning_rate": 2.11e-05,
      "log_odds_chosen": 5.235894203186035,
      "log_odds_ratio": -0.3245120048522949,
      "logits/chosen": 0.1398739218711853,
      "logits/rejected": 0.36136549711227417,
      "logps/chosen": -0.993781566619873,
      "logps/rejected": -5.917914390563965,
      "loss": 2.8791,
      "nll_loss": 2.8466100692749023,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09937816113233566,
      "rewards/margins": 0.4924132525920868,
      "rewards/rejected": -0.5917914509773254,
      "step": 1156
    },
    {
      "epoch": 0.7197511664074651,
      "grad_norm": 0.39877840876579285,
      "learning_rate": 2.1075e-05,
      "log_odds_chosen": 9.245841026306152,
      "log_odds_ratio": -0.13619744777679443,
      "logits/chosen": 0.3372015058994293,
      "logits/rejected": 0.5857275724411011,
      "logps/chosen": -0.8995043635368347,
      "logps/rejected": -9.580745697021484,
      "loss": 3.9532,
      "nll_loss": 3.939554452896118,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08995043486356735,
      "rewards/margins": 0.8681241273880005,
      "rewards/rejected": -0.9580744504928589,
      "step": 1157
    },
    {
      "epoch": 0.7203732503888025,
      "grad_norm": 0.4954541027545929,
      "learning_rate": 2.105e-05,
      "log_odds_chosen": 7.235276222229004,
      "log_odds_ratio": -0.4196551442146301,
      "logits/chosen": 0.24806271493434906,
      "logits/rejected": 0.6260412931442261,
      "logps/chosen": -1.104307770729065,
      "logps/rejected": -8.139066696166992,
      "loss": 2.7239,
      "nll_loss": 2.681976079940796,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1104307696223259,
      "rewards/margins": 0.703475832939148,
      "rewards/rejected": -0.8139066696166992,
      "step": 1158
    },
    {
      "epoch": 0.72099533437014,
      "grad_norm": 0.5344847440719604,
      "learning_rate": 2.1025e-05,
      "log_odds_chosen": 8.476509094238281,
      "log_odds_ratio": -0.1521918922662735,
      "logits/chosen": 0.05145689472556114,
      "logits/rejected": 0.5481857061386108,
      "logps/chosen": -0.9891431331634521,
      "logps/rejected": -9.036706924438477,
      "loss": 2.502,
      "nll_loss": 2.486774206161499,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09891431033611298,
      "rewards/margins": 0.8047564029693604,
      "rewards/rejected": -0.9036707878112793,
      "step": 1159
    },
    {
      "epoch": 0.7216174183514774,
      "grad_norm": 0.42739880084991455,
      "learning_rate": 2.1e-05,
      "log_odds_chosen": 4.0024614334106445,
      "log_odds_ratio": -0.2541469633579254,
      "logits/chosen": 0.2720869779586792,
      "logits/rejected": 0.26361843943595886,
      "logps/chosen": -1.196718692779541,
      "logps/rejected": -4.947728633880615,
      "loss": 3.7707,
      "nll_loss": 3.745288372039795,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11967186629772186,
      "rewards/margins": 0.3751010000705719,
      "rewards/rejected": -0.49477288126945496,
      "step": 1160
    },
    {
      "epoch": 0.722239502332815,
      "grad_norm": 0.3615352511405945,
      "learning_rate": 2.0975e-05,
      "log_odds_chosen": 10.977020263671875,
      "log_odds_ratio": -0.09265509247779846,
      "logits/chosen": 0.10474146902561188,
      "logits/rejected": 0.502599835395813,
      "logps/chosen": -0.8850367665290833,
      "logps/rejected": -11.234212875366211,
      "loss": 2.697,
      "nll_loss": 2.6877830028533936,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08850367367267609,
      "rewards/margins": 1.0349177122116089,
      "rewards/rejected": -1.1234214305877686,
      "step": 1161
    },
    {
      "epoch": 0.7228615863141524,
      "grad_norm": 0.597377359867096,
      "learning_rate": 2.095e-05,
      "log_odds_chosen": 10.106073379516602,
      "log_odds_ratio": -0.1266728639602661,
      "logits/chosen": 0.26163730025291443,
      "logits/rejected": 0.9807031154632568,
      "logps/chosen": -1.2175838947296143,
      "logps/rejected": -10.95804500579834,
      "loss": 2.8703,
      "nll_loss": 2.8576431274414062,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12175838649272919,
      "rewards/margins": 0.9740461111068726,
      "rewards/rejected": -1.0958044528961182,
      "step": 1162
    },
    {
      "epoch": 0.7234836702954899,
      "grad_norm": 0.6204828023910522,
      "learning_rate": 2.0925e-05,
      "log_odds_chosen": 10.355432510375977,
      "log_odds_ratio": -0.12557263672351837,
      "logits/chosen": 0.21904480457305908,
      "logits/rejected": 0.7396771311759949,
      "logps/chosen": -1.0724170207977295,
      "logps/rejected": -10.99392318725586,
      "loss": 2.9639,
      "nll_loss": 2.951324224472046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10724169760942459,
      "rewards/margins": 0.9921507239341736,
      "rewards/rejected": -1.0993924140930176,
      "step": 1163
    },
    {
      "epoch": 0.7241057542768273,
      "grad_norm": 0.3468632698059082,
      "learning_rate": 2.09e-05,
      "log_odds_chosen": 7.67006254196167,
      "log_odds_ratio": -0.1240687444806099,
      "logits/chosen": 0.30150219798088074,
      "logits/rejected": 0.5070857405662537,
      "logps/chosen": -1.1041569709777832,
      "logps/rejected": -8.211764335632324,
      "loss": 3.6166,
      "nll_loss": 3.604192018508911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11041569709777832,
      "rewards/margins": 0.710760772228241,
      "rewards/rejected": -0.8211764693260193,
      "step": 1164
    },
    {
      "epoch": 0.7247278382581649,
      "grad_norm": 0.5467685461044312,
      "learning_rate": 2.0875e-05,
      "log_odds_chosen": 7.7747802734375,
      "log_odds_ratio": -0.09820210188627243,
      "logits/chosen": 0.24225030839443207,
      "logits/rejected": 0.515572190284729,
      "logps/chosen": -0.9557793736457825,
      "logps/rejected": -7.976073265075684,
      "loss": 3.2347,
      "nll_loss": 3.2248952388763428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09557794034481049,
      "rewards/margins": 0.7020293474197388,
      "rewards/rejected": -0.7976073622703552,
      "step": 1165
    },
    {
      "epoch": 0.7253499222395023,
      "grad_norm": 0.5304726362228394,
      "learning_rate": 2.085e-05,
      "log_odds_chosen": 8.300542831420898,
      "log_odds_ratio": -0.4137728810310364,
      "logits/chosen": 0.22537773847579956,
      "logits/rejected": 0.5865037441253662,
      "logps/chosen": -0.9393926858901978,
      "logps/rejected": -8.798408508300781,
      "loss": 2.8672,
      "nll_loss": 2.825866460800171,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09393927454948425,
      "rewards/margins": 0.785901665687561,
      "rewards/rejected": -0.8798408508300781,
      "step": 1166
    },
    {
      "epoch": 0.7259720062208398,
      "grad_norm": 0.5175058245658875,
      "learning_rate": 2.0825e-05,
      "log_odds_chosen": 7.007765769958496,
      "log_odds_ratio": -0.257230281829834,
      "logits/chosen": 0.0327136255800724,
      "logits/rejected": 0.591026246547699,
      "logps/chosen": -0.9878470301628113,
      "logps/rejected": -7.626405239105225,
      "loss": 1.8304,
      "nll_loss": 1.8046331405639648,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09878470748662949,
      "rewards/margins": 0.6638558506965637,
      "rewards/rejected": -0.7626405954360962,
      "step": 1167
    },
    {
      "epoch": 0.7265940902021772,
      "grad_norm": 0.4381992220878601,
      "learning_rate": 2.08e-05,
      "log_odds_chosen": 10.2841215133667,
      "log_odds_ratio": -0.010029378347098827,
      "logits/chosen": 0.3482717275619507,
      "logits/rejected": 1.1683536767959595,
      "logps/chosen": -1.002194881439209,
      "logps/rejected": -10.818892478942871,
      "loss": 2.7471,
      "nll_loss": 2.7460508346557617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1002194732427597,
      "rewards/margins": 0.9816697835922241,
      "rewards/rejected": -1.081889271736145,
      "step": 1168
    },
    {
      "epoch": 0.7272161741835148,
      "grad_norm": 0.9522091150283813,
      "learning_rate": 2.0775e-05,
      "log_odds_chosen": 13.303786277770996,
      "log_odds_ratio": -0.09151265770196915,
      "logits/chosen": 0.16112789511680603,
      "logits/rejected": 0.7550581693649292,
      "logps/chosen": -1.8442214727401733,
      "logps/rejected": -14.88370132446289,
      "loss": 2.7658,
      "nll_loss": 2.756631851196289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18442216515541077,
      "rewards/margins": 1.3039480447769165,
      "rewards/rejected": -1.4883701801300049,
      "step": 1169
    },
    {
      "epoch": 0.7278382581648523,
      "grad_norm": 0.5005118250846863,
      "learning_rate": 2.075e-05,
      "log_odds_chosen": 9.357019424438477,
      "log_odds_ratio": -0.1308477371931076,
      "logits/chosen": 0.16465628147125244,
      "logits/rejected": 0.4376339018344879,
      "logps/chosen": -1.0926101207733154,
      "logps/rejected": -9.923200607299805,
      "loss": 3.1157,
      "nll_loss": 3.1025681495666504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10926100611686707,
      "rewards/margins": 0.883059024810791,
      "rewards/rejected": -0.9923200607299805,
      "step": 1170
    },
    {
      "epoch": 0.7284603421461897,
      "grad_norm": 0.5556590557098389,
      "learning_rate": 2.0725e-05,
      "log_odds_chosen": 8.283122062683105,
      "log_odds_ratio": -0.22327569127082825,
      "logits/chosen": 0.0778142660856247,
      "logits/rejected": 0.35505211353302,
      "logps/chosen": -0.9077842235565186,
      "logps/rejected": -8.489635467529297,
      "loss": 2.5729,
      "nll_loss": 2.550588607788086,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0907784253358841,
      "rewards/margins": 0.758185088634491,
      "rewards/rejected": -0.8489634990692139,
      "step": 1171
    },
    {
      "epoch": 0.7290824261275272,
      "grad_norm": 0.46830663084983826,
      "learning_rate": 2.07e-05,
      "log_odds_chosen": 6.0396928787231445,
      "log_odds_ratio": -0.23946940898895264,
      "logits/chosen": 0.1411685347557068,
      "logits/rejected": 0.3880601227283478,
      "logps/chosen": -1.1312123537063599,
      "logps/rejected": -6.760911464691162,
      "loss": 3.0798,
      "nll_loss": 3.055830717086792,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11312123388051987,
      "rewards/margins": 0.562969982624054,
      "rewards/rejected": -0.676091194152832,
      "step": 1172
    },
    {
      "epoch": 0.7297045101088647,
      "grad_norm": 2.1477103233337402,
      "learning_rate": 2.0675e-05,
      "log_odds_chosen": 9.316595077514648,
      "log_odds_ratio": -0.3114702105522156,
      "logits/chosen": 0.14469477534294128,
      "logits/rejected": 0.43679317831993103,
      "logps/chosen": -1.1385347843170166,
      "logps/rejected": -9.948036193847656,
      "loss": 2.5376,
      "nll_loss": 2.5064356327056885,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11385348439216614,
      "rewards/margins": 0.8809502124786377,
      "rewards/rejected": -0.9948036670684814,
      "step": 1173
    },
    {
      "epoch": 0.7303265940902022,
      "grad_norm": 0.48001450300216675,
      "learning_rate": 2.065e-05,
      "log_odds_chosen": 13.939992904663086,
      "log_odds_ratio": -0.14188605546951294,
      "logits/chosen": 0.16776786744594574,
      "logits/rejected": 0.987214207649231,
      "logps/chosen": -0.7123444676399231,
      "logps/rejected": -13.940017700195312,
      "loss": 2.7787,
      "nll_loss": 2.764474391937256,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07123444974422455,
      "rewards/margins": 1.3227672576904297,
      "rewards/rejected": -1.394001841545105,
      "step": 1174
    },
    {
      "epoch": 0.7309486780715396,
      "grad_norm": 0.46293526887893677,
      "learning_rate": 2.0625e-05,
      "log_odds_chosen": 7.249242782592773,
      "log_odds_ratio": -0.24686285853385925,
      "logits/chosen": 0.16540293395519257,
      "logits/rejected": 0.614818811416626,
      "logps/chosen": -1.5638196468353271,
      "logps/rejected": -8.594873428344727,
      "loss": 2.9741,
      "nll_loss": 2.9493930339813232,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15638196468353271,
      "rewards/margins": 0.7031053900718689,
      "rewards/rejected": -0.8594873547554016,
      "step": 1175
    },
    {
      "epoch": 0.7315707620528772,
      "grad_norm": 0.3698181211948395,
      "learning_rate": 2.06e-05,
      "log_odds_chosen": 9.808906555175781,
      "log_odds_ratio": -0.025691796094179153,
      "logits/chosen": 0.38358452916145325,
      "logits/rejected": 0.8628143072128296,
      "logps/chosen": -1.1391582489013672,
      "logps/rejected": -10.532305717468262,
      "loss": 3.9937,
      "nll_loss": 3.991159439086914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1139158234000206,
      "rewards/margins": 0.9393147826194763,
      "rewards/rejected": -1.0532306432724,
      "step": 1176
    },
    {
      "epoch": 0.7321928460342146,
      "grad_norm": 0.3679634928703308,
      "learning_rate": 2.0575e-05,
      "log_odds_chosen": 6.190854549407959,
      "log_odds_ratio": -0.20879708230495453,
      "logits/chosen": 0.4313851594924927,
      "logits/rejected": 0.8396764993667603,
      "logps/chosen": -1.062203049659729,
      "logps/rejected": -6.8761305809021,
      "loss": 3.8049,
      "nll_loss": 3.7840020656585693,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1062203049659729,
      "rewards/margins": 0.581392765045166,
      "rewards/rejected": -0.6876130700111389,
      "step": 1177
    },
    {
      "epoch": 0.7328149300155521,
      "grad_norm": 0.5036177039146423,
      "learning_rate": 2.055e-05,
      "log_odds_chosen": 9.593914985656738,
      "log_odds_ratio": -0.14493709802627563,
      "logits/chosen": 0.303832471370697,
      "logits/rejected": 0.7071282863616943,
      "logps/chosen": -1.0161112546920776,
      "logps/rejected": -10.205410957336426,
      "loss": 3.3057,
      "nll_loss": 3.291208267211914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10161112993955612,
      "rewards/margins": 0.9189299941062927,
      "rewards/rejected": -1.0205411911010742,
      "step": 1178
    },
    {
      "epoch": 0.7334370139968895,
      "grad_norm": 0.6572273969650269,
      "learning_rate": 2.0525e-05,
      "log_odds_chosen": 12.69178581237793,
      "log_odds_ratio": -0.24341538548469543,
      "logits/chosen": 0.22046558558940887,
      "logits/rejected": 0.9849398136138916,
      "logps/chosen": -1.237760066986084,
      "logps/rejected": -13.610360145568848,
      "loss": 2.8629,
      "nll_loss": 2.8385534286499023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12377601116895676,
      "rewards/margins": 1.237259864807129,
      "rewards/rejected": -1.3610360622406006,
      "step": 1179
    },
    {
      "epoch": 0.7340590979782271,
      "grad_norm": 0.41224968433380127,
      "learning_rate": 2.05e-05,
      "log_odds_chosen": 6.243864059448242,
      "log_odds_ratio": -0.4956444501876831,
      "logits/chosen": 0.09433531761169434,
      "logits/rejected": 0.4423058032989502,
      "logps/chosen": -1.109994649887085,
      "logps/rejected": -7.267676830291748,
      "loss": 3.0106,
      "nll_loss": 2.9610538482666016,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11099947988986969,
      "rewards/margins": 0.6157682538032532,
      "rewards/rejected": -0.7267676591873169,
      "step": 1180
    },
    {
      "epoch": 0.7346811819595646,
      "grad_norm": 0.5551636219024658,
      "learning_rate": 2.0475e-05,
      "log_odds_chosen": 12.350042343139648,
      "log_odds_ratio": -0.10465382039546967,
      "logits/chosen": 0.1587362140417099,
      "logits/rejected": 0.8673467636108398,
      "logps/chosen": -1.106932520866394,
      "logps/rejected": -13.051389694213867,
      "loss": 2.8993,
      "nll_loss": 2.8888251781463623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11069324612617493,
      "rewards/margins": 1.1944458484649658,
      "rewards/rejected": -1.3051390647888184,
      "step": 1181
    },
    {
      "epoch": 0.735303265940902,
      "grad_norm": 0.45288923382759094,
      "learning_rate": 2.045e-05,
      "log_odds_chosen": 17.103357315063477,
      "log_odds_ratio": -1.2666027942032088e-06,
      "logits/chosen": 0.30812063813209534,
      "logits/rejected": 1.196515679359436,
      "logps/chosen": -0.8819127082824707,
      "logps/rejected": -17.416370391845703,
      "loss": 3.2262,
      "nll_loss": 3.226205825805664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08819127082824707,
      "rewards/margins": 1.6534457206726074,
      "rewards/rejected": -1.7416372299194336,
      "step": 1182
    },
    {
      "epoch": 0.7359253499222395,
      "grad_norm": 0.5740547180175781,
      "learning_rate": 2.0425e-05,
      "log_odds_chosen": 11.144023895263672,
      "log_odds_ratio": -0.13786615431308746,
      "logits/chosen": 0.33946895599365234,
      "logits/rejected": 1.2792378664016724,
      "logps/chosen": -1.425415277481079,
      "logps/rejected": -12.355165481567383,
      "loss": 2.9367,
      "nll_loss": 2.9228994846343994,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1425415277481079,
      "rewards/margins": 1.0929750204086304,
      "rewards/rejected": -1.2355165481567383,
      "step": 1183
    },
    {
      "epoch": 0.736547433903577,
      "grad_norm": 0.5895634889602661,
      "learning_rate": 2.04e-05,
      "log_odds_chosen": 14.83269214630127,
      "log_odds_ratio": -0.07303274422883987,
      "logits/chosen": 0.09696874767541885,
      "logits/rejected": 1.0919651985168457,
      "logps/chosen": -1.0911352634429932,
      "logps/rejected": -15.479246139526367,
      "loss": 2.4192,
      "nll_loss": 2.411862373352051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10911353677511215,
      "rewards/margins": 1.4388110637664795,
      "rewards/rejected": -1.5479246377944946,
      "step": 1184
    },
    {
      "epoch": 0.7371695178849145,
      "grad_norm": 0.46647828817367554,
      "learning_rate": 2.0375e-05,
      "log_odds_chosen": 10.80234146118164,
      "log_odds_ratio": -0.21883045136928558,
      "logits/chosen": 0.19847188889980316,
      "logits/rejected": 0.8998786211013794,
      "logps/chosen": -1.2073338031768799,
      "logps/rejected": -11.712199211120605,
      "loss": 3.0496,
      "nll_loss": 3.027686595916748,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12073338031768799,
      "rewards/margins": 1.0504865646362305,
      "rewards/rejected": -1.171220064163208,
      "step": 1185
    },
    {
      "epoch": 0.7377916018662519,
      "grad_norm": 0.5875625014305115,
      "learning_rate": 2.035e-05,
      "log_odds_chosen": 4.775179862976074,
      "log_odds_ratio": -0.42339253425598145,
      "logits/chosen": 0.04878817871212959,
      "logits/rejected": 0.5890315175056458,
      "logps/chosen": -1.3256160020828247,
      "logps/rejected": -5.962397575378418,
      "loss": 2.4948,
      "nll_loss": 2.4524922370910645,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.132561594247818,
      "rewards/margins": 0.46367818117141724,
      "rewards/rejected": -0.5962398052215576,
      "step": 1186
    },
    {
      "epoch": 0.7384136858475894,
      "grad_norm": 0.5900309085845947,
      "learning_rate": 2.0325e-05,
      "log_odds_chosen": 12.905298233032227,
      "log_odds_ratio": -0.13436618447303772,
      "logits/chosen": 0.22156783938407898,
      "logits/rejected": 0.969600260257721,
      "logps/chosen": -1.015760898590088,
      "logps/rejected": -13.515449523925781,
      "loss": 2.7867,
      "nll_loss": 2.7732279300689697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10157608985900879,
      "rewards/margins": 1.2499690055847168,
      "rewards/rejected": -1.3515450954437256,
      "step": 1187
    },
    {
      "epoch": 0.7390357698289269,
      "grad_norm": 0.4624529182910919,
      "learning_rate": 2.0300000000000002e-05,
      "log_odds_chosen": 17.161376953125,
      "log_odds_ratio": -6.544519419549033e-05,
      "logits/chosen": 0.1179753914475441,
      "logits/rejected": 0.8243229985237122,
      "logps/chosen": -1.4011576175689697,
      "logps/rejected": -18.230491638183594,
      "loss": 3.0534,
      "nll_loss": 3.053412437438965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14011575281620026,
      "rewards/margins": 1.6829335689544678,
      "rewards/rejected": -1.8230493068695068,
      "step": 1188
    },
    {
      "epoch": 0.7396578538102644,
      "grad_norm": 0.393663614988327,
      "learning_rate": 2.0275e-05,
      "log_odds_chosen": 15.431196212768555,
      "log_odds_ratio": -3.939667294616811e-05,
      "logits/chosen": 0.39448249340057373,
      "logits/rejected": 1.1981537342071533,
      "logps/chosen": -1.0742273330688477,
      "logps/rejected": -15.890814781188965,
      "loss": 3.54,
      "nll_loss": 3.5400190353393555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10742272436618805,
      "rewards/margins": 1.4816588163375854,
      "rewards/rejected": -1.5890816450119019,
      "step": 1189
    },
    {
      "epoch": 0.7402799377916018,
      "grad_norm": 0.6178926825523376,
      "learning_rate": 2.025e-05,
      "log_odds_chosen": 9.652229309082031,
      "log_odds_ratio": -0.16254249215126038,
      "logits/chosen": 0.3837137520313263,
      "logits/rejected": 0.9006325006484985,
      "logps/chosen": -1.0126757621765137,
      "logps/rejected": -10.24832534790039,
      "loss": 3.3,
      "nll_loss": 3.28379487991333,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10126757621765137,
      "rewards/margins": 0.9235650300979614,
      "rewards/rejected": -1.0248326063156128,
      "step": 1190
    },
    {
      "epoch": 0.7409020217729394,
      "grad_norm": 0.5684689283370972,
      "learning_rate": 2.0225000000000004e-05,
      "log_odds_chosen": 11.131904602050781,
      "log_odds_ratio": -0.11341032385826111,
      "logits/chosen": 0.2865346074104309,
      "logits/rejected": 0.7895666360855103,
      "logps/chosen": -0.9756239056587219,
      "logps/rejected": -11.580063819885254,
      "loss": 3.0371,
      "nll_loss": 3.0257105827331543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09756238758563995,
      "rewards/margins": 1.0604439973831177,
      "rewards/rejected": -1.1580064296722412,
      "step": 1191
    },
    {
      "epoch": 0.7415241057542769,
      "grad_norm": 0.47814252972602844,
      "learning_rate": 2.0200000000000003e-05,
      "log_odds_chosen": 13.986754417419434,
      "log_odds_ratio": -0.060169950127601624,
      "logits/chosen": 0.5951923727989197,
      "logits/rejected": 1.518169641494751,
      "logps/chosen": -1.3645962476730347,
      "logps/rejected": -15.049614906311035,
      "loss": 4.0235,
      "nll_loss": 4.01747465133667,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.136459618806839,
      "rewards/margins": 1.368501901626587,
      "rewards/rejected": -1.5049614906311035,
      "step": 1192
    },
    {
      "epoch": 0.7421461897356143,
      "grad_norm": 0.4866500794887543,
      "learning_rate": 2.0175000000000003e-05,
      "log_odds_chosen": 13.448404312133789,
      "log_odds_ratio": -0.11860945075750351,
      "logits/chosen": 0.3314950466156006,
      "logits/rejected": 1.0111912488937378,
      "logps/chosen": -1.384364366531372,
      "logps/rejected": -14.545913696289062,
      "loss": 3.0946,
      "nll_loss": 3.0826945304870605,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1384364366531372,
      "rewards/margins": 1.316154956817627,
      "rewards/rejected": -1.4545913934707642,
      "step": 1193
    },
    {
      "epoch": 0.7427682737169518,
      "grad_norm": 0.6490217447280884,
      "learning_rate": 2.0150000000000002e-05,
      "log_odds_chosen": 16.155012130737305,
      "log_odds_ratio": -0.07999815791845322,
      "logits/chosen": 0.10705263912677765,
      "logits/rejected": 1.093653917312622,
      "logps/chosen": -1.271554708480835,
      "logps/rejected": -17.101104736328125,
      "loss": 2.5376,
      "nll_loss": 2.529562473297119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12715548276901245,
      "rewards/margins": 1.5829551219940186,
      "rewards/rejected": -1.7101104259490967,
      "step": 1194
    },
    {
      "epoch": 0.7433903576982893,
      "grad_norm": 0.5161392688751221,
      "learning_rate": 2.0125e-05,
      "log_odds_chosen": 11.450945854187012,
      "log_odds_ratio": -0.14187860488891602,
      "logits/chosen": 0.4051249921321869,
      "logits/rejected": 1.25907301902771,
      "logps/chosen": -1.2657084465026855,
      "logps/rejected": -12.435002326965332,
      "loss": 3.3354,
      "nll_loss": 3.3212480545043945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12657085061073303,
      "rewards/margins": 1.1169294118881226,
      "rewards/rejected": -1.2435002326965332,
      "step": 1195
    },
    {
      "epoch": 0.7440124416796268,
      "grad_norm": 0.4983889162540436,
      "learning_rate": 2.01e-05,
      "log_odds_chosen": 15.152313232421875,
      "log_odds_ratio": -0.002952311420813203,
      "logits/chosen": 0.20715823769569397,
      "logits/rejected": 1.1005414724349976,
      "logps/chosen": -1.2786905765533447,
      "logps/rejected": -16.09111213684082,
      "loss": 2.8896,
      "nll_loss": 2.8893094062805176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12786906957626343,
      "rewards/margins": 1.4812421798706055,
      "rewards/rejected": -1.6091113090515137,
      "step": 1196
    },
    {
      "epoch": 0.7446345256609642,
      "grad_norm": 0.3791523277759552,
      "learning_rate": 2.0075000000000003e-05,
      "log_odds_chosen": 16.003755569458008,
      "log_odds_ratio": -0.0002835804771166295,
      "logits/chosen": 0.3261744976043701,
      "logits/rejected": 1.0396631956100464,
      "logps/chosen": -0.8366090059280396,
      "logps/rejected": -15.989496231079102,
      "loss": 3.6367,
      "nll_loss": 3.6366686820983887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08366090059280396,
      "rewards/margins": 1.5152887105941772,
      "rewards/rejected": -1.598949670791626,
      "step": 1197
    },
    {
      "epoch": 0.7452566096423017,
      "grad_norm": 0.594624936580658,
      "learning_rate": 2.0050000000000003e-05,
      "log_odds_chosen": 15.558506965637207,
      "log_odds_ratio": -0.0004107086278963834,
      "logits/chosen": 0.210353821516037,
      "logits/rejected": 0.9793530106544495,
      "logps/chosen": -0.8724642395973206,
      "logps/rejected": -15.786218643188477,
      "loss": 2.7662,
      "nll_loss": 2.7662012577056885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08724642544984818,
      "rewards/margins": 1.49137544631958,
      "rewards/rejected": -1.5786218643188477,
      "step": 1198
    },
    {
      "epoch": 0.7458786936236392,
      "grad_norm": 3.2293317317962646,
      "learning_rate": 2.0025000000000002e-05,
      "log_odds_chosen": 12.671496391296387,
      "log_odds_ratio": -0.10842613130807877,
      "logits/chosen": 0.23085221648216248,
      "logits/rejected": 1.1595039367675781,
      "logps/chosen": -1.3611061573028564,
      "logps/rejected": -13.689292907714844,
      "loss": 2.904,
      "nll_loss": 2.8931233882904053,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13611061871051788,
      "rewards/margins": 1.232818603515625,
      "rewards/rejected": -1.3689292669296265,
      "step": 1199
    },
    {
      "epoch": 0.7465007776049767,
      "grad_norm": 0.4771862328052521,
      "learning_rate": 2e-05,
      "log_odds_chosen": 22.102876663208008,
      "log_odds_ratio": -0.001636207103729248,
      "logits/chosen": 0.20666168630123138,
      "logits/rejected": 1.0230656862258911,
      "logps/chosen": -1.34181547164917,
      "logps/rejected": -23.12851333618164,
      "loss": 3.159,
      "nll_loss": 3.1588516235351562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13418152928352356,
      "rewards/margins": 2.1786699295043945,
      "rewards/rejected": -2.3128514289855957,
      "step": 1200
    },
    {
      "epoch": 0.7471228615863141,
      "grad_norm": 0.4705706536769867,
      "learning_rate": 1.9975e-05,
      "log_odds_chosen": 13.418475151062012,
      "log_odds_ratio": -0.17137664556503296,
      "logits/chosen": 0.3189517855644226,
      "logits/rejected": 0.7763543725013733,
      "logps/chosen": -1.1539502143859863,
      "logps/rejected": -14.102031707763672,
      "loss": 3.4356,
      "nll_loss": 3.418506383895874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11539502441883087,
      "rewards/margins": 1.2948081493377686,
      "rewards/rejected": -1.410203218460083,
      "step": 1201
    },
    {
      "epoch": 0.7477449455676516,
      "grad_norm": 1.8948168754577637,
      "learning_rate": 1.995e-05,
      "log_odds_chosen": 16.753978729248047,
      "log_odds_ratio": -2.4275623218272813e-05,
      "logits/chosen": 0.33771950006484985,
      "logits/rejected": 1.016837239265442,
      "logps/chosen": -3.4603779315948486,
      "logps/rejected": -19.40519905090332,
      "loss": 3.5931,
      "nll_loss": 3.593096971511841,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3460378050804138,
      "rewards/margins": 1.594482183456421,
      "rewards/rejected": -1.94051992893219,
      "step": 1202
    },
    {
      "epoch": 0.7483670295489891,
      "grad_norm": 0.5406678915023804,
      "learning_rate": 1.9925000000000003e-05,
      "log_odds_chosen": 13.841324806213379,
      "log_odds_ratio": -0.025037458166480064,
      "logits/chosen": 0.422190397977829,
      "logits/rejected": 1.2798744440078735,
      "logps/chosen": -1.0436694622039795,
      "logps/rejected": -14.358189582824707,
      "loss": 3.5902,
      "nll_loss": 3.5876641273498535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10436695069074631,
      "rewards/margins": 1.3314520120620728,
      "rewards/rejected": -1.4358189105987549,
      "step": 1203
    },
    {
      "epoch": 0.7489891135303266,
      "grad_norm": 0.8551453948020935,
      "learning_rate": 1.9900000000000003e-05,
      "log_odds_chosen": 9.493968963623047,
      "log_odds_ratio": -0.198939248919487,
      "logits/chosen": 0.15752126276493073,
      "logits/rejected": 0.5775033235549927,
      "logps/chosen": -2.0665013790130615,
      "logps/rejected": -11.248618125915527,
      "loss": 3.215,
      "nll_loss": 3.1950745582580566,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20665015280246735,
      "rewards/margins": 0.9182117581367493,
      "rewards/rejected": -1.1248618364334106,
      "step": 1204
    },
    {
      "epoch": 0.749611197511664,
      "grad_norm": 0.5271828770637512,
      "learning_rate": 1.9875000000000002e-05,
      "log_odds_chosen": 12.457223892211914,
      "log_odds_ratio": -0.1530085653066635,
      "logits/chosen": 0.21353940665721893,
      "logits/rejected": 0.7389912605285645,
      "logps/chosen": -1.1457003355026245,
      "logps/rejected": -13.223892211914062,
      "loss": 2.9948,
      "nll_loss": 2.9794840812683105,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11457002907991409,
      "rewards/margins": 1.2078192234039307,
      "rewards/rejected": -1.3223892450332642,
      "step": 1205
    },
    {
      "epoch": 0.7502332814930015,
      "grad_norm": 0.769097089767456,
      "learning_rate": 1.985e-05,
      "log_odds_chosen": 14.107221603393555,
      "log_odds_ratio": -0.1394292563199997,
      "logits/chosen": 0.155813530087471,
      "logits/rejected": 1.3736224174499512,
      "logps/chosen": -1.1750849485397339,
      "logps/rejected": -14.946660995483398,
      "loss": 2.6975,
      "nll_loss": 2.6836023330688477,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11750849336385727,
      "rewards/margins": 1.3771576881408691,
      "rewards/rejected": -1.4946662187576294,
      "step": 1206
    },
    {
      "epoch": 0.7508553654743391,
      "grad_norm": 2.034923553466797,
      "learning_rate": 1.9825e-05,
      "log_odds_chosen": 9.450210571289062,
      "log_odds_ratio": -0.09710147231817245,
      "logits/chosen": 0.12488899379968643,
      "logits/rejected": 0.5777722597122192,
      "logps/chosen": -1.1520278453826904,
      "logps/rejected": -10.04898738861084,
      "loss": 3.1795,
      "nll_loss": 3.1698226928710938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11520278453826904,
      "rewards/margins": 0.8896960020065308,
      "rewards/rejected": -1.0048987865447998,
      "step": 1207
    },
    {
      "epoch": 0.7514774494556765,
      "grad_norm": 1.6488580703735352,
      "learning_rate": 1.9800000000000004e-05,
      "log_odds_chosen": 7.367974281311035,
      "log_odds_ratio": -0.11552385240793228,
      "logits/chosen": 0.10463400185108185,
      "logits/rejected": 0.6088777184486389,
      "logps/chosen": -1.4335150718688965,
      "logps/rejected": -8.32715892791748,
      "loss": 2.7639,
      "nll_loss": 2.7523436546325684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1433514952659607,
      "rewards/margins": 0.6893644332885742,
      "rewards/rejected": -0.8327159285545349,
      "step": 1208
    },
    {
      "epoch": 0.752099533437014,
      "grad_norm": 0.4398881196975708,
      "learning_rate": 1.9775000000000003e-05,
      "log_odds_chosen": 8.662324905395508,
      "log_odds_ratio": -0.0030407479498535395,
      "logits/chosen": 0.07866299152374268,
      "logits/rejected": 0.6986607909202576,
      "logps/chosen": -1.4439903497695923,
      "logps/rejected": -9.80562973022461,
      "loss": 3.1564,
      "nll_loss": 3.1560733318328857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14439904689788818,
      "rewards/margins": 0.8361639976501465,
      "rewards/rejected": -0.9805630445480347,
      "step": 1209
    },
    {
      "epoch": 0.7527216174183515,
      "grad_norm": 0.5735336542129517,
      "learning_rate": 1.9750000000000002e-05,
      "log_odds_chosen": 5.275170803070068,
      "log_odds_ratio": -0.3947235345840454,
      "logits/chosen": 0.18500526249408722,
      "logits/rejected": 0.6523492932319641,
      "logps/chosen": -1.1334285736083984,
      "logps/rejected": -6.1533002853393555,
      "loss": 3.3193,
      "nll_loss": 3.279778003692627,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11334285140037537,
      "rewards/margins": 0.5019872188568115,
      "rewards/rejected": -0.6153301000595093,
      "step": 1210
    },
    {
      "epoch": 0.753343701399689,
      "grad_norm": 0.4635184407234192,
      "learning_rate": 1.9725000000000002e-05,
      "log_odds_chosen": 4.65528678894043,
      "log_odds_ratio": -0.32718393206596375,
      "logits/chosen": 0.12875179946422577,
      "logits/rejected": 0.4438689351081848,
      "logps/chosen": -1.1663978099822998,
      "logps/rejected": -5.531558036804199,
      "loss": 3.2701,
      "nll_loss": 3.2373604774475098,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11663977801799774,
      "rewards/margins": 0.43651604652404785,
      "rewards/rejected": -0.5531558394432068,
      "step": 1211
    },
    {
      "epoch": 0.7539657853810264,
      "grad_norm": 0.37020620703697205,
      "learning_rate": 1.97e-05,
      "log_odds_chosen": 7.262178897857666,
      "log_odds_ratio": -0.1279047131538391,
      "logits/chosen": 0.03921634703874588,
      "logits/rejected": 0.5070078372955322,
      "logps/chosen": -1.1373803615570068,
      "logps/rejected": -8.027933120727539,
      "loss": 2.796,
      "nll_loss": 2.783205032348633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1137380301952362,
      "rewards/margins": 0.6890553832054138,
      "rewards/rejected": -0.8027933835983276,
      "step": 1212
    },
    {
      "epoch": 0.7545878693623639,
      "grad_norm": 0.5390399694442749,
      "learning_rate": 1.9675e-05,
      "log_odds_chosen": 4.89652156829834,
      "log_odds_ratio": -0.0828741043806076,
      "logits/chosen": 0.10957112163305283,
      "logits/rejected": 0.38588717579841614,
      "logps/chosen": -0.9967465400695801,
      "logps/rejected": -5.436730861663818,
      "loss": 2.9941,
      "nll_loss": 2.985842704772949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09967464953660965,
      "rewards/margins": 0.44399845600128174,
      "rewards/rejected": -0.5436730980873108,
      "step": 1213
    },
    {
      "epoch": 0.7552099533437014,
      "grad_norm": 0.5554744601249695,
      "learning_rate": 1.9650000000000003e-05,
      "log_odds_chosen": 5.830607891082764,
      "log_odds_ratio": -0.15100139379501343,
      "logits/chosen": 0.3689972162246704,
      "logits/rejected": 0.6765709519386292,
      "logps/chosen": -1.0688714981079102,
      "logps/rejected": -6.527162551879883,
      "loss": 3.2168,
      "nll_loss": 3.2016830444335938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10688715428113937,
      "rewards/margins": 0.545829176902771,
      "rewards/rejected": -0.6527162790298462,
      "step": 1214
    },
    {
      "epoch": 0.7558320373250389,
      "grad_norm": 0.4852057695388794,
      "learning_rate": 1.9625000000000003e-05,
      "log_odds_chosen": 6.636116981506348,
      "log_odds_ratio": -0.2725330591201782,
      "logits/chosen": 0.10936164855957031,
      "logits/rejected": 0.3851650059223175,
      "logps/chosen": -1.1456758975982666,
      "logps/rejected": -7.426181793212891,
      "loss": 3.1894,
      "nll_loss": 3.1621837615966797,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1145675927400589,
      "rewards/margins": 0.6280506253242493,
      "rewards/rejected": -0.742618203163147,
      "step": 1215
    },
    {
      "epoch": 0.7564541213063763,
      "grad_norm": 0.4729749262332916,
      "learning_rate": 1.9600000000000002e-05,
      "log_odds_chosen": 3.1161768436431885,
      "log_odds_ratio": -0.2609820067882538,
      "logits/chosen": -0.028117988258600235,
      "logits/rejected": 0.07570795714855194,
      "logps/chosen": -1.1444841623306274,
      "logps/rejected": -3.978320598602295,
      "loss": 2.8225,
      "nll_loss": 2.796421766281128,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1144484132528305,
      "rewards/margins": 0.28338363766670227,
      "rewards/rejected": -0.39783206582069397,
      "step": 1216
    },
    {
      "epoch": 0.7570762052877138,
      "grad_norm": 1.1897724866867065,
      "learning_rate": 1.9575e-05,
      "log_odds_chosen": 6.136231422424316,
      "log_odds_ratio": -0.28731441497802734,
      "logits/chosen": 0.09058015048503876,
      "logits/rejected": 0.37315765023231506,
      "logps/chosen": -1.25123929977417,
      "logps/rejected": -6.890491008758545,
      "loss": 2.9212,
      "nll_loss": 2.8924193382263184,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12512391805648804,
      "rewards/margins": 0.5639252066612244,
      "rewards/rejected": -0.6890491247177124,
      "step": 1217
    },
    {
      "epoch": 0.7576982892690514,
      "grad_norm": 0.6197946667671204,
      "learning_rate": 1.955e-05,
      "log_odds_chosen": 6.062702655792236,
      "log_odds_ratio": -0.08814633637666702,
      "logits/chosen": 0.21369720995426178,
      "logits/rejected": 0.5749257206916809,
      "logps/chosen": -1.2833783626556396,
      "logps/rejected": -7.003604412078857,
      "loss": 2.8867,
      "nll_loss": 2.877854347229004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12833784520626068,
      "rewards/margins": 0.572022557258606,
      "rewards/rejected": -0.7003604173660278,
      "step": 1218
    },
    {
      "epoch": 0.7583203732503888,
      "grad_norm": 0.48757174611091614,
      "learning_rate": 1.9525e-05,
      "log_odds_chosen": 6.600275039672852,
      "log_odds_ratio": -0.08514364063739777,
      "logits/chosen": 0.21454694867134094,
      "logits/rejected": 0.7358257174491882,
      "logps/chosen": -0.9961150288581848,
      "logps/rejected": -7.084440231323242,
      "loss": 2.9164,
      "nll_loss": 2.90787672996521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09961150586605072,
      "rewards/margins": 0.6088324785232544,
      "rewards/rejected": -0.7084439992904663,
      "step": 1219
    },
    {
      "epoch": 0.7589424572317263,
      "grad_norm": 0.45170265436172485,
      "learning_rate": 1.9500000000000003e-05,
      "log_odds_chosen": 6.516654968261719,
      "log_odds_ratio": -0.2116156965494156,
      "logits/chosen": 0.2163967788219452,
      "logits/rejected": 0.47737014293670654,
      "logps/chosen": -1.4186476469039917,
      "logps/rejected": -7.741004943847656,
      "loss": 3.5083,
      "nll_loss": 3.487144947052002,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14186474680900574,
      "rewards/margins": 0.6322357654571533,
      "rewards/rejected": -0.7741005420684814,
      "step": 1220
    },
    {
      "epoch": 0.7595645412130637,
      "grad_norm": 0.33577945828437805,
      "learning_rate": 1.9475000000000002e-05,
      "log_odds_chosen": 5.055993556976318,
      "log_odds_ratio": -0.16657014191150665,
      "logits/chosen": 0.1790454089641571,
      "logits/rejected": 0.42467498779296875,
      "logps/chosen": -0.9846662282943726,
      "logps/rejected": -5.626519680023193,
      "loss": 3.2669,
      "nll_loss": 3.250199556350708,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09846662729978561,
      "rewards/margins": 0.46418535709381104,
      "rewards/rejected": -0.5626519918441772,
      "step": 1221
    },
    {
      "epoch": 0.7601866251944013,
      "grad_norm": 0.6504151821136475,
      "learning_rate": 1.9450000000000002e-05,
      "log_odds_chosen": 6.201626777648926,
      "log_odds_ratio": -0.06464290618896484,
      "logits/chosen": 0.04870191961526871,
      "logits/rejected": 0.5722784996032715,
      "logps/chosen": -0.9842745661735535,
      "logps/rejected": -6.750699043273926,
      "loss": 2.2712,
      "nll_loss": 2.264713764190674,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09842745959758759,
      "rewards/margins": 0.5766425132751465,
      "rewards/rejected": -0.6750699281692505,
      "step": 1222
    },
    {
      "epoch": 0.7608087091757387,
      "grad_norm": 0.7092140913009644,
      "learning_rate": 1.9425e-05,
      "log_odds_chosen": 2.3777732849121094,
      "log_odds_ratio": -0.3354724943637848,
      "logits/chosen": 0.11024624109268188,
      "logits/rejected": 0.2564601004123688,
      "logps/chosen": -1.0713608264923096,
      "logps/rejected": -3.2102227210998535,
      "loss": 2.7334,
      "nll_loss": 2.6998963356018066,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1071360781788826,
      "rewards/margins": 0.21388620138168335,
      "rewards/rejected": -0.32102227210998535,
      "step": 1223
    },
    {
      "epoch": 0.7614307931570762,
      "grad_norm": 0.48227447271347046,
      "learning_rate": 1.94e-05,
      "log_odds_chosen": 4.966604232788086,
      "log_odds_ratio": -0.19418928027153015,
      "logits/chosen": 0.17129558324813843,
      "logits/rejected": 0.4398013651371002,
      "logps/chosen": -1.319381833076477,
      "logps/rejected": -5.987258434295654,
      "loss": 3.2791,
      "nll_loss": 3.2596778869628906,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13193818926811218,
      "rewards/margins": 0.4667876362800598,
      "rewards/rejected": -0.5987257957458496,
      "step": 1224
    },
    {
      "epoch": 0.7620528771384136,
      "grad_norm": 0.5294142365455627,
      "learning_rate": 1.9375e-05,
      "log_odds_chosen": 3.5622966289520264,
      "log_odds_ratio": -0.25366440415382385,
      "logits/chosen": 0.3200629949569702,
      "logits/rejected": 0.5575484037399292,
      "logps/chosen": -1.051979899406433,
      "logps/rejected": -4.283415794372559,
      "loss": 3.1231,
      "nll_loss": 3.0977699756622314,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1051979809999466,
      "rewards/margins": 0.3231435716152191,
      "rewards/rejected": -0.4283415675163269,
      "step": 1225
    },
    {
      "epoch": 0.7626749611197512,
      "grad_norm": 1.2428282499313354,
      "learning_rate": 1.9350000000000003e-05,
      "log_odds_chosen": 5.334478378295898,
      "log_odds_ratio": -0.14889080822467804,
      "logits/chosen": 0.2575344145298004,
      "logits/rejected": 0.5913892984390259,
      "logps/chosen": -1.0900267362594604,
      "logps/rejected": -6.102575778961182,
      "loss": 3.0042,
      "nll_loss": 2.9893410205841064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10900267213582993,
      "rewards/margins": 0.5012549161911011,
      "rewards/rejected": -0.610257625579834,
      "step": 1226
    },
    {
      "epoch": 0.7632970451010886,
      "grad_norm": 0.9420454502105713,
      "learning_rate": 1.9325000000000002e-05,
      "log_odds_chosen": 6.546796798706055,
      "log_odds_ratio": -0.18457339704036713,
      "logits/chosen": 0.03549651801586151,
      "logits/rejected": 0.4685467481613159,
      "logps/chosen": -1.2222778797149658,
      "logps/rejected": -7.488654136657715,
      "loss": 2.3855,
      "nll_loss": 2.367008924484253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12222778797149658,
      "rewards/margins": 0.6266376972198486,
      "rewards/rejected": -0.7488654851913452,
      "step": 1227
    },
    {
      "epoch": 0.7639191290824261,
      "grad_norm": 0.5488473176956177,
      "learning_rate": 1.93e-05,
      "log_odds_chosen": 3.563775062561035,
      "log_odds_ratio": -0.5443033576011658,
      "logits/chosen": 0.23504066467285156,
      "logits/rejected": 0.41619762778282166,
      "logps/chosen": -1.547841191291809,
      "logps/rejected": -5.002470970153809,
      "loss": 3.5391,
      "nll_loss": 3.4846863746643066,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15478411316871643,
      "rewards/margins": 0.34546297788619995,
      "rewards/rejected": -0.5002471208572388,
      "step": 1228
    },
    {
      "epoch": 0.7645412130637637,
      "grad_norm": 0.8373052477836609,
      "learning_rate": 1.9275e-05,
      "log_odds_chosen": 3.503401041030884,
      "log_odds_ratio": -0.4526802897453308,
      "logits/chosen": 0.2677290439605713,
      "logits/rejected": 0.4133756160736084,
      "logps/chosen": -1.0986255407333374,
      "logps/rejected": -4.06947660446167,
      "loss": 3.2135,
      "nll_loss": 3.1682019233703613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1098625585436821,
      "rewards/margins": 0.29708513617515564,
      "rewards/rejected": -0.40694767236709595,
      "step": 1229
    },
    {
      "epoch": 0.7651632970451011,
      "grad_norm": 0.5462814569473267,
      "learning_rate": 1.925e-05,
      "log_odds_chosen": 7.460938930511475,
      "log_odds_ratio": -0.36012497544288635,
      "logits/chosen": 0.15392319858074188,
      "logits/rejected": 0.5377407670021057,
      "logps/chosen": -1.2988390922546387,
      "logps/rejected": -8.361101150512695,
      "loss": 2.9064,
      "nll_loss": 2.8703439235687256,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12988391518592834,
      "rewards/margins": 0.7062261700630188,
      "rewards/rejected": -0.8361101150512695,
      "step": 1230
    },
    {
      "epoch": 0.7657853810264386,
      "grad_norm": 0.70634526014328,
      "learning_rate": 1.9225e-05,
      "log_odds_chosen": 6.823087692260742,
      "log_odds_ratio": -0.15130549669265747,
      "logits/chosen": 0.24029546976089478,
      "logits/rejected": 0.8185093402862549,
      "logps/chosen": -1.2564562559127808,
      "logps/rejected": -7.753248691558838,
      "loss": 2.9292,
      "nll_loss": 2.9141178131103516,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12564562261104584,
      "rewards/margins": 0.6496793031692505,
      "rewards/rejected": -0.775324821472168,
      "step": 1231
    },
    {
      "epoch": 0.766407465007776,
      "grad_norm": 0.596670389175415,
      "learning_rate": 1.9200000000000003e-05,
      "log_odds_chosen": 6.75174617767334,
      "log_odds_ratio": -0.20460355281829834,
      "logits/chosen": 0.15210659801959991,
      "logits/rejected": 0.29200881719589233,
      "logps/chosen": -0.8481846451759338,
      "logps/rejected": -7.019330978393555,
      "loss": 3.118,
      "nll_loss": 3.0975799560546875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08481846749782562,
      "rewards/margins": 0.6171146631240845,
      "rewards/rejected": -0.7019331455230713,
      "step": 1232
    },
    {
      "epoch": 0.7670295489891136,
      "grad_norm": 1.2167954444885254,
      "learning_rate": 1.9175000000000002e-05,
      "log_odds_chosen": 7.575688362121582,
      "log_odds_ratio": -0.04037388786673546,
      "logits/chosen": 0.16587582230567932,
      "logits/rejected": 0.7371760606765747,
      "logps/chosen": -1.3697576522827148,
      "logps/rejected": -8.56037712097168,
      "loss": 3.0045,
      "nll_loss": 3.000467300415039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13697576522827148,
      "rewards/margins": 0.7190619111061096,
      "rewards/rejected": -0.8560376167297363,
      "step": 1233
    },
    {
      "epoch": 0.767651632970451,
      "grad_norm": 0.5440263152122498,
      "learning_rate": 1.915e-05,
      "log_odds_chosen": 5.056829452514648,
      "log_odds_ratio": -0.18205265700817108,
      "logits/chosen": 0.08534614741802216,
      "logits/rejected": 0.38158437609672546,
      "logps/chosen": -1.172467827796936,
      "logps/rejected": -5.902404308319092,
      "loss": 2.4827,
      "nll_loss": 2.4645302295684814,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11724679172039032,
      "rewards/margins": 0.4729937016963959,
      "rewards/rejected": -0.590240478515625,
      "step": 1234
    },
    {
      "epoch": 0.7682737169517885,
      "grad_norm": 0.47702041268348694,
      "learning_rate": 1.9125e-05,
      "log_odds_chosen": 6.737882614135742,
      "log_odds_ratio": -0.1277119666337967,
      "logits/chosen": 0.1412731111049652,
      "logits/rejected": 0.39362943172454834,
      "logps/chosen": -1.181764841079712,
      "logps/rejected": -7.580643653869629,
      "loss": 3.2685,
      "nll_loss": 3.255748748779297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11817647516727448,
      "rewards/margins": 0.6398878693580627,
      "rewards/rejected": -0.7580643892288208,
      "step": 1235
    },
    {
      "epoch": 0.7688958009331259,
      "grad_norm": 0.5562854409217834,
      "learning_rate": 1.91e-05,
      "log_odds_chosen": 5.550592422485352,
      "log_odds_ratio": -0.05007840692996979,
      "logits/chosen": 0.15344038605690002,
      "logits/rejected": 0.5068711638450623,
      "logps/chosen": -1.3114280700683594,
      "logps/rejected": -6.509056091308594,
      "loss": 2.9737,
      "nll_loss": 2.9687082767486572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13114279508590698,
      "rewards/margins": 0.5197628140449524,
      "rewards/rejected": -0.6509056687355042,
      "step": 1236
    },
    {
      "epoch": 0.7695178849144635,
      "grad_norm": 0.5004754662513733,
      "learning_rate": 1.9075000000000003e-05,
      "log_odds_chosen": 9.40194034576416,
      "log_odds_ratio": -0.15184959769248962,
      "logits/chosen": 0.2978594899177551,
      "logits/rejected": 0.9191336631774902,
      "logps/chosen": -1.4295010566711426,
      "logps/rejected": -10.342602729797363,
      "loss": 3.2896,
      "nll_loss": 3.274416923522949,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1429501175880432,
      "rewards/margins": 0.8913102149963379,
      "rewards/rejected": -1.0342603921890259,
      "step": 1237
    },
    {
      "epoch": 0.7701399688958009,
      "grad_norm": 0.46838802099227905,
      "learning_rate": 1.9050000000000002e-05,
      "log_odds_chosen": 7.5221266746521,
      "log_odds_ratio": -0.21935422718524933,
      "logits/chosen": 0.22468611598014832,
      "logits/rejected": 0.4334770441055298,
      "logps/chosen": -0.879149854183197,
      "logps/rejected": -7.8995866775512695,
      "loss": 3.4049,
      "nll_loss": 3.3830127716064453,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08791498839855194,
      "rewards/margins": 0.7020437121391296,
      "rewards/rejected": -0.7899587154388428,
      "step": 1238
    },
    {
      "epoch": 0.7707620528771384,
      "grad_norm": 0.517456591129303,
      "learning_rate": 1.9025e-05,
      "log_odds_chosen": 6.344799995422363,
      "log_odds_ratio": -0.27640923857688904,
      "logits/chosen": 0.12453234195709229,
      "logits/rejected": 0.4667607545852661,
      "logps/chosen": -1.0894925594329834,
      "logps/rejected": -7.145116329193115,
      "loss": 2.604,
      "nll_loss": 2.5763416290283203,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10894925892353058,
      "rewards/margins": 0.6055623888969421,
      "rewards/rejected": -0.7145116925239563,
      "step": 1239
    },
    {
      "epoch": 0.7713841368584758,
      "grad_norm": 0.6335269808769226,
      "learning_rate": 1.9e-05,
      "log_odds_chosen": 5.562659740447998,
      "log_odds_ratio": -0.2744099497795105,
      "logits/chosen": 0.23254889249801636,
      "logits/rejected": 0.3621961772441864,
      "logps/chosen": -0.8226711750030518,
      "logps/rejected": -5.803696155548096,
      "loss": 3.2831,
      "nll_loss": 3.2556722164154053,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08226712048053741,
      "rewards/margins": 0.4981025159358978,
      "rewards/rejected": -0.5803696513175964,
      "step": 1240
    },
    {
      "epoch": 0.7720062208398134,
      "grad_norm": 0.6518879532814026,
      "learning_rate": 1.8975e-05,
      "log_odds_chosen": 7.919089317321777,
      "log_odds_ratio": -0.3184622526168823,
      "logits/chosen": 0.043444469571113586,
      "logits/rejected": 0.4088301658630371,
      "logps/chosen": -1.1469988822937012,
      "logps/rejected": -8.743982315063477,
      "loss": 2.9404,
      "nll_loss": 2.9085514545440674,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11469988524913788,
      "rewards/margins": 0.7596983909606934,
      "rewards/rejected": -0.8743982315063477,
      "step": 1241
    },
    {
      "epoch": 0.7726283048211509,
      "grad_norm": 0.5594248175621033,
      "learning_rate": 1.895e-05,
      "log_odds_chosen": 3.817509889602661,
      "log_odds_ratio": -0.41913437843322754,
      "logits/chosen": 0.22413289546966553,
      "logits/rejected": 0.42118874192237854,
      "logps/chosen": -1.2284491062164307,
      "logps/rejected": -4.893716812133789,
      "loss": 3.0401,
      "nll_loss": 2.9981565475463867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12284490466117859,
      "rewards/margins": 0.3665267825126648,
      "rewards/rejected": -0.489371657371521,
      "step": 1242
    },
    {
      "epoch": 0.7732503888024883,
      "grad_norm": 0.508392870426178,
      "learning_rate": 1.8925000000000003e-05,
      "log_odds_chosen": 10.411176681518555,
      "log_odds_ratio": -0.10630620270967484,
      "logits/chosen": 0.24550384283065796,
      "logits/rejected": 1.1542640924453735,
      "logps/chosen": -1.27555251121521,
      "logps/rejected": -11.42074203491211,
      "loss": 3.0676,
      "nll_loss": 3.0569348335266113,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.127555251121521,
      "rewards/margins": 1.0145189762115479,
      "rewards/rejected": -1.1420742273330688,
      "step": 1243
    },
    {
      "epoch": 0.7738724727838259,
      "grad_norm": 0.4941979944705963,
      "learning_rate": 1.8900000000000002e-05,
      "log_odds_chosen": 9.65025806427002,
      "log_odds_ratio": -0.18365329504013062,
      "logits/chosen": 0.12889467179775238,
      "logits/rejected": 0.5754642486572266,
      "logps/chosen": -1.2588895559310913,
      "logps/rejected": -10.60193920135498,
      "loss": 3.0992,
      "nll_loss": 3.080859422683716,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12588895857334137,
      "rewards/margins": 0.9343051314353943,
      "rewards/rejected": -1.0601940155029297,
      "step": 1244
    },
    {
      "epoch": 0.7744945567651633,
      "grad_norm": 0.4508911073207855,
      "learning_rate": 1.8875e-05,
      "log_odds_chosen": 9.714991569519043,
      "log_odds_ratio": -0.18170523643493652,
      "logits/chosen": 0.19511641561985016,
      "logits/rejected": 1.2209863662719727,
      "logps/chosen": -1.2065391540527344,
      "logps/rejected": -10.629037857055664,
      "loss": 2.6773,
      "nll_loss": 2.659090042114258,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1206539198756218,
      "rewards/margins": 0.9422498941421509,
      "rewards/rejected": -1.0629037618637085,
      "step": 1245
    },
    {
      "epoch": 0.7751166407465008,
      "grad_norm": 0.8513448238372803,
      "learning_rate": 1.885e-05,
      "log_odds_chosen": 13.84284782409668,
      "log_odds_ratio": -0.002261546440422535,
      "logits/chosen": 0.2347557246685028,
      "logits/rejected": 0.9577349424362183,
      "logps/chosen": -1.4056241512298584,
      "logps/rejected": -14.550141334533691,
      "loss": 3.0029,
      "nll_loss": 3.002654552459717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14056241512298584,
      "rewards/margins": 1.3144516944885254,
      "rewards/rejected": -1.4550142288208008,
      "step": 1246
    },
    {
      "epoch": 0.7757387247278382,
      "grad_norm": 0.7846004962921143,
      "learning_rate": 1.8825e-05,
      "log_odds_chosen": 6.237817764282227,
      "log_odds_ratio": -0.4033685028553009,
      "logits/chosen": 0.2902659773826599,
      "logits/rejected": 0.553649365901947,
      "logps/chosen": -1.4968538284301758,
      "logps/rejected": -7.529121398925781,
      "loss": 3.3566,
      "nll_loss": 3.3162434101104736,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14968539774417877,
      "rewards/margins": 0.6032267808914185,
      "rewards/rejected": -0.7529121041297913,
      "step": 1247
    },
    {
      "epoch": 0.7763608087091758,
      "grad_norm": 0.4592325985431671,
      "learning_rate": 1.88e-05,
      "log_odds_chosen": 7.852267265319824,
      "log_odds_ratio": -0.09105556458234787,
      "logits/chosen": 0.3451773524284363,
      "logits/rejected": 0.6059900522232056,
      "logps/chosen": -0.6744275093078613,
      "logps/rejected": -7.7031354904174805,
      "loss": 3.3566,
      "nll_loss": 3.3474724292755127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06744275242090225,
      "rewards/margins": 0.7028707265853882,
      "rewards/rejected": -0.7703135013580322,
      "step": 1248
    },
    {
      "epoch": 0.7769828926905132,
      "grad_norm": 0.5257586240768433,
      "learning_rate": 1.8775000000000002e-05,
      "log_odds_chosen": 9.442645072937012,
      "log_odds_ratio": -0.26668232679367065,
      "logits/chosen": 0.2947607636451721,
      "logits/rejected": 0.8265994787216187,
      "logps/chosen": -1.1143666505813599,
      "logps/rejected": -10.230154991149902,
      "loss": 3.19,
      "nll_loss": 3.163308620452881,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11143667250871658,
      "rewards/margins": 0.911578893661499,
      "rewards/rejected": -1.0230154991149902,
      "step": 1249
    },
    {
      "epoch": 0.7776049766718507,
      "grad_norm": 0.6123740077018738,
      "learning_rate": 1.8750000000000002e-05,
      "log_odds_chosen": 11.2650146484375,
      "log_odds_ratio": -0.08563439548015594,
      "logits/chosen": 0.25241217017173767,
      "logits/rejected": 1.0535790920257568,
      "logps/chosen": -0.7609401345252991,
      "logps/rejected": -11.140617370605469,
      "loss": 2.7892,
      "nll_loss": 2.780632972717285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07609401643276215,
      "rewards/margins": 1.0379679203033447,
      "rewards/rejected": -1.1140618324279785,
      "step": 1250
    },
    {
      "epoch": 0.7782270606531881,
      "grad_norm": 0.43171098828315735,
      "learning_rate": 1.8725e-05,
      "log_odds_chosen": 8.13177490234375,
      "log_odds_ratio": -0.1548667997121811,
      "logits/chosen": 0.23373821377754211,
      "logits/rejected": 0.5560605525970459,
      "logps/chosen": -0.9117122888565063,
      "logps/rejected": -8.333196640014648,
      "loss": 3.3968,
      "nll_loss": 3.3813085556030273,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09117123484611511,
      "rewards/margins": 0.7421483397483826,
      "rewards/rejected": -0.8333196640014648,
      "step": 1251
    },
    {
      "epoch": 0.7788491446345257,
      "grad_norm": 0.5646179914474487,
      "learning_rate": 1.87e-05,
      "log_odds_chosen": 9.996572494506836,
      "log_odds_ratio": -0.07235460728406906,
      "logits/chosen": 0.1836574822664261,
      "logits/rejected": 0.9524223208427429,
      "logps/chosen": -1.001600980758667,
      "logps/rejected": -10.505570411682129,
      "loss": 2.9528,
      "nll_loss": 2.945585250854492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10016010701656342,
      "rewards/margins": 0.9503970146179199,
      "rewards/rejected": -1.0505571365356445,
      "step": 1252
    },
    {
      "epoch": 0.7794712286158632,
      "grad_norm": 0.5360233783721924,
      "learning_rate": 1.8675e-05,
      "log_odds_chosen": 8.815251350402832,
      "log_odds_ratio": -0.2755727171897888,
      "logits/chosen": 0.17109717428684235,
      "logits/rejected": 0.4950663447380066,
      "logps/chosen": -0.9574038982391357,
      "logps/rejected": -9.33325481414795,
      "loss": 2.9651,
      "nll_loss": 2.9375319480895996,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09574039280414581,
      "rewards/margins": 0.8375850915908813,
      "rewards/rejected": -0.9333254098892212,
      "step": 1253
    },
    {
      "epoch": 0.7800933125972006,
      "grad_norm": 0.5576890110969543,
      "learning_rate": 1.865e-05,
      "log_odds_chosen": 12.056558609008789,
      "log_odds_ratio": -0.0050803543999791145,
      "logits/chosen": 0.157872274518013,
      "logits/rejected": 0.9030807614326477,
      "logps/chosen": -0.9107524156570435,
      "logps/rejected": -12.303834915161133,
      "loss": 2.8042,
      "nll_loss": 2.8037121295928955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09107524901628494,
      "rewards/margins": 1.1393083333969116,
      "rewards/rejected": -1.2303835153579712,
      "step": 1254
    },
    {
      "epoch": 0.7807153965785381,
      "grad_norm": 0.4668339788913727,
      "learning_rate": 1.8625000000000002e-05,
      "log_odds_chosen": 7.626652717590332,
      "log_odds_ratio": -0.1740056425333023,
      "logits/chosen": 0.19568422436714172,
      "logits/rejected": 0.7308311462402344,
      "logps/chosen": -1.2841660976409912,
      "logps/rejected": -8.55526351928711,
      "loss": 3.0909,
      "nll_loss": 3.0734806060791016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12841661274433136,
      "rewards/margins": 0.7271097898483276,
      "rewards/rejected": -0.8555263876914978,
      "step": 1255
    },
    {
      "epoch": 0.7813374805598756,
      "grad_norm": 0.7606042623519897,
      "learning_rate": 1.86e-05,
      "log_odds_chosen": 11.425827026367188,
      "log_odds_ratio": -0.06626426428556442,
      "logits/chosen": 0.22620922327041626,
      "logits/rejected": 1.0646039247512817,
      "logps/chosen": -1.135450005531311,
      "logps/rejected": -11.916156768798828,
      "loss": 3.2562,
      "nll_loss": 3.249558448791504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1135450154542923,
      "rewards/margins": 1.0780707597732544,
      "rewards/rejected": -1.1916157007217407,
      "step": 1256
    },
    {
      "epoch": 0.7819595645412131,
      "grad_norm": 0.6749795079231262,
      "learning_rate": 1.8575e-05,
      "log_odds_chosen": 10.228057861328125,
      "log_odds_ratio": -0.09839068353176117,
      "logits/chosen": 0.22929708659648895,
      "logits/rejected": 0.9187569618225098,
      "logps/chosen": -1.2574495077133179,
      "logps/rejected": -11.117281913757324,
      "loss": 2.9522,
      "nll_loss": 2.942385196685791,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12574495375156403,
      "rewards/margins": 0.9859832525253296,
      "rewards/rejected": -1.1117281913757324,
      "step": 1257
    },
    {
      "epoch": 0.7825816485225505,
      "grad_norm": 0.4381241500377655,
      "learning_rate": 1.855e-05,
      "log_odds_chosen": 15.236520767211914,
      "log_odds_ratio": -0.2253989428281784,
      "logits/chosen": 0.24419057369232178,
      "logits/rejected": 1.12411367893219,
      "logps/chosen": -0.9931966066360474,
      "logps/rejected": -15.840814590454102,
      "loss": 3.4943,
      "nll_loss": 3.4717283248901367,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09931966662406921,
      "rewards/margins": 1.4847617149353027,
      "rewards/rejected": -1.5840815305709839,
      "step": 1258
    },
    {
      "epoch": 0.783203732503888,
      "grad_norm": 0.5113958120346069,
      "learning_rate": 1.8525e-05,
      "log_odds_chosen": 15.052928924560547,
      "log_odds_ratio": -0.012858058325946331,
      "logits/chosen": 0.1784391850233078,
      "logits/rejected": 1.2252875566482544,
      "logps/chosen": -1.29570734500885,
      "logps/rejected": -16.01512336730957,
      "loss": 2.7647,
      "nll_loss": 2.7634403705596924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12957073748111725,
      "rewards/margins": 1.4719417095184326,
      "rewards/rejected": -1.6015124320983887,
      "step": 1259
    },
    {
      "epoch": 0.7838258164852255,
      "grad_norm": 0.6102534532546997,
      "learning_rate": 1.85e-05,
      "log_odds_chosen": 17.90291976928711,
      "log_odds_ratio": -3.650830876722466e-06,
      "logits/chosen": 0.19943833351135254,
      "logits/rejected": 1.5740082263946533,
      "logps/chosen": -1.2210125923156738,
      "logps/rejected": -18.73495101928711,
      "loss": 2.8752,
      "nll_loss": 2.875162363052368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12210125476121902,
      "rewards/margins": 1.7513937950134277,
      "rewards/rejected": -1.8734949827194214,
      "step": 1260
    },
    {
      "epoch": 0.784447900466563,
      "grad_norm": 1.0051997900009155,
      "learning_rate": 1.8475000000000002e-05,
      "log_odds_chosen": 13.351705551147461,
      "log_odds_ratio": -0.005376896820962429,
      "logits/chosen": 0.1484776884317398,
      "logits/rejected": 0.7685915231704712,
      "logps/chosen": -1.3591516017913818,
      "logps/rejected": -14.330547332763672,
      "loss": 2.9603,
      "nll_loss": 2.959794044494629,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13591517508029938,
      "rewards/margins": 1.2971396446228027,
      "rewards/rejected": -1.433054804801941,
      "step": 1261
    },
    {
      "epoch": 0.7850699844479004,
      "grad_norm": 0.5121684670448303,
      "learning_rate": 1.845e-05,
      "log_odds_chosen": 15.99807357788086,
      "log_odds_ratio": -0.12559840083122253,
      "logits/chosen": 0.23574836552143097,
      "logits/rejected": 0.7866611480712891,
      "logps/chosen": -0.8544904589653015,
      "logps/rejected": -16.086206436157227,
      "loss": 2.9929,
      "nll_loss": 2.98030424118042,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08544904738664627,
      "rewards/margins": 1.5231716632843018,
      "rewards/rejected": -1.6086206436157227,
      "step": 1262
    },
    {
      "epoch": 0.785692068429238,
      "grad_norm": 0.6973137855529785,
      "learning_rate": 1.8425e-05,
      "log_odds_chosen": 17.51459503173828,
      "log_odds_ratio": -0.104143887758255,
      "logits/chosen": 0.3150438666343689,
      "logits/rejected": 1.629845142364502,
      "logps/chosen": -1.1097285747528076,
      "logps/rejected": -18.256704330444336,
      "loss": 2.4323,
      "nll_loss": 2.4219167232513428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11097285151481628,
      "rewards/margins": 1.7146975994110107,
      "rewards/rejected": -1.8256704807281494,
      "step": 1263
    },
    {
      "epoch": 0.7863141524105755,
      "grad_norm": 3.6229288578033447,
      "learning_rate": 1.84e-05,
      "log_odds_chosen": 15.211271286010742,
      "log_odds_ratio": -0.15237554907798767,
      "logits/chosen": 0.19041389226913452,
      "logits/rejected": 1.0611919164657593,
      "logps/chosen": -1.5682026147842407,
      "logps/rejected": -16.546676635742188,
      "loss": 2.726,
      "nll_loss": 2.710737943649292,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15682026743888855,
      "rewards/margins": 1.497847557067871,
      "rewards/rejected": -1.654667615890503,
      "step": 1264
    },
    {
      "epoch": 0.7869362363919129,
      "grad_norm": 0.43393415212631226,
      "learning_rate": 1.8375e-05,
      "log_odds_chosen": 15.434120178222656,
      "log_odds_ratio": -0.004888943396508694,
      "logits/chosen": 0.360336035490036,
      "logits/rejected": 1.6232982873916626,
      "logps/chosen": -0.9606151580810547,
      "logps/rejected": -15.767781257629395,
      "loss": 3.4923,
      "nll_loss": 3.4917798042297363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09606151282787323,
      "rewards/margins": 1.4807167053222656,
      "rewards/rejected": -1.5767782926559448,
      "step": 1265
    },
    {
      "epoch": 0.7875583203732504,
      "grad_norm": 0.41161367297172546,
      "learning_rate": 1.8350000000000002e-05,
      "log_odds_chosen": 10.203583717346191,
      "log_odds_ratio": -0.26046648621559143,
      "logits/chosen": 0.22639605402946472,
      "logits/rejected": 0.8604259490966797,
      "logps/chosen": -1.0054662227630615,
      "logps/rejected": -10.743423461914062,
      "loss": 3.1646,
      "nll_loss": 3.138522148132324,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10054662823677063,
      "rewards/margins": 0.9737956523895264,
      "rewards/rejected": -1.0743423700332642,
      "step": 1266
    },
    {
      "epoch": 0.7881804043545879,
      "grad_norm": 5.74237585067749,
      "learning_rate": 1.8325e-05,
      "log_odds_chosen": 7.807779312133789,
      "log_odds_ratio": -0.18478822708129883,
      "logits/chosen": 0.19614526629447937,
      "logits/rejected": 0.776382565498352,
      "logps/chosen": -1.2011544704437256,
      "logps/rejected": -8.654647827148438,
      "loss": 3.0134,
      "nll_loss": 2.994901418685913,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12011544406414032,
      "rewards/margins": 0.7453492879867554,
      "rewards/rejected": -0.8654647469520569,
      "step": 1267
    },
    {
      "epoch": 0.7888024883359254,
      "grad_norm": 0.4839012026786804,
      "learning_rate": 1.83e-05,
      "log_odds_chosen": 17.199420928955078,
      "log_odds_ratio": -0.14982233941555023,
      "logits/chosen": 0.3670274615287781,
      "logits/rejected": 1.3338403701782227,
      "logps/chosen": -1.1166799068450928,
      "logps/rejected": -17.874608993530273,
      "loss": 3.4317,
      "nll_loss": 3.416668176651001,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11166798323392868,
      "rewards/margins": 1.6757930517196655,
      "rewards/rejected": -1.7874610424041748,
      "step": 1268
    },
    {
      "epoch": 0.7894245723172628,
      "grad_norm": 1.5299826860427856,
      "learning_rate": 1.8275e-05,
      "log_odds_chosen": 11.565797805786133,
      "log_odds_ratio": -0.22419725358486176,
      "logits/chosen": 0.14468568563461304,
      "logits/rejected": 0.8828893303871155,
      "logps/chosen": -1.188100814819336,
      "logps/rejected": -12.22629451751709,
      "loss": 2.772,
      "nll_loss": 2.749605655670166,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11881007999181747,
      "rewards/margins": 1.1038193702697754,
      "rewards/rejected": -1.222629427909851,
      "step": 1269
    },
    {
      "epoch": 0.7900466562986003,
      "grad_norm": 0.3719753324985504,
      "learning_rate": 1.825e-05,
      "log_odds_chosen": 12.311816215515137,
      "log_odds_ratio": -0.007772353012114763,
      "logits/chosen": 0.21790693700313568,
      "logits/rejected": 1.1610301733016968,
      "logps/chosen": -1.1040740013122559,
      "logps/rejected": -12.842395782470703,
      "loss": 3.4675,
      "nll_loss": 3.4667301177978516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11040741205215454,
      "rewards/margins": 1.1738322973251343,
      "rewards/rejected": -1.284239649772644,
      "step": 1270
    },
    {
      "epoch": 0.7906687402799378,
      "grad_norm": 0.575977087020874,
      "learning_rate": 1.8225e-05,
      "log_odds_chosen": 10.100236892700195,
      "log_odds_ratio": -0.13360543549060822,
      "logits/chosen": 0.1812443733215332,
      "logits/rejected": 0.5013841390609741,
      "logps/chosen": -0.996255099773407,
      "logps/rejected": -10.562983512878418,
      "loss": 3.1107,
      "nll_loss": 3.097329616546631,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09962552040815353,
      "rewards/margins": 0.9566728472709656,
      "rewards/rejected": -1.0562983751296997,
      "step": 1271
    },
    {
      "epoch": 0.7912908242612753,
      "grad_norm": 0.547460675239563,
      "learning_rate": 1.8200000000000002e-05,
      "log_odds_chosen": 7.538238048553467,
      "log_odds_ratio": -0.10196790844202042,
      "logits/chosen": -0.03788933530449867,
      "logits/rejected": 0.4808521270751953,
      "logps/chosen": -1.1122915744781494,
      "logps/rejected": -8.138171195983887,
      "loss": 2.3522,
      "nll_loss": 2.341975688934326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11122915148735046,
      "rewards/margins": 0.702587902545929,
      "rewards/rejected": -0.8138170838356018,
      "step": 1272
    },
    {
      "epoch": 0.7919129082426127,
      "grad_norm": 0.5085967183113098,
      "learning_rate": 1.8175e-05,
      "log_odds_chosen": 10.72020149230957,
      "log_odds_ratio": -0.12706585228443146,
      "logits/chosen": -0.05211986228823662,
      "logits/rejected": 0.890866756439209,
      "logps/chosen": -0.9368724822998047,
      "logps/rejected": -11.051326751708984,
      "loss": 2.2072,
      "nll_loss": 2.19453763961792,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09368725121021271,
      "rewards/margins": 1.01144540309906,
      "rewards/rejected": -1.1051325798034668,
      "step": 1273
    },
    {
      "epoch": 0.7925349922239502,
      "grad_norm": 0.5295844674110413,
      "learning_rate": 1.815e-05,
      "log_odds_chosen": 6.329776763916016,
      "log_odds_ratio": -0.1255560964345932,
      "logits/chosen": 0.12830644845962524,
      "logits/rejected": 0.505806565284729,
      "logps/chosen": -1.2834556102752686,
      "logps/rejected": -7.303755283355713,
      "loss": 3.2161,
      "nll_loss": 3.2035398483276367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1283455491065979,
      "rewards/margins": 0.6020300388336182,
      "rewards/rejected": -0.7303755879402161,
      "step": 1274
    },
    {
      "epoch": 0.7931570762052877,
      "grad_norm": 0.45801711082458496,
      "learning_rate": 1.8125e-05,
      "log_odds_chosen": 7.928872108459473,
      "log_odds_ratio": -0.16928905248641968,
      "logits/chosen": 0.1209368109703064,
      "logits/rejected": 0.31357628107070923,
      "logps/chosen": -0.9660826325416565,
      "logps/rejected": -8.398103713989258,
      "loss": 3.2755,
      "nll_loss": 3.2585678100585938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09660826623439789,
      "rewards/margins": 0.7432020306587219,
      "rewards/rejected": -0.839810311794281,
      "step": 1275
    },
    {
      "epoch": 0.7937791601866252,
      "grad_norm": 0.4788154363632202,
      "learning_rate": 1.81e-05,
      "log_odds_chosen": 14.399748802185059,
      "log_odds_ratio": -0.00020558810501825064,
      "logits/chosen": 0.13003887236118317,
      "logits/rejected": 1.1363701820373535,
      "logps/chosen": -0.9204753041267395,
      "logps/rejected": -14.68822956085205,
      "loss": 2.664,
      "nll_loss": 2.663971185684204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09204752743244171,
      "rewards/margins": 1.3767753839492798,
      "rewards/rejected": -1.468822956085205,
      "step": 1276
    },
    {
      "epoch": 0.7944012441679627,
      "grad_norm": 0.4245346486568451,
      "learning_rate": 1.8075e-05,
      "log_odds_chosen": 9.295736312866211,
      "log_odds_ratio": -0.05224517732858658,
      "logits/chosen": 0.0967981144785881,
      "logits/rejected": 1.0028589963912964,
      "logps/chosen": -1.1117591857910156,
      "logps/rejected": -9.941202163696289,
      "loss": 2.4129,
      "nll_loss": 2.4077231884002686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11117591708898544,
      "rewards/margins": 0.8829442262649536,
      "rewards/rejected": -0.9941201210021973,
      "step": 1277
    },
    {
      "epoch": 0.7950233281493001,
      "grad_norm": 0.46397557854652405,
      "learning_rate": 1.805e-05,
      "log_odds_chosen": 8.459473609924316,
      "log_odds_ratio": -0.0300842747092247,
      "logits/chosen": 0.11754687130451202,
      "logits/rejected": 0.4644424617290497,
      "logps/chosen": -0.7050676941871643,
      "logps/rejected": -8.29973030090332,
      "loss": 2.7823,
      "nll_loss": 2.779327392578125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07050677388906479,
      "rewards/margins": 0.7594663500785828,
      "rewards/rejected": -0.8299731016159058,
      "step": 1278
    },
    {
      "epoch": 0.7956454121306377,
      "grad_norm": 0.4239655137062073,
      "learning_rate": 1.8025e-05,
      "log_odds_chosen": 9.256122589111328,
      "log_odds_ratio": -0.001191396964713931,
      "logits/chosen": 0.24952790141105652,
      "logits/rejected": 0.8297731280326843,
      "logps/chosen": -0.9855543971061707,
      "logps/rejected": -9.722004890441895,
      "loss": 3.376,
      "nll_loss": 3.3759002685546875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09855544567108154,
      "rewards/margins": 0.8736451268196106,
      "rewards/rejected": -0.9722005724906921,
      "step": 1279
    },
    {
      "epoch": 0.7962674961119751,
      "grad_norm": 0.5233640670776367,
      "learning_rate": 1.8e-05,
      "log_odds_chosen": 4.858011245727539,
      "log_odds_ratio": -0.24006202816963196,
      "logits/chosen": 0.10965058207511902,
      "logits/rejected": 0.2803415358066559,
      "logps/chosen": -0.7928171157836914,
      "logps/rejected": -5.210036277770996,
      "loss": 2.898,
      "nll_loss": 2.8740434646606445,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07928171753883362,
      "rewards/margins": 0.44172191619873047,
      "rewards/rejected": -0.5210036039352417,
      "step": 1280
    },
    {
      "epoch": 0.7968895800933126,
      "grad_norm": 0.49103713035583496,
      "learning_rate": 1.7975e-05,
      "log_odds_chosen": 8.95180892944336,
      "log_odds_ratio": -0.08506196737289429,
      "logits/chosen": 0.14098000526428223,
      "logits/rejected": 0.6167709231376648,
      "logps/chosen": -0.9210264086723328,
      "logps/rejected": -9.39538860321045,
      "loss": 2.9812,
      "nll_loss": 2.9726974964141846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09210263937711716,
      "rewards/margins": 0.8474361896514893,
      "rewards/rejected": -0.9395389556884766,
      "step": 1281
    },
    {
      "epoch": 0.7975116640746501,
      "grad_norm": 0.3628371059894562,
      "learning_rate": 1.795e-05,
      "log_odds_chosen": 12.051719665527344,
      "log_odds_ratio": -0.0018485789187252522,
      "logits/chosen": 0.18384167551994324,
      "logits/rejected": 0.9371912479400635,
      "logps/chosen": -1.0759706497192383,
      "logps/rejected": -12.439555168151855,
      "loss": 3.1226,
      "nll_loss": 3.1224188804626465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10759708285331726,
      "rewards/margins": 1.136358380317688,
      "rewards/rejected": -1.2439554929733276,
      "step": 1282
    },
    {
      "epoch": 0.7981337480559876,
      "grad_norm": 0.5358335971832275,
      "learning_rate": 1.7925e-05,
      "log_odds_chosen": 8.796299934387207,
      "log_odds_ratio": -0.15110205113887787,
      "logits/chosen": 0.31277087330818176,
      "logits/rejected": 1.0025912523269653,
      "logps/chosen": -0.9938040971755981,
      "logps/rejected": -9.312862396240234,
      "loss": 2.9431,
      "nll_loss": 2.9279847145080566,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09938041865825653,
      "rewards/margins": 0.8319058418273926,
      "rewards/rejected": -0.9312862157821655,
      "step": 1283
    },
    {
      "epoch": 0.798755832037325,
      "grad_norm": 0.3374326825141907,
      "learning_rate": 1.79e-05,
      "log_odds_chosen": 9.138998031616211,
      "log_odds_ratio": -0.11299619823694229,
      "logits/chosen": 0.2334498018026352,
      "logits/rejected": 0.8283179998397827,
      "logps/chosen": -1.2180832624435425,
      "logps/rejected": -10.032180786132812,
      "loss": 3.4034,
      "nll_loss": 3.3921141624450684,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12180833518505096,
      "rewards/margins": 0.881409764289856,
      "rewards/rejected": -1.003218173980713,
      "step": 1284
    },
    {
      "epoch": 0.7993779160186625,
      "grad_norm": 0.6933339238166809,
      "learning_rate": 1.7875e-05,
      "log_odds_chosen": 13.547820091247559,
      "log_odds_ratio": -0.1648188680410385,
      "logits/chosen": 0.10697955638170242,
      "logits/rejected": 0.722811758518219,
      "logps/chosen": -1.0336053371429443,
      "logps/rejected": -14.206631660461426,
      "loss": 2.5359,
      "nll_loss": 2.5193803310394287,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10336053371429443,
      "rewards/margins": 1.3173027038574219,
      "rewards/rejected": -1.4206632375717163,
      "step": 1285
    },
    {
      "epoch": 0.8,
      "grad_norm": 0.6732850074768066,
      "learning_rate": 1.785e-05,
      "log_odds_chosen": 7.440114974975586,
      "log_odds_ratio": -0.18558666110038757,
      "logits/chosen": 0.32390734553337097,
      "logits/rejected": 0.7920575141906738,
      "logps/chosen": -1.029671549797058,
      "logps/rejected": -7.974099159240723,
      "loss": 3.3675,
      "nll_loss": 3.3489205837249756,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10296716541051865,
      "rewards/margins": 0.6944427490234375,
      "rewards/rejected": -0.7974098920822144,
      "step": 1286
    },
    {
      "epoch": 0.8006220839813375,
      "grad_norm": 0.776902973651886,
      "learning_rate": 1.7825e-05,
      "log_odds_chosen": 12.03702449798584,
      "log_odds_ratio": -0.09163986146450043,
      "logits/chosen": 0.3653428256511688,
      "logits/rejected": 1.1969035863876343,
      "logps/chosen": -1.0206536054611206,
      "logps/rejected": -12.635235786437988,
      "loss": 3.414,
      "nll_loss": 3.4048497676849365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10206536203622818,
      "rewards/margins": 1.1614582538604736,
      "rewards/rejected": -1.2635235786437988,
      "step": 1287
    },
    {
      "epoch": 0.801244167962675,
      "grad_norm": 0.5694941282272339,
      "learning_rate": 1.78e-05,
      "log_odds_chosen": 13.658385276794434,
      "log_odds_ratio": -0.10558748990297318,
      "logits/chosen": 0.21393881738185883,
      "logits/rejected": 0.8981702923774719,
      "logps/chosen": -1.2607628107070923,
      "logps/rejected": -14.39406967163086,
      "loss": 2.6656,
      "nll_loss": 2.6550426483154297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12607628107070923,
      "rewards/margins": 1.3133307695388794,
      "rewards/rejected": -1.4394071102142334,
      "step": 1288
    },
    {
      "epoch": 0.8018662519440124,
      "grad_norm": 0.4627145230770111,
      "learning_rate": 1.7775e-05,
      "log_odds_chosen": 8.23801040649414,
      "log_odds_ratio": -0.3038644790649414,
      "logits/chosen": 0.16624096035957336,
      "logits/rejected": 0.5100173950195312,
      "logps/chosen": -1.0921432971954346,
      "logps/rejected": -8.8154296875,
      "loss": 3.0926,
      "nll_loss": 3.0622284412384033,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10921433568000793,
      "rewards/margins": 0.7723287343978882,
      "rewards/rejected": -0.8815430402755737,
      "step": 1289
    },
    {
      "epoch": 0.80248833592535,
      "grad_norm": 0.39549392461776733,
      "learning_rate": 1.775e-05,
      "log_odds_chosen": 10.66400146484375,
      "log_odds_ratio": -0.07293321192264557,
      "logits/chosen": 0.40711355209350586,
      "logits/rejected": 0.8906251192092896,
      "logps/chosen": -1.3841667175292969,
      "logps/rejected": -11.750348091125488,
      "loss": 4.0269,
      "nll_loss": 4.019584655761719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13841667771339417,
      "rewards/margins": 1.0366181135177612,
      "rewards/rejected": -1.175034761428833,
      "step": 1290
    },
    {
      "epoch": 0.8031104199066874,
      "grad_norm": 0.5006654858589172,
      "learning_rate": 1.7725e-05,
      "log_odds_chosen": 10.659064292907715,
      "log_odds_ratio": -0.013071035034954548,
      "logits/chosen": 0.2032037228345871,
      "logits/rejected": 1.0664845705032349,
      "logps/chosen": -1.228370189666748,
      "logps/rejected": -11.474655151367188,
      "loss": 2.8391,
      "nll_loss": 2.8378067016601562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12283702194690704,
      "rewards/margins": 1.0246284008026123,
      "rewards/rejected": -1.147465467453003,
      "step": 1291
    },
    {
      "epoch": 0.8037325038880249,
      "grad_norm": 0.47223320603370667,
      "learning_rate": 1.77e-05,
      "log_odds_chosen": 13.39532470703125,
      "log_odds_ratio": -0.002649889327585697,
      "logits/chosen": 0.1537506878376007,
      "logits/rejected": 0.9822484254837036,
      "logps/chosen": -1.164747714996338,
      "logps/rejected": -14.148645401000977,
      "loss": 3.4009,
      "nll_loss": 3.400646448135376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11647477746009827,
      "rewards/margins": 1.2983897924423218,
      "rewards/rejected": -1.4148645401000977,
      "step": 1292
    },
    {
      "epoch": 0.8043545878693623,
      "grad_norm": 0.5693787336349487,
      "learning_rate": 1.7675e-05,
      "log_odds_chosen": 12.302108764648438,
      "log_odds_ratio": -0.1323574185371399,
      "logits/chosen": 0.2963750958442688,
      "logits/rejected": 1.2182402610778809,
      "logps/chosen": -1.1627578735351562,
      "logps/rejected": -13.141119956970215,
      "loss": 2.8084,
      "nll_loss": 2.7951581478118896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11627580225467682,
      "rewards/margins": 1.1978362798690796,
      "rewards/rejected": -1.3141121864318848,
      "step": 1293
    },
    {
      "epoch": 0.8049766718506999,
      "grad_norm": 0.444240540266037,
      "learning_rate": 1.765e-05,
      "log_odds_chosen": 12.026140213012695,
      "log_odds_ratio": -0.12117686867713928,
      "logits/chosen": 0.18682369589805603,
      "logits/rejected": 0.8227996230125427,
      "logps/chosen": -0.9900913238525391,
      "logps/rejected": -12.605978012084961,
      "loss": 3.1138,
      "nll_loss": 3.101719856262207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09900914132595062,
      "rewards/margins": 1.1615886688232422,
      "rewards/rejected": -1.2605977058410645,
      "step": 1294
    },
    {
      "epoch": 0.8055987558320373,
      "grad_norm": 0.5121841430664062,
      "learning_rate": 1.7625e-05,
      "log_odds_chosen": 8.10305404663086,
      "log_odds_ratio": -0.12781129777431488,
      "logits/chosen": 0.24159939587116241,
      "logits/rejected": 0.5816590785980225,
      "logps/chosen": -0.9372666478157043,
      "logps/rejected": -8.444114685058594,
      "loss": 3.1081,
      "nll_loss": 3.0953598022460938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09372667223215103,
      "rewards/margins": 0.7506847381591797,
      "rewards/rejected": -0.8444114923477173,
      "step": 1295
    },
    {
      "epoch": 0.8062208398133748,
      "grad_norm": 0.6881003975868225,
      "learning_rate": 1.76e-05,
      "log_odds_chosen": 11.331974983215332,
      "log_odds_ratio": -0.11589650809764862,
      "logits/chosen": 0.15356194972991943,
      "logits/rejected": 0.8771441578865051,
      "logps/chosen": -0.9373379349708557,
      "logps/rejected": -11.583261489868164,
      "loss": 2.9828,
      "nll_loss": 2.9712398052215576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09373379498720169,
      "rewards/margins": 1.0645923614501953,
      "rewards/rejected": -1.1583261489868164,
      "step": 1296
    },
    {
      "epoch": 0.8068429237947123,
      "grad_norm": 0.5609170794487,
      "learning_rate": 1.7575e-05,
      "log_odds_chosen": 14.964519500732422,
      "log_odds_ratio": -0.2053382247686386,
      "logits/chosen": 0.22482171654701233,
      "logits/rejected": 0.9772385358810425,
      "logps/chosen": -0.9464631080627441,
      "logps/rejected": -15.543478965759277,
      "loss": 2.9799,
      "nll_loss": 2.959413528442383,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09464631229639053,
      "rewards/margins": 1.4597017765045166,
      "rewards/rejected": -1.5543479919433594,
      "step": 1297
    },
    {
      "epoch": 0.8074650077760498,
      "grad_norm": 0.5253718495368958,
      "learning_rate": 1.755e-05,
      "log_odds_chosen": 13.896406173706055,
      "log_odds_ratio": -0.15764853358268738,
      "logits/chosen": 0.3127559423446655,
      "logits/rejected": 1.3013142347335815,
      "logps/chosen": -1.011033296585083,
      "logps/rejected": -14.529682159423828,
      "loss": 3.3237,
      "nll_loss": 3.3079025745391846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10110333561897278,
      "rewards/margins": 1.3518648147583008,
      "rewards/rejected": -1.4529681205749512,
      "step": 1298
    },
    {
      "epoch": 0.8080870917573872,
      "grad_norm": 0.4803771376609802,
      "learning_rate": 1.7525e-05,
      "log_odds_chosen": 9.528873443603516,
      "log_odds_ratio": -0.077350914478302,
      "logits/chosen": 0.21042805910110474,
      "logits/rejected": 0.8243046998977661,
      "logps/chosen": -0.8366771936416626,
      "logps/rejected": -9.790960311889648,
      "loss": 2.9032,
      "nll_loss": 2.8954741954803467,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08366772532463074,
      "rewards/margins": 0.8954282999038696,
      "rewards/rejected": -0.9790959358215332,
      "step": 1299
    },
    {
      "epoch": 0.8087091757387247,
      "grad_norm": 5.892223834991455,
      "learning_rate": 1.75e-05,
      "log_odds_chosen": 4.356164932250977,
      "log_odds_ratio": -0.544304370880127,
      "logits/chosen": 0.14717110991477966,
      "logits/rejected": 0.4145839810371399,
      "logps/chosen": -2.111706018447876,
      "logps/rejected": -6.22900915145874,
      "loss": 3.3116,
      "nll_loss": 3.2571868896484375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21117059886455536,
      "rewards/margins": 0.4117303192615509,
      "rewards/rejected": -0.6229009628295898,
      "step": 1300
    },
    {
      "epoch": 0.8093312597200623,
      "grad_norm": 0.5840747952461243,
      "learning_rate": 1.7475e-05,
      "log_odds_chosen": 14.291379928588867,
      "log_odds_ratio": -0.12850309908390045,
      "logits/chosen": 0.1304207295179367,
      "logits/rejected": 0.9051514267921448,
      "logps/chosen": -0.6998139023780823,
      "logps/rejected": -14.083181381225586,
      "loss": 2.7482,
      "nll_loss": 2.7353763580322266,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06998138874769211,
      "rewards/margins": 1.3383368253707886,
      "rewards/rejected": -1.4083181619644165,
      "step": 1301
    },
    {
      "epoch": 0.8099533437013997,
      "grad_norm": 0.679797887802124,
      "learning_rate": 1.745e-05,
      "log_odds_chosen": 10.464731216430664,
      "log_odds_ratio": -0.007538243196904659,
      "logits/chosen": 0.2101728618144989,
      "logits/rejected": 0.9237760901451111,
      "logps/chosen": -1.0783690214157104,
      "logps/rejected": -11.021986961364746,
      "loss": 2.3857,
      "nll_loss": 2.384983539581299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10783690214157104,
      "rewards/margins": 0.9943617582321167,
      "rewards/rejected": -1.1021987199783325,
      "step": 1302
    },
    {
      "epoch": 0.8105754276827372,
      "grad_norm": 0.5377840399742126,
      "learning_rate": 1.7425e-05,
      "log_odds_chosen": 12.532342910766602,
      "log_odds_ratio": -0.044694434851408005,
      "logits/chosen": 0.15078105032444,
      "logits/rejected": 0.8079558610916138,
      "logps/chosen": -0.9499520659446716,
      "logps/rejected": -12.720598220825195,
      "loss": 2.845,
      "nll_loss": 2.840527057647705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09499520808458328,
      "rewards/margins": 1.1770645380020142,
      "rewards/rejected": -1.2720599174499512,
      "step": 1303
    },
    {
      "epoch": 0.8111975116640746,
      "grad_norm": 0.5700002312660217,
      "learning_rate": 1.74e-05,
      "log_odds_chosen": 9.45265007019043,
      "log_odds_ratio": -0.24174627661705017,
      "logits/chosen": 0.28236669301986694,
      "logits/rejected": 0.8490161299705505,
      "logps/chosen": -1.2051925659179688,
      "logps/rejected": -10.107574462890625,
      "loss": 3.3028,
      "nll_loss": 3.278662919998169,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.120519258081913,
      "rewards/margins": 0.8902381658554077,
      "rewards/rejected": -1.0107574462890625,
      "step": 1304
    },
    {
      "epoch": 0.8118195956454122,
      "grad_norm": 0.4791751503944397,
      "learning_rate": 1.7375e-05,
      "log_odds_chosen": 15.476375579833984,
      "log_odds_ratio": -0.00826264824718237,
      "logits/chosen": 0.29491129517555237,
      "logits/rejected": 0.9962074756622314,
      "logps/chosen": -1.0027536153793335,
      "logps/rejected": -15.781121253967285,
      "loss": 3.181,
      "nll_loss": 3.180189609527588,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10027536749839783,
      "rewards/margins": 1.4778368473052979,
      "rewards/rejected": -1.5781121253967285,
      "step": 1305
    },
    {
      "epoch": 0.8124416796267496,
      "grad_norm": 0.5712665915489197,
      "learning_rate": 1.7349999999999998e-05,
      "log_odds_chosen": 9.660453796386719,
      "log_odds_ratio": -0.08943241089582443,
      "logits/chosen": 0.12958277761936188,
      "logits/rejected": 0.5500991940498352,
      "logps/chosen": -1.430905818939209,
      "logps/rejected": -10.829052925109863,
      "loss": 3.432,
      "nll_loss": 3.4230165481567383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14309059083461761,
      "rewards/margins": 0.9398147463798523,
      "rewards/rejected": -1.0829052925109863,
      "step": 1306
    },
    {
      "epoch": 0.8130637636080871,
      "grad_norm": 0.514840841293335,
      "learning_rate": 1.7325e-05,
      "log_odds_chosen": 13.178714752197266,
      "log_odds_ratio": -0.005788641981780529,
      "logits/chosen": 0.336585134267807,
      "logits/rejected": 1.2146720886230469,
      "logps/chosen": -1.1430599689483643,
      "logps/rejected": -13.882668495178223,
      "loss": 3.133,
      "nll_loss": 3.1323795318603516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1143060028553009,
      "rewards/margins": 1.273960828781128,
      "rewards/rejected": -1.3882668018341064,
      "step": 1307
    },
    {
      "epoch": 0.8136858475894245,
      "grad_norm": 0.6594363451004028,
      "learning_rate": 1.73e-05,
      "log_odds_chosen": 10.469732284545898,
      "log_odds_ratio": -0.1577916145324707,
      "logits/chosen": 0.3112362027168274,
      "logits/rejected": 1.0076191425323486,
      "logps/chosen": -1.172290563583374,
      "logps/rejected": -11.286930084228516,
      "loss": 2.9662,
      "nll_loss": 2.9504287242889404,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11722905933856964,
      "rewards/margins": 1.0114638805389404,
      "rewards/rejected": -1.128692865371704,
      "step": 1308
    },
    {
      "epoch": 0.8143079315707621,
      "grad_norm": 0.5135230422019958,
      "learning_rate": 1.7275e-05,
      "log_odds_chosen": 6.890636444091797,
      "log_odds_ratio": -0.19977234303951263,
      "logits/chosen": 0.11931162327528,
      "logits/rejected": 0.6851118803024292,
      "logps/chosen": -1.028935432434082,
      "logps/rejected": -7.384231090545654,
      "loss": 2.6068,
      "nll_loss": 2.586785316467285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10289355367422104,
      "rewards/margins": 0.6355295181274414,
      "rewards/rejected": -0.7384231090545654,
      "step": 1309
    },
    {
      "epoch": 0.8149300155520995,
      "grad_norm": 0.45454153418540955,
      "learning_rate": 1.725e-05,
      "log_odds_chosen": 11.916482925415039,
      "log_odds_ratio": -0.22180667519569397,
      "logits/chosen": 0.1590307205915451,
      "logits/rejected": 0.8525173664093018,
      "logps/chosen": -1.089916706085205,
      "logps/rejected": -12.624712944030762,
      "loss": 2.9198,
      "nll_loss": 2.897609233856201,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10899168252944946,
      "rewards/margins": 1.1534795761108398,
      "rewards/rejected": -1.2624711990356445,
      "step": 1310
    },
    {
      "epoch": 0.815552099533437,
      "grad_norm": 0.467376172542572,
      "learning_rate": 1.7225e-05,
      "log_odds_chosen": 15.533833503723145,
      "log_odds_ratio": -1.8030461887974525e-06,
      "logits/chosen": 0.17015010118484497,
      "logits/rejected": 0.8839508295059204,
      "logps/chosen": -1.1618465185165405,
      "logps/rejected": -16.2960147857666,
      "loss": 3.1071,
      "nll_loss": 3.1071434020996094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11618465930223465,
      "rewards/margins": 1.5134167671203613,
      "rewards/rejected": -1.6296014785766602,
      "step": 1311
    },
    {
      "epoch": 0.8161741835147744,
      "grad_norm": 0.5810384750366211,
      "learning_rate": 1.7199999999999998e-05,
      "log_odds_chosen": 5.953010559082031,
      "log_odds_ratio": -0.20768365263938904,
      "logits/chosen": 0.2892778515815735,
      "logits/rejected": 0.5529869198799133,
      "logps/chosen": -1.1497933864593506,
      "logps/rejected": -6.790042400360107,
      "loss": 3.2464,
      "nll_loss": 3.2256784439086914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1149793416261673,
      "rewards/margins": 0.5640249252319336,
      "rewards/rejected": -0.6790042519569397,
      "step": 1312
    },
    {
      "epoch": 0.816796267496112,
      "grad_norm": 0.46326449513435364,
      "learning_rate": 1.7175e-05,
      "log_odds_chosen": 14.900135040283203,
      "log_odds_ratio": -0.005678822286427021,
      "logits/chosen": 0.25347965955734253,
      "logits/rejected": 0.990155041217804,
      "logps/chosen": -1.4208537340164185,
      "logps/rejected": -15.883021354675293,
      "loss": 3.3431,
      "nll_loss": 3.34252667427063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14208537340164185,
      "rewards/margins": 1.4462168216705322,
      "rewards/rejected": -1.5883021354675293,
      "step": 1313
    },
    {
      "epoch": 0.8174183514774495,
      "grad_norm": 0.4483345150947571,
      "learning_rate": 1.7150000000000004e-05,
      "log_odds_chosen": 12.23083209991455,
      "log_odds_ratio": -0.008293919265270233,
      "logits/chosen": 0.32675403356552124,
      "logits/rejected": 0.7867611050605774,
      "logps/chosen": -0.9917525053024292,
      "logps/rejected": -12.401079177856445,
      "loss": 3.4186,
      "nll_loss": 3.417818546295166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09917524456977844,
      "rewards/margins": 1.1409326791763306,
      "rewards/rejected": -1.2401078939437866,
      "step": 1314
    },
    {
      "epoch": 0.8180404354587869,
      "grad_norm": 0.3912915885448456,
      "learning_rate": 1.7125000000000003e-05,
      "log_odds_chosen": 15.708026885986328,
      "log_odds_ratio": -9.631742432247847e-05,
      "logits/chosen": 0.25834500789642334,
      "logits/rejected": 0.781840980052948,
      "logps/chosen": -1.0959607362747192,
      "logps/rejected": -16.29987907409668,
      "loss": 3.6535,
      "nll_loss": 3.6535050868988037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10959607362747192,
      "rewards/margins": 1.5203917026519775,
      "rewards/rejected": -1.6299877166748047,
      "step": 1315
    },
    {
      "epoch": 0.8186625194401245,
      "grad_norm": 0.5564141869544983,
      "learning_rate": 1.7100000000000002e-05,
      "log_odds_chosen": 16.535018920898438,
      "log_odds_ratio": -0.12745706737041473,
      "logits/chosen": 0.30819642543792725,
      "logits/rejected": 1.2198446989059448,
      "logps/chosen": -1.0392193794250488,
      "logps/rejected": -17.128646850585938,
      "loss": 2.7837,
      "nll_loss": 2.7709245681762695,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10392193496227264,
      "rewards/margins": 1.6089427471160889,
      "rewards/rejected": -1.7128647565841675,
      "step": 1316
    },
    {
      "epoch": 0.8192846034214619,
      "grad_norm": 0.6600592136383057,
      "learning_rate": 1.7075e-05,
      "log_odds_chosen": 10.514989852905273,
      "log_odds_ratio": -0.004317562095820904,
      "logits/chosen": 0.23065491020679474,
      "logits/rejected": 0.8288361430168152,
      "logps/chosen": -1.0467638969421387,
      "logps/rejected": -11.060022354125977,
      "loss": 2.5248,
      "nll_loss": 2.5243237018585205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10467638820409775,
      "rewards/margins": 1.0013258457183838,
      "rewards/rejected": -1.1060023307800293,
      "step": 1317
    },
    {
      "epoch": 0.8199066874027994,
      "grad_norm": 0.4944377839565277,
      "learning_rate": 1.705e-05,
      "log_odds_chosen": 11.389673233032227,
      "log_odds_ratio": -0.06905973702669144,
      "logits/chosen": 0.1772024929523468,
      "logits/rejected": 0.6251535415649414,
      "logps/chosen": -0.8641712665557861,
      "logps/rejected": -11.602714538574219,
      "loss": 2.9104,
      "nll_loss": 2.903456687927246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08641713857650757,
      "rewards/margins": 1.0738542079925537,
      "rewards/rejected": -1.160271406173706,
      "step": 1318
    },
    {
      "epoch": 0.8205287713841368,
      "grad_norm": 0.5295370221138,
      "learning_rate": 1.7025e-05,
      "log_odds_chosen": 9.117376327514648,
      "log_odds_ratio": -0.46799957752227783,
      "logits/chosen": 0.22593256831169128,
      "logits/rejected": 0.6932124495506287,
      "logps/chosen": -1.3216909170150757,
      "logps/rejected": -10.34849739074707,
      "loss": 2.7903,
      "nll_loss": 2.7435405254364014,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13216909766197205,
      "rewards/margins": 0.9026807546615601,
      "rewards/rejected": -1.0348498821258545,
      "step": 1319
    },
    {
      "epoch": 0.8211508553654744,
      "grad_norm": 0.518757164478302,
      "learning_rate": 1.7000000000000003e-05,
      "log_odds_chosen": 12.372821807861328,
      "log_odds_ratio": -0.2226611226797104,
      "logits/chosen": 0.2919222414493561,
      "logits/rejected": 0.8774810433387756,
      "logps/chosen": -1.1752337217330933,
      "logps/rejected": -13.215282440185547,
      "loss": 3.1039,
      "nll_loss": 3.0816164016723633,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11752337962388992,
      "rewards/margins": 1.2040050029754639,
      "rewards/rejected": -1.3215281963348389,
      "step": 1320
    },
    {
      "epoch": 0.8217729393468118,
      "grad_norm": 0.4607633352279663,
      "learning_rate": 1.6975000000000003e-05,
      "log_odds_chosen": 11.539329528808594,
      "log_odds_ratio": -0.10293762385845184,
      "logits/chosen": 0.19818146526813507,
      "logits/rejected": 1.1686244010925293,
      "logps/chosen": -1.1125632524490356,
      "logps/rejected": -12.267274856567383,
      "loss": 2.5394,
      "nll_loss": 2.5291550159454346,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11125633120536804,
      "rewards/margins": 1.1154711246490479,
      "rewards/rejected": -1.2267273664474487,
      "step": 1321
    },
    {
      "epoch": 0.8223950233281493,
      "grad_norm": 0.5890390276908875,
      "learning_rate": 1.6950000000000002e-05,
      "log_odds_chosen": 5.568022727966309,
      "log_odds_ratio": -0.10367351770401001,
      "logits/chosen": 0.35675832629203796,
      "logits/rejected": 0.8858349323272705,
      "logps/chosen": -1.0774688720703125,
      "logps/rejected": -6.139078617095947,
      "loss": 2.6681,
      "nll_loss": 2.6576852798461914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10774689167737961,
      "rewards/margins": 0.5061609745025635,
      "rewards/rejected": -0.6139078736305237,
      "step": 1322
    },
    {
      "epoch": 0.8230171073094867,
      "grad_norm": 0.9928566813468933,
      "learning_rate": 1.6925e-05,
      "log_odds_chosen": 12.515892028808594,
      "log_odds_ratio": -0.06518105417490005,
      "logits/chosen": 0.31150442361831665,
      "logits/rejected": 1.2270563840866089,
      "logps/chosen": -1.189500331878662,
      "logps/rejected": -13.337007522583008,
      "loss": 2.5602,
      "nll_loss": 2.55371356010437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1189500242471695,
      "rewards/margins": 1.2147507667541504,
      "rewards/rejected": -1.3337006568908691,
      "step": 1323
    },
    {
      "epoch": 0.8236391912908243,
      "grad_norm": 0.7025359869003296,
      "learning_rate": 1.69e-05,
      "log_odds_chosen": 9.76104736328125,
      "log_odds_ratio": -0.19300048053264618,
      "logits/chosen": 0.28773853182792664,
      "logits/rejected": 0.9788026809692383,
      "logps/chosen": -1.1405911445617676,
      "logps/rejected": -10.510811805725098,
      "loss": 2.6634,
      "nll_loss": 2.6441402435302734,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11405910551548004,
      "rewards/margins": 0.9370220303535461,
      "rewards/rejected": -1.0510811805725098,
      "step": 1324
    },
    {
      "epoch": 0.8242612752721618,
      "grad_norm": 0.4265744686126709,
      "learning_rate": 1.6875000000000004e-05,
      "log_odds_chosen": 10.766847610473633,
      "log_odds_ratio": -0.0524822473526001,
      "logits/chosen": 0.2768998444080353,
      "logits/rejected": 0.9822081327438354,
      "logps/chosen": -1.0433452129364014,
      "logps/rejected": -11.404045104980469,
      "loss": 2.9502,
      "nll_loss": 2.9449455738067627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1043345257639885,
      "rewards/margins": 1.0360699892044067,
      "rewards/rejected": -1.140404462814331,
      "step": 1325
    },
    {
      "epoch": 0.8248833592534992,
      "grad_norm": 0.587645947933197,
      "learning_rate": 1.6850000000000003e-05,
      "log_odds_chosen": 6.577492713928223,
      "log_odds_ratio": -0.14230968058109283,
      "logits/chosen": 0.030120672658085823,
      "logits/rejected": 0.39918053150177,
      "logps/chosen": -1.1130759716033936,
      "logps/rejected": -7.328277587890625,
      "loss": 2.5104,
      "nll_loss": 2.4961977005004883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11130759119987488,
      "rewards/margins": 0.6215201616287231,
      "rewards/rejected": -0.7328277826309204,
      "step": 1326
    },
    {
      "epoch": 0.8255054432348367,
      "grad_norm": 0.5352939367294312,
      "learning_rate": 1.6825000000000002e-05,
      "log_odds_chosen": 12.196432113647461,
      "log_odds_ratio": -0.1495496928691864,
      "logits/chosen": 0.1835860013961792,
      "logits/rejected": 1.0073965787887573,
      "logps/chosen": -1.0454256534576416,
      "logps/rejected": -12.764404296875,
      "loss": 2.84,
      "nll_loss": 2.8250114917755127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1045425683259964,
      "rewards/margins": 1.1718978881835938,
      "rewards/rejected": -1.2764403820037842,
      "step": 1327
    },
    {
      "epoch": 0.8261275272161742,
      "grad_norm": 0.4474393129348755,
      "learning_rate": 1.6800000000000002e-05,
      "log_odds_chosen": 17.64754295349121,
      "log_odds_ratio": -0.001237626769579947,
      "logits/chosen": 0.33965516090393066,
      "logits/rejected": 1.2508803606033325,
      "logps/chosen": -1.0787427425384521,
      "logps/rejected": -18.28293228149414,
      "loss": 3.2162,
      "nll_loss": 3.2160513401031494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10787426680326462,
      "rewards/margins": 1.72041916847229,
      "rewards/rejected": -1.8282933235168457,
      "step": 1328
    },
    {
      "epoch": 0.8267496111975117,
      "grad_norm": 0.44476574659347534,
      "learning_rate": 1.6775e-05,
      "log_odds_chosen": 11.613157272338867,
      "log_odds_ratio": -0.1723821610212326,
      "logits/chosen": 0.345926433801651,
      "logits/rejected": 1.0617916584014893,
      "logps/chosen": -1.2812693119049072,
      "logps/rejected": -12.575846672058105,
      "loss": 3.3687,
      "nll_loss": 3.3514435291290283,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12812693417072296,
      "rewards/margins": 1.129457712173462,
      "rewards/rejected": -1.257584571838379,
      "step": 1329
    },
    {
      "epoch": 0.8273716951788491,
      "grad_norm": 0.6063930988311768,
      "learning_rate": 1.675e-05,
      "log_odds_chosen": 9.643231391906738,
      "log_odds_ratio": -0.2011903077363968,
      "logits/chosen": 0.338983416557312,
      "logits/rejected": 1.080698013305664,
      "logps/chosen": -1.4720268249511719,
      "logps/rejected": -10.901077270507812,
      "loss": 2.9527,
      "nll_loss": 2.9325718879699707,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14720268547534943,
      "rewards/margins": 0.9429050087928772,
      "rewards/rejected": -1.0901076793670654,
      "step": 1330
    },
    {
      "epoch": 0.8279937791601866,
      "grad_norm": 0.4499737620353699,
      "learning_rate": 1.6725000000000003e-05,
      "log_odds_chosen": 9.355201721191406,
      "log_odds_ratio": -0.11042997241020203,
      "logits/chosen": 0.1041167601943016,
      "logits/rejected": 0.7633360624313354,
      "logps/chosen": -1.1983859539031982,
      "logps/rejected": -10.103107452392578,
      "loss": 2.7849,
      "nll_loss": 2.773890972137451,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11983861029148102,
      "rewards/margins": 0.8904721140861511,
      "rewards/rejected": -1.0103106498718262,
      "step": 1331
    },
    {
      "epoch": 0.8286158631415241,
      "grad_norm": 0.46015751361846924,
      "learning_rate": 1.6700000000000003e-05,
      "log_odds_chosen": 10.06387996673584,
      "log_odds_ratio": -0.05467294156551361,
      "logits/chosen": 0.1695416122674942,
      "logits/rejected": 0.6716436147689819,
      "logps/chosen": -0.9862855076789856,
      "logps/rejected": -10.507351875305176,
      "loss": 2.9439,
      "nll_loss": 2.938450336456299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09862855821847916,
      "rewards/margins": 0.9521065950393677,
      "rewards/rejected": -1.0507352352142334,
      "step": 1332
    },
    {
      "epoch": 0.8292379471228616,
      "grad_norm": 0.4361891746520996,
      "learning_rate": 1.6675000000000002e-05,
      "log_odds_chosen": 12.508257865905762,
      "log_odds_ratio": -0.026293473318219185,
      "logits/chosen": 0.2983800172805786,
      "logits/rejected": 1.2380685806274414,
      "logps/chosen": -1.0879460573196411,
      "logps/rejected": -13.190286636352539,
      "loss": 2.9347,
      "nll_loss": 2.9320545196533203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10879460722208023,
      "rewards/margins": 1.2102341651916504,
      "rewards/rejected": -1.3190287351608276,
      "step": 1333
    },
    {
      "epoch": 0.829860031104199,
      "grad_norm": 0.4361533522605896,
      "learning_rate": 1.665e-05,
      "log_odds_chosen": 7.443196773529053,
      "log_odds_ratio": -0.30054956674575806,
      "logits/chosen": 0.3471451997756958,
      "logits/rejected": 0.7988330125808716,
      "logps/chosen": -1.1569123268127441,
      "logps/rejected": -8.317983627319336,
      "loss": 3.2357,
      "nll_loss": 3.205623149871826,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11569122970104218,
      "rewards/margins": 0.7161070704460144,
      "rewards/rejected": -0.831798255443573,
      "step": 1334
    },
    {
      "epoch": 0.8304821150855366,
      "grad_norm": 0.7759374976158142,
      "learning_rate": 1.6625e-05,
      "log_odds_chosen": 8.826431274414062,
      "log_odds_ratio": -0.3530728220939636,
      "logits/chosen": 0.21989214420318604,
      "logits/rejected": 0.6294089555740356,
      "logps/chosen": -1.4264800548553467,
      "logps/rejected": -10.073540687561035,
      "loss": 3.312,
      "nll_loss": 3.2766780853271484,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14264801144599915,
      "rewards/margins": 0.8647060394287109,
      "rewards/rejected": -1.0073541402816772,
      "step": 1335
    },
    {
      "epoch": 0.831104199066874,
      "grad_norm": 0.9619458317756653,
      "learning_rate": 1.66e-05,
      "log_odds_chosen": 7.348890781402588,
      "log_odds_ratio": -0.2723667323589325,
      "logits/chosen": 0.18200621008872986,
      "logits/rejected": 0.7618054747581482,
      "logps/chosen": -1.065969467163086,
      "logps/rejected": -7.974705219268799,
      "loss": 2.7613,
      "nll_loss": 2.734013557434082,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1065969318151474,
      "rewards/margins": 0.6908735632896423,
      "rewards/rejected": -0.7974704504013062,
      "step": 1336
    },
    {
      "epoch": 0.8317262830482115,
      "grad_norm": 0.5136169791221619,
      "learning_rate": 1.6575000000000003e-05,
      "log_odds_chosen": 7.726633071899414,
      "log_odds_ratio": -0.2186044454574585,
      "logits/chosen": 0.34979715943336487,
      "logits/rejected": 0.7437832951545715,
      "logps/chosen": -1.210748553276062,
      "logps/rejected": -8.55751895904541,
      "loss": 3.2908,
      "nll_loss": 3.268960475921631,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1210748553276062,
      "rewards/margins": 0.7346770167350769,
      "rewards/rejected": -0.8557519316673279,
      "step": 1337
    },
    {
      "epoch": 0.832348367029549,
      "grad_norm": 0.45952683687210083,
      "learning_rate": 1.6550000000000002e-05,
      "log_odds_chosen": 15.065681457519531,
      "log_odds_ratio": -0.0024630618281662464,
      "logits/chosen": 0.13940998911857605,
      "logits/rejected": 0.7494243383407593,
      "logps/chosen": -0.9125413298606873,
      "logps/rejected": -15.253064155578613,
      "loss": 2.8324,
      "nll_loss": 2.8321423530578613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09125413745641708,
      "rewards/margins": 1.4340523481369019,
      "rewards/rejected": -1.5253064632415771,
      "step": 1338
    },
    {
      "epoch": 0.8329704510108865,
      "grad_norm": 0.6703150868415833,
      "learning_rate": 1.6525000000000002e-05,
      "log_odds_chosen": 10.943548202514648,
      "log_odds_ratio": -0.11169109493494034,
      "logits/chosen": 0.29511356353759766,
      "logits/rejected": 0.77367103099823,
      "logps/chosen": -1.0926419496536255,
      "logps/rejected": -11.617069244384766,
      "loss": 2.7409,
      "nll_loss": 2.7297096252441406,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10926420241594315,
      "rewards/margins": 1.0524427890777588,
      "rewards/rejected": -1.1617070436477661,
      "step": 1339
    },
    {
      "epoch": 0.833592534992224,
      "grad_norm": 0.5566127300262451,
      "learning_rate": 1.65e-05,
      "log_odds_chosen": 12.820943832397461,
      "log_odds_ratio": -0.010184680111706257,
      "logits/chosen": 0.16678117215633392,
      "logits/rejected": 0.7523674964904785,
      "logps/chosen": -0.9320427775382996,
      "logps/rejected": -13.13653564453125,
      "loss": 3.0966,
      "nll_loss": 3.0955657958984375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09320428222417831,
      "rewards/margins": 1.2204493284225464,
      "rewards/rejected": -1.313653588294983,
      "step": 1340
    },
    {
      "epoch": 0.8342146189735614,
      "grad_norm": 1.249294400215149,
      "learning_rate": 1.6475e-05,
      "log_odds_chosen": 13.536139488220215,
      "log_odds_ratio": -0.09148459136486053,
      "logits/chosen": 0.17023460566997528,
      "logits/rejected": 1.1541365385055542,
      "logps/chosen": -1.294020652770996,
      "logps/rejected": -14.549723625183105,
      "loss": 2.466,
      "nll_loss": 2.4568064212799072,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1294020712375641,
      "rewards/margins": 1.3255703449249268,
      "rewards/rejected": -1.454972267150879,
      "step": 1341
    },
    {
      "epoch": 0.8348367029548989,
      "grad_norm": 0.34336620569229126,
      "learning_rate": 1.645e-05,
      "log_odds_chosen": 11.245887756347656,
      "log_odds_ratio": -0.08879686892032623,
      "logits/chosen": 0.4275479018688202,
      "logits/rejected": 1.1122469902038574,
      "logps/chosen": -1.0759177207946777,
      "logps/rejected": -11.819448471069336,
      "loss": 3.6854,
      "nll_loss": 3.6764721870422363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10759177803993225,
      "rewards/margins": 1.0743530988693237,
      "rewards/rejected": -1.1819448471069336,
      "step": 1342
    },
    {
      "epoch": 0.8354587869362364,
      "grad_norm": 0.4405929446220398,
      "learning_rate": 1.6425000000000003e-05,
      "log_odds_chosen": 15.990028381347656,
      "log_odds_ratio": -0.00034587905975058675,
      "logits/chosen": 0.4133451581001282,
      "logits/rejected": 1.1971858739852905,
      "logps/chosen": -1.0017715692520142,
      "logps/rejected": -16.374874114990234,
      "loss": 3.6086,
      "nll_loss": 3.6085314750671387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10017715394496918,
      "rewards/margins": 1.537310242652893,
      "rewards/rejected": -1.6374874114990234,
      "step": 1343
    },
    {
      "epoch": 0.8360808709175739,
      "grad_norm": 0.43593719601631165,
      "learning_rate": 1.6400000000000002e-05,
      "log_odds_chosen": 11.253133773803711,
      "log_odds_ratio": -0.14229533076286316,
      "logits/chosen": 0.22059793770313263,
      "logits/rejected": 0.9180048108100891,
      "logps/chosen": -0.9364933967590332,
      "logps/rejected": -11.65854263305664,
      "loss": 2.7832,
      "nll_loss": 2.7689208984375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09364934265613556,
      "rewards/margins": 1.0722049474716187,
      "rewards/rejected": -1.1658543348312378,
      "step": 1344
    },
    {
      "epoch": 0.8367029548989113,
      "grad_norm": 6.538516044616699,
      "learning_rate": 1.6375e-05,
      "log_odds_chosen": 11.31826114654541,
      "log_odds_ratio": -0.12284395098686218,
      "logits/chosen": 0.3450809717178345,
      "logits/rejected": 0.881148099899292,
      "logps/chosen": -0.9467482566833496,
      "logps/rejected": -11.267402648925781,
      "loss": 3.2558,
      "nll_loss": 3.243523597717285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09467482566833496,
      "rewards/margins": 1.0320653915405273,
      "rewards/rejected": -1.1267402172088623,
      "step": 1345
    },
    {
      "epoch": 0.8373250388802488,
      "grad_norm": 0.45118892192840576,
      "learning_rate": 1.635e-05,
      "log_odds_chosen": 8.51021957397461,
      "log_odds_ratio": -0.24373793601989746,
      "logits/chosen": 0.05161430686712265,
      "logits/rejected": 0.45903950929641724,
      "logps/chosen": -1.0309022665023804,
      "logps/rejected": -9.030682563781738,
      "loss": 2.5431,
      "nll_loss": 2.518749713897705,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10309022665023804,
      "rewards/margins": 0.7999781370162964,
      "rewards/rejected": -0.9030683040618896,
      "step": 1346
    },
    {
      "epoch": 0.8379471228615863,
      "grad_norm": 0.8182756900787354,
      "learning_rate": 1.6325e-05,
      "log_odds_chosen": 9.616655349731445,
      "log_odds_ratio": -0.15306349098682404,
      "logits/chosen": 0.20947861671447754,
      "logits/rejected": 0.5957802534103394,
      "logps/chosen": -1.1415512561798096,
      "logps/rejected": -10.409021377563477,
      "loss": 3.2896,
      "nll_loss": 3.274308443069458,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11415514349937439,
      "rewards/margins": 0.9267469644546509,
      "rewards/rejected": -1.0409021377563477,
      "step": 1347
    },
    {
      "epoch": 0.8385692068429238,
      "grad_norm": 0.5537295341491699,
      "learning_rate": 1.63e-05,
      "log_odds_chosen": 18.167945861816406,
      "log_odds_ratio": -0.027182959020137787,
      "logits/chosen": 0.30054154992103577,
      "logits/rejected": 1.0188592672348022,
      "logps/chosen": -0.8114546537399292,
      "logps/rejected": -18.324691772460938,
      "loss": 2.6675,
      "nll_loss": 2.6647937297821045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08114546537399292,
      "rewards/margins": 1.7513236999511719,
      "rewards/rejected": -1.83246910572052,
      "step": 1348
    },
    {
      "epoch": 0.8391912908242612,
      "grad_norm": 0.7026792764663696,
      "learning_rate": 1.6275000000000003e-05,
      "log_odds_chosen": 14.551230430603027,
      "log_odds_ratio": -0.09296683967113495,
      "logits/chosen": 0.3387167453765869,
      "logits/rejected": 0.8440542221069336,
      "logps/chosen": -0.9055599570274353,
      "logps/rejected": -14.753962516784668,
      "loss": 3.3845,
      "nll_loss": 3.3752224445343018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09055599570274353,
      "rewards/margins": 1.3848403692245483,
      "rewards/rejected": -1.4753963947296143,
      "step": 1349
    },
    {
      "epoch": 0.8398133748055988,
      "grad_norm": 0.4853743314743042,
      "learning_rate": 1.6250000000000002e-05,
      "log_odds_chosen": 15.019338607788086,
      "log_odds_ratio": -0.0688743144273758,
      "logits/chosen": 0.2787754237651825,
      "logits/rejected": 1.2702405452728271,
      "logps/chosen": -1.029766321182251,
      "logps/rejected": -15.654651641845703,
      "loss": 2.6953,
      "nll_loss": 2.688436985015869,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10297662764787674,
      "rewards/margins": 1.4624884128570557,
      "rewards/rejected": -1.5654650926589966,
      "step": 1350
    },
    {
      "epoch": 0.8404354587869363,
      "grad_norm": 1.4702285528182983,
      "learning_rate": 1.6225e-05,
      "log_odds_chosen": 7.845383167266846,
      "log_odds_ratio": -0.0672604888677597,
      "logits/chosen": 0.3774755001068115,
      "logits/rejected": 0.8473259210586548,
      "logps/chosen": -0.9018560647964478,
      "logps/rejected": -8.014701843261719,
      "loss": 3.4515,
      "nll_loss": 3.4447741508483887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09018560498952866,
      "rewards/margins": 0.7112846374511719,
      "rewards/rejected": -0.801470160484314,
      "step": 1351
    },
    {
      "epoch": 0.8410575427682737,
      "grad_norm": 0.5535169839859009,
      "learning_rate": 1.62e-05,
      "log_odds_chosen": 14.512027740478516,
      "log_odds_ratio": -0.09268511831760406,
      "logits/chosen": 0.1820366382598877,
      "logits/rejected": 0.9162260293960571,
      "logps/chosen": -0.9260967969894409,
      "logps/rejected": -14.837316513061523,
      "loss": 3.0602,
      "nll_loss": 3.050899028778076,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09260967373847961,
      "rewards/margins": 1.3911221027374268,
      "rewards/rejected": -1.4837316274642944,
      "step": 1352
    },
    {
      "epoch": 0.8416796267496112,
      "grad_norm": 0.4414682388305664,
      "learning_rate": 1.6175e-05,
      "log_odds_chosen": 16.720481872558594,
      "log_odds_ratio": -0.10064268112182617,
      "logits/chosen": 0.27310311794281006,
      "logits/rejected": 1.1602401733398438,
      "logps/chosen": -1.3199316263198853,
      "logps/rejected": -17.78493881225586,
      "loss": 3.1636,
      "nll_loss": 3.1535096168518066,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13199317455291748,
      "rewards/margins": 1.646500825881958,
      "rewards/rejected": -1.778494119644165,
      "step": 1353
    },
    {
      "epoch": 0.8423017107309487,
      "grad_norm": 0.46305572986602783,
      "learning_rate": 1.6150000000000003e-05,
      "log_odds_chosen": 9.767788887023926,
      "log_odds_ratio": -0.2507934272289276,
      "logits/chosen": 0.15547390282154083,
      "logits/rejected": 0.7884215116500854,
      "logps/chosen": -1.4701690673828125,
      "logps/rejected": -11.03095817565918,
      "loss": 2.8695,
      "nll_loss": 2.8444552421569824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14701691269874573,
      "rewards/margins": 0.9560788869857788,
      "rewards/rejected": -1.1030957698822021,
      "step": 1354
    },
    {
      "epoch": 0.8429237947122862,
      "grad_norm": 0.42249321937561035,
      "learning_rate": 1.6125000000000002e-05,
      "log_odds_chosen": 19.457439422607422,
      "log_odds_ratio": -2.3841880647523794e-07,
      "logits/chosen": 0.2563483417034149,
      "logits/rejected": 1.0023006200790405,
      "logps/chosen": -0.9900782108306885,
      "logps/rejected": -19.732521057128906,
      "loss": 3.4363,
      "nll_loss": 3.4362709522247314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09900782257318497,
      "rewards/margins": 1.8742443323135376,
      "rewards/rejected": -1.9732520580291748,
      "step": 1355
    },
    {
      "epoch": 0.8435458786936236,
      "grad_norm": 0.5480783581733704,
      "learning_rate": 1.6100000000000002e-05,
      "log_odds_chosen": 12.938760757446289,
      "log_odds_ratio": -0.20344708859920502,
      "logits/chosen": 0.382476806640625,
      "logits/rejected": 1.0104025602340698,
      "logps/chosen": -0.9441956281661987,
      "logps/rejected": -13.267868995666504,
      "loss": 3.296,
      "nll_loss": 3.2756340503692627,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09441956877708435,
      "rewards/margins": 1.2323673963546753,
      "rewards/rejected": -1.326786994934082,
      "step": 1356
    },
    {
      "epoch": 0.8441679626749611,
      "grad_norm": 0.6357641220092773,
      "learning_rate": 1.6075e-05,
      "log_odds_chosen": 9.309688568115234,
      "log_odds_ratio": -0.06144268810749054,
      "logits/chosen": 0.19917547702789307,
      "logits/rejected": 0.6526224613189697,
      "logps/chosen": -0.8743935823440552,
      "logps/rejected": -9.478578567504883,
      "loss": 2.9935,
      "nll_loss": 2.9873154163360596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08743935823440552,
      "rewards/margins": 0.8604185581207275,
      "rewards/rejected": -0.9478578567504883,
      "step": 1357
    },
    {
      "epoch": 0.8447900466562986,
      "grad_norm": 0.40837082266807556,
      "learning_rate": 1.605e-05,
      "log_odds_chosen": 12.638919830322266,
      "log_odds_ratio": -0.1141565814614296,
      "logits/chosen": 0.2843877375125885,
      "logits/rejected": 1.202309250831604,
      "logps/chosen": -1.1809484958648682,
      "logps/rejected": -13.486981391906738,
      "loss": 2.9087,
      "nll_loss": 2.897247314453125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11809486150741577,
      "rewards/margins": 1.2306030988693237,
      "rewards/rejected": -1.3486980199813843,
      "step": 1358
    },
    {
      "epoch": 0.8454121306376361,
      "grad_norm": 0.619658350944519,
      "learning_rate": 1.6025e-05,
      "log_odds_chosen": 12.037456512451172,
      "log_odds_ratio": -0.10075844824314117,
      "logits/chosen": 0.3413699269294739,
      "logits/rejected": 0.8576540946960449,
      "logps/chosen": -0.8795664310455322,
      "logps/rejected": -12.2918062210083,
      "loss": 3.488,
      "nll_loss": 3.477937698364258,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08795663714408875,
      "rewards/margins": 1.1412239074707031,
      "rewards/rejected": -1.2291805744171143,
      "step": 1359
    },
    {
      "epoch": 0.8460342146189735,
      "grad_norm": 0.410817950963974,
      "learning_rate": 1.6000000000000003e-05,
      "log_odds_chosen": 10.435465812683105,
      "log_odds_ratio": -0.2789941430091858,
      "logits/chosen": 0.1286708265542984,
      "logits/rejected": 0.44350332021713257,
      "logps/chosen": -1.1692880392074585,
      "logps/rejected": -11.24249267578125,
      "loss": 2.9848,
      "nll_loss": 2.9569311141967773,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11692880839109421,
      "rewards/margins": 1.007320523262024,
      "rewards/rejected": -1.1242493391036987,
      "step": 1360
    },
    {
      "epoch": 0.846656298600311,
      "grad_norm": 0.5594186186790466,
      "learning_rate": 1.5975000000000002e-05,
      "log_odds_chosen": 11.055662155151367,
      "log_odds_ratio": -0.04379449784755707,
      "logits/chosen": 0.31934839487075806,
      "logits/rejected": 1.1686406135559082,
      "logps/chosen": -1.1824477910995483,
      "logps/rejected": -11.840726852416992,
      "loss": 2.8957,
      "nll_loss": 2.8913650512695312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11824478209018707,
      "rewards/margins": 1.0658278465270996,
      "rewards/rejected": -1.184072732925415,
      "step": 1361
    },
    {
      "epoch": 0.8472783825816486,
      "grad_norm": 0.5316110253334045,
      "learning_rate": 1.595e-05,
      "log_odds_chosen": 5.959257125854492,
      "log_odds_ratio": -0.056952349841594696,
      "logits/chosen": 0.4386367201805115,
      "logits/rejected": 0.5847421884536743,
      "logps/chosen": -0.9345696568489075,
      "logps/rejected": -6.30491304397583,
      "loss": 3.2712,
      "nll_loss": 3.2655532360076904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09345697611570358,
      "rewards/margins": 0.5370343923568726,
      "rewards/rejected": -0.6304913759231567,
      "step": 1362
    },
    {
      "epoch": 0.847900466562986,
      "grad_norm": 0.4794823229312897,
      "learning_rate": 1.5925e-05,
      "log_odds_chosen": 10.93669319152832,
      "log_odds_ratio": -0.013783779926598072,
      "logits/chosen": 0.20791217684745789,
      "logits/rejected": 0.7548701763153076,
      "logps/chosen": -1.2015867233276367,
      "logps/rejected": -11.472372055053711,
      "loss": 3.1463,
      "nll_loss": 3.1448731422424316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12015866488218307,
      "rewards/margins": 1.0270785093307495,
      "rewards/rejected": -1.1472373008728027,
      "step": 1363
    },
    {
      "epoch": 0.8485225505443235,
      "grad_norm": 0.5277031660079956,
      "learning_rate": 1.59e-05,
      "log_odds_chosen": 8.793513298034668,
      "log_odds_ratio": -0.3329012989997864,
      "logits/chosen": 0.36735790967941284,
      "logits/rejected": 0.7775804996490479,
      "logps/chosen": -1.056816816329956,
      "logps/rejected": -9.48435115814209,
      "loss": 3.1065,
      "nll_loss": 3.0732502937316895,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10568168014287949,
      "rewards/margins": 0.8427534699440002,
      "rewards/rejected": -0.9484350681304932,
      "step": 1364
    },
    {
      "epoch": 0.8491446345256609,
      "grad_norm": 0.9272438287734985,
      "learning_rate": 1.5875e-05,
      "log_odds_chosen": 11.923362731933594,
      "log_odds_ratio": -0.06839544326066971,
      "logits/chosen": 0.14544972777366638,
      "logits/rejected": 0.8777525424957275,
      "logps/chosen": -1.2184803485870361,
      "logps/rejected": -12.776731491088867,
      "loss": 2.6382,
      "nll_loss": 2.631315231323242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12184803187847137,
      "rewards/margins": 1.155825138092041,
      "rewards/rejected": -1.2776731252670288,
      "step": 1365
    },
    {
      "epoch": 0.8497667185069985,
      "grad_norm": 0.44855692982673645,
      "learning_rate": 1.5850000000000002e-05,
      "log_odds_chosen": 8.991267204284668,
      "log_odds_ratio": -0.09500695019960403,
      "logits/chosen": 0.3809604346752167,
      "logits/rejected": 1.16422438621521,
      "logps/chosen": -1.182137370109558,
      "logps/rejected": -9.790534019470215,
      "loss": 3.465,
      "nll_loss": 3.455484390258789,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11821374297142029,
      "rewards/margins": 0.8608396649360657,
      "rewards/rejected": -0.9790534377098083,
      "step": 1366
    },
    {
      "epoch": 0.8503888024883359,
      "grad_norm": 0.5001557469367981,
      "learning_rate": 1.5825000000000002e-05,
      "log_odds_chosen": 11.170446395874023,
      "log_odds_ratio": -0.019392523914575577,
      "logits/chosen": 0.18927381932735443,
      "logits/rejected": 1.0548985004425049,
      "logps/chosen": -1.0558704137802124,
      "logps/rejected": -11.756693840026855,
      "loss": 2.6287,
      "nll_loss": 2.6267898082733154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10558703541755676,
      "rewards/margins": 1.0700823068618774,
      "rewards/rejected": -1.1756694316864014,
      "step": 1367
    },
    {
      "epoch": 0.8510108864696734,
      "grad_norm": 0.553524374961853,
      "learning_rate": 1.58e-05,
      "log_odds_chosen": 10.177559852600098,
      "log_odds_ratio": -0.12094742059707642,
      "logits/chosen": 0.17707903683185577,
      "logits/rejected": 0.9806874394416809,
      "logps/chosen": -1.1824595928192139,
      "logps/rejected": -10.97883415222168,
      "loss": 2.6865,
      "nll_loss": 2.67435622215271,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11824595183134079,
      "rewards/margins": 0.9796375036239624,
      "rewards/rejected": -1.0978834629058838,
      "step": 1368
    },
    {
      "epoch": 0.8516329704510109,
      "grad_norm": 0.6485047340393066,
      "learning_rate": 1.5775e-05,
      "log_odds_chosen": 10.440032958984375,
      "log_odds_ratio": -0.2638293206691742,
      "logits/chosen": 0.2368917465209961,
      "logits/rejected": 1.0509949922561646,
      "logps/chosen": -1.2486120462417603,
      "logps/rejected": -11.450448989868164,
      "loss": 2.642,
      "nll_loss": 2.615628242492676,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1248612031340599,
      "rewards/margins": 1.020183801651001,
      "rewards/rejected": -1.1450450420379639,
      "step": 1369
    },
    {
      "epoch": 0.8522550544323484,
      "grad_norm": 0.472089558839798,
      "learning_rate": 1.575e-05,
      "log_odds_chosen": 2.847292423248291,
      "log_odds_ratio": -0.5115450620651245,
      "logits/chosen": 0.1676526665687561,
      "logits/rejected": 0.23850813508033752,
      "logps/chosen": -1.163887619972229,
      "logps/rejected": -3.8988606929779053,
      "loss": 2.9222,
      "nll_loss": 2.871020793914795,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11638876795768738,
      "rewards/margins": 0.2734973132610321,
      "rewards/rejected": -0.3898860514163971,
      "step": 1370
    },
    {
      "epoch": 0.8528771384136858,
      "grad_norm": 1.902549386024475,
      "learning_rate": 1.5725e-05,
      "log_odds_chosen": 9.938411712646484,
      "log_odds_ratio": -0.16199146211147308,
      "logits/chosen": 0.32466983795166016,
      "logits/rejected": 1.023469090461731,
      "logps/chosen": -0.9341733455657959,
      "logps/rejected": -10.43630599975586,
      "loss": 3.0852,
      "nll_loss": 3.0690159797668457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09341733902692795,
      "rewards/margins": 0.9502133131027222,
      "rewards/rejected": -1.043630599975586,
      "step": 1371
    },
    {
      "epoch": 0.8534992223950233,
      "grad_norm": 0.5687903761863708,
      "learning_rate": 1.5700000000000002e-05,
      "log_odds_chosen": 9.373761177062988,
      "log_odds_ratio": -0.13499051332473755,
      "logits/chosen": 0.2836533188819885,
      "logits/rejected": 0.8441271781921387,
      "logps/chosen": -0.9228062629699707,
      "logps/rejected": -9.52553939819336,
      "loss": 3.0099,
      "nll_loss": 2.9963736534118652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09228062629699707,
      "rewards/margins": 0.8602732419967651,
      "rewards/rejected": -0.9525538682937622,
      "step": 1372
    },
    {
      "epoch": 0.8541213063763609,
      "grad_norm": 0.8326713442802429,
      "learning_rate": 1.5675e-05,
      "log_odds_chosen": 10.913453102111816,
      "log_odds_ratio": -0.16819366812705994,
      "logits/chosen": 0.3695370554924011,
      "logits/rejected": 0.9171431064605713,
      "logps/chosen": -1.0279958248138428,
      "logps/rejected": -11.530767440795898,
      "loss": 2.7988,
      "nll_loss": 2.7819666862487793,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10279958695173264,
      "rewards/margins": 1.0502771139144897,
      "rewards/rejected": -1.1530767679214478,
      "step": 1373
    },
    {
      "epoch": 0.8547433903576983,
      "grad_norm": 0.47954127192497253,
      "learning_rate": 1.565e-05,
      "log_odds_chosen": 11.381818771362305,
      "log_odds_ratio": -0.08887384831905365,
      "logits/chosen": 0.22594711184501648,
      "logits/rejected": 1.1135444641113281,
      "logps/chosen": -1.155029535293579,
      "logps/rejected": -12.046524047851562,
      "loss": 2.631,
      "nll_loss": 2.622159004211426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11550294607877731,
      "rewards/margins": 1.0891494750976562,
      "rewards/rejected": -1.2046523094177246,
      "step": 1374
    },
    {
      "epoch": 0.8553654743390358,
      "grad_norm": 1.0418078899383545,
      "learning_rate": 1.5625e-05,
      "log_odds_chosen": 5.646578311920166,
      "log_odds_ratio": -0.22757403552532196,
      "logits/chosen": 0.3599763512611389,
      "logits/rejected": 0.44046440720558167,
      "logps/chosen": -0.9107646346092224,
      "logps/rejected": -6.134728908538818,
      "loss": 2.9993,
      "nll_loss": 2.9765775203704834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09107646346092224,
      "rewards/margins": 0.522396445274353,
      "rewards/rejected": -0.6134728789329529,
      "step": 1375
    },
    {
      "epoch": 0.8559875583203732,
      "grad_norm": 1.584696650505066,
      "learning_rate": 1.56e-05,
      "log_odds_chosen": 15.953493118286133,
      "log_odds_ratio": -0.13301169872283936,
      "logits/chosen": 0.3697129189968109,
      "logits/rejected": 1.2205836772918701,
      "logps/chosen": -1.839653730392456,
      "logps/rejected": -17.536806106567383,
      "loss": 3.5137,
      "nll_loss": 3.5003836154937744,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18396538496017456,
      "rewards/margins": 1.5697153806686401,
      "rewards/rejected": -1.75368070602417,
      "step": 1376
    },
    {
      "epoch": 0.8566096423017108,
      "grad_norm": 0.394852876663208,
      "learning_rate": 1.5575e-05,
      "log_odds_chosen": 10.27212142944336,
      "log_odds_ratio": -0.19856134057044983,
      "logits/chosen": 0.3533197045326233,
      "logits/rejected": 0.7798941135406494,
      "logps/chosen": -0.886967658996582,
      "logps/rejected": -10.664911270141602,
      "loss": 3.3432,
      "nll_loss": 3.3233642578125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08869676291942596,
      "rewards/margins": 0.9777944087982178,
      "rewards/rejected": -1.0664911270141602,
      "step": 1377
    },
    {
      "epoch": 0.8572317262830482,
      "grad_norm": 0.5107656121253967,
      "learning_rate": 1.5550000000000002e-05,
      "log_odds_chosen": 10.686027526855469,
      "log_odds_ratio": -0.18517914414405823,
      "logits/chosen": 0.13516952097415924,
      "logits/rejected": 0.5872347950935364,
      "logps/chosen": -0.9439507722854614,
      "logps/rejected": -11.0865478515625,
      "loss": 2.3468,
      "nll_loss": 2.328307628631592,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09439507126808167,
      "rewards/margins": 1.014259696006775,
      "rewards/rejected": -1.1086547374725342,
      "step": 1378
    },
    {
      "epoch": 0.8578538102643857,
      "grad_norm": 0.7838579416275024,
      "learning_rate": 1.5525e-05,
      "log_odds_chosen": 8.990863800048828,
      "log_odds_ratio": -0.24506224691867828,
      "logits/chosen": 0.3935107886791229,
      "logits/rejected": 1.1674621105194092,
      "logps/chosen": -1.3738632202148438,
      "logps/rejected": -10.04948902130127,
      "loss": 3.5415,
      "nll_loss": 3.516964912414551,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13738630712032318,
      "rewards/margins": 0.8675625920295715,
      "rewards/rejected": -1.0049489736557007,
      "step": 1379
    },
    {
      "epoch": 0.8584758942457231,
      "grad_norm": 0.5477966666221619,
      "learning_rate": 1.55e-05,
      "log_odds_chosen": 11.66675090789795,
      "log_odds_ratio": -0.23476821184158325,
      "logits/chosen": 0.27057936787605286,
      "logits/rejected": 0.9670227766036987,
      "logps/chosen": -1.2665765285491943,
      "logps/rejected": -12.575981140136719,
      "loss": 2.9679,
      "nll_loss": 2.9444386959075928,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1266576498746872,
      "rewards/margins": 1.1309404373168945,
      "rewards/rejected": -1.2575981616973877,
      "step": 1380
    },
    {
      "epoch": 0.8590979782270607,
      "grad_norm": 0.6186298727989197,
      "learning_rate": 1.5475e-05,
      "log_odds_chosen": 6.804127216339111,
      "log_odds_ratio": -0.29192444682121277,
      "logits/chosen": 0.4237322211265564,
      "logits/rejected": 0.6319191455841064,
      "logps/chosen": -1.1395926475524902,
      "logps/rejected": -7.732236385345459,
      "loss": 3.1793,
      "nll_loss": 3.150071859359741,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11395925283432007,
      "rewards/margins": 0.659264326095581,
      "rewards/rejected": -0.7732236385345459,
      "step": 1381
    },
    {
      "epoch": 0.8597200622083981,
      "grad_norm": 0.3818768262863159,
      "learning_rate": 1.545e-05,
      "log_odds_chosen": 15.892073631286621,
      "log_odds_ratio": -0.08740878105163574,
      "logits/chosen": 0.367888867855072,
      "logits/rejected": 1.3535079956054688,
      "logps/chosen": -1.0746917724609375,
      "logps/rejected": -16.52185821533203,
      "loss": 3.2333,
      "nll_loss": 3.2245535850524902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10746918618679047,
      "rewards/margins": 1.544716715812683,
      "rewards/rejected": -1.6521859169006348,
      "step": 1382
    },
    {
      "epoch": 0.8603421461897356,
      "grad_norm": 0.5417101383209229,
      "learning_rate": 1.5425000000000002e-05,
      "log_odds_chosen": 15.340182304382324,
      "log_odds_ratio": -0.05749611184000969,
      "logits/chosen": 0.338359534740448,
      "logits/rejected": 1.1421986818313599,
      "logps/chosen": -0.9286530017852783,
      "logps/rejected": -15.792999267578125,
      "loss": 2.9544,
      "nll_loss": 2.948611259460449,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09286530315876007,
      "rewards/margins": 1.4864346981048584,
      "rewards/rejected": -1.579300045967102,
      "step": 1383
    },
    {
      "epoch": 0.860964230171073,
      "grad_norm": 0.7678720355033875,
      "learning_rate": 1.54e-05,
      "log_odds_chosen": 16.128339767456055,
      "log_odds_ratio": -0.10341505706310272,
      "logits/chosen": 0.2992256283760071,
      "logits/rejected": 1.0462899208068848,
      "logps/chosen": -1.1855686902999878,
      "logps/rejected": -16.78167152404785,
      "loss": 3.2203,
      "nll_loss": 3.209942102432251,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11855687201023102,
      "rewards/margins": 1.5596102476119995,
      "rewards/rejected": -1.6781671047210693,
      "step": 1384
    },
    {
      "epoch": 0.8615863141524106,
      "grad_norm": 0.5416978597640991,
      "learning_rate": 1.5375e-05,
      "log_odds_chosen": 9.19207763671875,
      "log_odds_ratio": -0.09668044000864029,
      "logits/chosen": 0.30826884508132935,
      "logits/rejected": 1.0236417055130005,
      "logps/chosen": -1.0911591053009033,
      "logps/rejected": -9.906354904174805,
      "loss": 2.6784,
      "nll_loss": 2.6686997413635254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10911591351032257,
      "rewards/margins": 0.8815194964408875,
      "rewards/rejected": -0.9906355142593384,
      "step": 1385
    },
    {
      "epoch": 0.862208398133748,
      "grad_norm": 0.41511037945747375,
      "learning_rate": 1.535e-05,
      "log_odds_chosen": 13.163442611694336,
      "log_odds_ratio": -0.0031957889441400766,
      "logits/chosen": 0.3195352554321289,
      "logits/rejected": 1.319638967514038,
      "logps/chosen": -1.1745014190673828,
      "logps/rejected": -13.899215698242188,
      "loss": 3.3087,
      "nll_loss": 3.3083767890930176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11745014786720276,
      "rewards/margins": 1.272471308708191,
      "rewards/rejected": -1.3899214267730713,
      "step": 1386
    },
    {
      "epoch": 0.8628304821150855,
      "grad_norm": 0.52939373254776,
      "learning_rate": 1.5325e-05,
      "log_odds_chosen": 15.242445945739746,
      "log_odds_ratio": -0.08322323113679886,
      "logits/chosen": 0.16968780755996704,
      "logits/rejected": 1.290441870689392,
      "logps/chosen": -1.0485494136810303,
      "logps/rejected": -15.906908988952637,
      "loss": 2.486,
      "nll_loss": 2.4777145385742188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10485494136810303,
      "rewards/margins": 1.4858359098434448,
      "rewards/rejected": -1.5906908512115479,
      "step": 1387
    },
    {
      "epoch": 0.8634525660964231,
      "grad_norm": 0.5325446724891663,
      "learning_rate": 1.53e-05,
      "log_odds_chosen": 15.141090393066406,
      "log_odds_ratio": -0.21355324983596802,
      "logits/chosen": 0.35596466064453125,
      "logits/rejected": 1.269651174545288,
      "logps/chosen": -1.2448396682739258,
      "logps/rejected": -16.096210479736328,
      "loss": 3.0043,
      "nll_loss": 2.982938766479492,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12448396533727646,
      "rewards/margins": 1.4851369857788086,
      "rewards/rejected": -1.6096210479736328,
      "step": 1388
    },
    {
      "epoch": 0.8640746500777605,
      "grad_norm": 0.6094441413879395,
      "learning_rate": 1.5275000000000002e-05,
      "log_odds_chosen": 14.076630592346191,
      "log_odds_ratio": -0.19824470579624176,
      "logits/chosen": 0.4293285608291626,
      "logits/rejected": 1.2163474559783936,
      "logps/chosen": -1.2386701107025146,
      "logps/rejected": -15.064802169799805,
      "loss": 3.1363,
      "nll_loss": 3.1165122985839844,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12386702001094818,
      "rewards/margins": 1.382613182067871,
      "rewards/rejected": -1.5064802169799805,
      "step": 1389
    },
    {
      "epoch": 0.864696734059098,
      "grad_norm": 0.4554082155227661,
      "learning_rate": 1.525e-05,
      "log_odds_chosen": 14.69717025756836,
      "log_odds_ratio": -0.10522916167974472,
      "logits/chosen": 0.3967858552932739,
      "logits/rejected": 1.06642746925354,
      "logps/chosen": -0.7577130794525146,
      "logps/rejected": -14.894484519958496,
      "loss": 3.5846,
      "nll_loss": 3.5740511417388916,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07577130943536758,
      "rewards/margins": 1.4136772155761719,
      "rewards/rejected": -1.4894484281539917,
      "step": 1390
    },
    {
      "epoch": 0.8653188180404354,
      "grad_norm": 0.4598383605480194,
      "learning_rate": 1.5225e-05,
      "log_odds_chosen": 17.62496566772461,
      "log_odds_ratio": -2.831222616350715e-07,
      "logits/chosen": 0.29642823338508606,
      "logits/rejected": 1.3097069263458252,
      "logps/chosen": -1.1783454418182373,
      "logps/rejected": -18.41707992553711,
      "loss": 3.0947,
      "nll_loss": 3.0946526527404785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11783455312252045,
      "rewards/margins": 1.7238733768463135,
      "rewards/rejected": -1.8417079448699951,
      "step": 1391
    },
    {
      "epoch": 0.865940902021773,
      "grad_norm": 0.45408785343170166,
      "learning_rate": 1.52e-05,
      "log_odds_chosen": 11.176227569580078,
      "log_odds_ratio": -0.11154050379991531,
      "logits/chosen": 0.11151541024446487,
      "logits/rejected": 0.945254921913147,
      "logps/chosen": -1.101738452911377,
      "logps/rejected": -11.896114349365234,
      "loss": 2.6586,
      "nll_loss": 2.647481918334961,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11017385125160217,
      "rewards/margins": 1.0794377326965332,
      "rewards/rejected": -1.1896114349365234,
      "step": 1392
    },
    {
      "epoch": 0.8665629860031104,
      "grad_norm": 0.38992127776145935,
      "learning_rate": 1.5175e-05,
      "log_odds_chosen": 16.277109146118164,
      "log_odds_ratio": -0.00011801975051639602,
      "logits/chosen": 0.32067984342575073,
      "logits/rejected": 1.577821969985962,
      "logps/chosen": -1.2991023063659668,
      "logps/rejected": -17.236865997314453,
      "loss": 3.2608,
      "nll_loss": 3.260770797729492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12991024553775787,
      "rewards/margins": 1.5937763452529907,
      "rewards/rejected": -1.723686695098877,
      "step": 1393
    },
    {
      "epoch": 0.8671850699844479,
      "grad_norm": 0.6620752811431885,
      "learning_rate": 1.515e-05,
      "log_odds_chosen": 8.851555824279785,
      "log_odds_ratio": -0.09632323682308197,
      "logits/chosen": 0.25070205330848694,
      "logits/rejected": 0.5107638835906982,
      "logps/chosen": -0.8554275035858154,
      "logps/rejected": -9.137062072753906,
      "loss": 3.2416,
      "nll_loss": 3.2320029735565186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08554275333881378,
      "rewards/margins": 0.8281635046005249,
      "rewards/rejected": -0.9137061834335327,
      "step": 1394
    },
    {
      "epoch": 0.8678071539657853,
      "grad_norm": 0.8303456902503967,
      "learning_rate": 1.5125e-05,
      "log_odds_chosen": 16.057336807250977,
      "log_odds_ratio": -0.00020326176309026778,
      "logits/chosen": 0.24806132912635803,
      "logits/rejected": 1.1189615726470947,
      "logps/chosen": -1.0641580820083618,
      "logps/rejected": -16.5771484375,
      "loss": 3.136,
      "nll_loss": 3.1359386444091797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10641580075025558,
      "rewards/margins": 1.551298975944519,
      "rewards/rejected": -1.6577147245407104,
      "step": 1395
    },
    {
      "epoch": 0.8684292379471229,
      "grad_norm": 0.6153732538223267,
      "learning_rate": 1.51e-05,
      "log_odds_chosen": 8.939095497131348,
      "log_odds_ratio": -0.13549737632274628,
      "logits/chosen": -0.030933350324630737,
      "logits/rejected": 0.6015803813934326,
      "logps/chosen": -1.3215835094451904,
      "logps/rejected": -9.989289283752441,
      "loss": 2.6042,
      "nll_loss": 2.5906641483306885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1321583390235901,
      "rewards/margins": 0.8667706251144409,
      "rewards/rejected": -0.9989289045333862,
      "step": 1396
    },
    {
      "epoch": 0.8690513219284604,
      "grad_norm": 0.546840250492096,
      "learning_rate": 1.5075e-05,
      "log_odds_chosen": 13.476724624633789,
      "log_odds_ratio": -0.120454341173172,
      "logits/chosen": 0.33575960993766785,
      "logits/rejected": 1.0339983701705933,
      "logps/chosen": -1.5418827533721924,
      "logps/rejected": -14.785687446594238,
      "loss": 3.1669,
      "nll_loss": 3.1548702716827393,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15418827533721924,
      "rewards/margins": 1.3243803977966309,
      "rewards/rejected": -1.4785687923431396,
      "step": 1397
    },
    {
      "epoch": 0.8696734059097978,
      "grad_norm": 0.4228728413581848,
      "learning_rate": 1.505e-05,
      "log_odds_chosen": 13.623929977416992,
      "log_odds_ratio": -0.10882086306810379,
      "logits/chosen": 0.24190104007720947,
      "logits/rejected": 0.7488356828689575,
      "logps/chosen": -1.416288137435913,
      "logps/rejected": -14.760062217712402,
      "loss": 3.3408,
      "nll_loss": 3.32993221282959,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14162881672382355,
      "rewards/margins": 1.3343775272369385,
      "rewards/rejected": -1.476006269454956,
      "step": 1398
    },
    {
      "epoch": 0.8702954898911353,
      "grad_norm": 2.191300630569458,
      "learning_rate": 1.5025000000000001e-05,
      "log_odds_chosen": 8.226001739501953,
      "log_odds_ratio": -0.5000618696212769,
      "logits/chosen": 0.060526616871356964,
      "logits/rejected": 0.4570578932762146,
      "logps/chosen": -1.5750854015350342,
      "logps/rejected": -9.476434707641602,
      "loss": 2.8869,
      "nll_loss": 2.836855888366699,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15750853717327118,
      "rewards/margins": 0.7901349067687988,
      "rewards/rejected": -0.9476434588432312,
      "step": 1399
    },
    {
      "epoch": 0.8709175738724728,
      "grad_norm": 0.4874240756034851,
      "learning_rate": 1.5e-05,
      "log_odds_chosen": 10.55546760559082,
      "log_odds_ratio": -0.17145481705665588,
      "logits/chosen": 0.24024158716201782,
      "logits/rejected": 1.0146418809890747,
      "logps/chosen": -1.0970916748046875,
      "logps/rejected": -11.358686447143555,
      "loss": 2.9935,
      "nll_loss": 2.9763572216033936,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10970918089151382,
      "rewards/margins": 1.026159405708313,
      "rewards/rejected": -1.1358685493469238,
      "step": 1400
    },
    {
      "epoch": 0.8715396578538103,
      "grad_norm": 0.6221321225166321,
      "learning_rate": 1.4975e-05,
      "log_odds_chosen": 10.754582405090332,
      "log_odds_ratio": -0.47946444153785706,
      "logits/chosen": 0.2002388834953308,
      "logits/rejected": 0.7997665405273438,
      "logps/chosen": -1.3535938262939453,
      "logps/rejected": -11.846307754516602,
      "loss": 3.3366,
      "nll_loss": 3.288651943206787,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13535939157009125,
      "rewards/margins": 1.0492714643478394,
      "rewards/rejected": -1.1846308708190918,
      "step": 1401
    },
    {
      "epoch": 0.8721617418351477,
      "grad_norm": 0.5414087176322937,
      "learning_rate": 1.4950000000000001e-05,
      "log_odds_chosen": 12.587039947509766,
      "log_odds_ratio": -0.2860456109046936,
      "logits/chosen": 0.10869123786687851,
      "logits/rejected": 0.6310420036315918,
      "logps/chosen": -1.1774928569793701,
      "logps/rejected": -13.448193550109863,
      "loss": 2.6884,
      "nll_loss": 2.659811496734619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11774928122758865,
      "rewards/margins": 1.2270700931549072,
      "rewards/rejected": -1.3448193073272705,
      "step": 1402
    },
    {
      "epoch": 0.8727838258164852,
      "grad_norm": 0.6045001149177551,
      "learning_rate": 1.4925e-05,
      "log_odds_chosen": 9.989900588989258,
      "log_odds_ratio": -0.20460441708564758,
      "logits/chosen": 0.31068581342697144,
      "logits/rejected": 0.7650317549705505,
      "logps/chosen": -1.0151677131652832,
      "logps/rejected": -10.629185676574707,
      "loss": 3.2557,
      "nll_loss": 3.2352700233459473,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10151677578687668,
      "rewards/margins": 0.9614019393920898,
      "rewards/rejected": -1.0629186630249023,
      "step": 1403
    },
    {
      "epoch": 0.8734059097978227,
      "grad_norm": 0.5089529752731323,
      "learning_rate": 1.49e-05,
      "log_odds_chosen": 8.479745864868164,
      "log_odds_ratio": -0.24054519832134247,
      "logits/chosen": 0.08574174344539642,
      "logits/rejected": 0.42926856875419617,
      "logps/chosen": -0.8743036985397339,
      "logps/rejected": -8.764272689819336,
      "loss": 2.8783,
      "nll_loss": 2.8542098999023438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08743037283420563,
      "rewards/margins": 0.7889969348907471,
      "rewards/rejected": -0.8764272928237915,
      "step": 1404
    },
    {
      "epoch": 0.8740279937791602,
      "grad_norm": 0.4408000707626343,
      "learning_rate": 1.4875e-05,
      "log_odds_chosen": 15.424028396606445,
      "log_odds_ratio": -0.1469992995262146,
      "logits/chosen": 0.41973283886909485,
      "logits/rejected": 1.132185935974121,
      "logps/chosen": -1.2691795825958252,
      "logps/rejected": -16.375694274902344,
      "loss": 3.9522,
      "nll_loss": 3.937525749206543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12691795825958252,
      "rewards/margins": 1.5106515884399414,
      "rewards/rejected": -1.637569546699524,
      "step": 1405
    },
    {
      "epoch": 0.8746500777604976,
      "grad_norm": 0.5824654698371887,
      "learning_rate": 1.485e-05,
      "log_odds_chosen": 10.513412475585938,
      "log_odds_ratio": -0.2729822099208832,
      "logits/chosen": 0.22236670553684235,
      "logits/rejected": 1.3081042766571045,
      "logps/chosen": -1.083417296409607,
      "logps/rejected": -11.281618118286133,
      "loss": 2.7582,
      "nll_loss": 2.730898141860962,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10834173113107681,
      "rewards/margins": 1.0198200941085815,
      "rewards/rejected": -1.1281617879867554,
      "step": 1406
    },
    {
      "epoch": 0.8752721617418352,
      "grad_norm": 15.712387084960938,
      "learning_rate": 1.4825e-05,
      "log_odds_chosen": 12.111908912658691,
      "log_odds_ratio": -0.07724400609731674,
      "logits/chosen": 0.30688637495040894,
      "logits/rejected": 1.0119655132293701,
      "logps/chosen": -1.4687293767929077,
      "logps/rejected": -13.193370819091797,
      "loss": 3.3589,
      "nll_loss": 3.3511881828308105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14687293767929077,
      "rewards/margins": 1.1724642515182495,
      "rewards/rejected": -1.3193371295928955,
      "step": 1407
    },
    {
      "epoch": 0.8758942457231726,
      "grad_norm": 0.5521935820579529,
      "learning_rate": 1.48e-05,
      "log_odds_chosen": 17.45698356628418,
      "log_odds_ratio": -0.01309966016560793,
      "logits/chosen": 0.3117651343345642,
      "logits/rejected": 0.9942529201507568,
      "logps/chosen": -0.940118670463562,
      "logps/rejected": -17.889507293701172,
      "loss": 2.691,
      "nll_loss": 2.689737319946289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09401187300682068,
      "rewards/margins": 1.6949388980865479,
      "rewards/rejected": -1.7889506816864014,
      "step": 1408
    },
    {
      "epoch": 0.8765163297045101,
      "grad_norm": 0.42602846026420593,
      "learning_rate": 1.4775e-05,
      "log_odds_chosen": 16.168874740600586,
      "log_odds_ratio": -0.10667488723993301,
      "logits/chosen": 0.25878027081489563,
      "logits/rejected": 1.3639453649520874,
      "logps/chosen": -1.0515450239181519,
      "logps/rejected": -16.774581909179688,
      "loss": 2.7998,
      "nll_loss": 2.7891736030578613,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10515449941158295,
      "rewards/margins": 1.5723037719726562,
      "rewards/rejected": -1.67745840549469,
      "step": 1409
    },
    {
      "epoch": 0.8771384136858476,
      "grad_norm": 0.4654007852077484,
      "learning_rate": 1.475e-05,
      "log_odds_chosen": 13.330875396728516,
      "log_odds_ratio": -0.21977800130844116,
      "logits/chosen": 0.4110845923423767,
      "logits/rejected": 0.8825340867042542,
      "logps/chosen": -0.8441970348358154,
      "logps/rejected": -13.64559555053711,
      "loss": 3.4113,
      "nll_loss": 3.3893280029296875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08441969752311707,
      "rewards/margins": 1.2801399230957031,
      "rewards/rejected": -1.3645596504211426,
      "step": 1410
    },
    {
      "epoch": 0.8777604976671851,
      "grad_norm": 0.8660902380943298,
      "learning_rate": 1.4725e-05,
      "log_odds_chosen": 14.066307067871094,
      "log_odds_ratio": -0.031187426298856735,
      "logits/chosen": 0.20439361035823822,
      "logits/rejected": 1.059504508972168,
      "logps/chosen": -1.189511775970459,
      "logps/rejected": -14.74959659576416,
      "loss": 2.3309,
      "nll_loss": 2.327756404876709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11895118653774261,
      "rewards/margins": 1.356008529663086,
      "rewards/rejected": -1.4749597311019897,
      "step": 1411
    },
    {
      "epoch": 0.8783825816485226,
      "grad_norm": 0.4791586995124817,
      "learning_rate": 1.47e-05,
      "log_odds_chosen": 9.729087829589844,
      "log_odds_ratio": -0.11456810683012009,
      "logits/chosen": 0.2344791293144226,
      "logits/rejected": 0.9708684682846069,
      "logps/chosen": -1.1141091585159302,
      "logps/rejected": -10.431741714477539,
      "loss": 2.9354,
      "nll_loss": 2.9239559173583984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11141092330217361,
      "rewards/margins": 0.9317631721496582,
      "rewards/rejected": -1.043174147605896,
      "step": 1412
    },
    {
      "epoch": 0.87900466562986,
      "grad_norm": 0.4437407851219177,
      "learning_rate": 1.4675e-05,
      "log_odds_chosen": 17.249923706054688,
      "log_odds_ratio": -0.10913048684597015,
      "logits/chosen": 0.25980401039123535,
      "logits/rejected": 1.4403678178787231,
      "logps/chosen": -1.3583660125732422,
      "logps/rejected": -18.27130126953125,
      "loss": 2.9153,
      "nll_loss": 2.9043660163879395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1358366161584854,
      "rewards/margins": 1.691293716430664,
      "rewards/rejected": -1.8271300792694092,
      "step": 1413
    },
    {
      "epoch": 0.8796267496111975,
      "grad_norm": 0.47129300236701965,
      "learning_rate": 1.465e-05,
      "log_odds_chosen": 15.95350170135498,
      "log_odds_ratio": -0.07945066690444946,
      "logits/chosen": 0.4288966655731201,
      "logits/rejected": 1.349419116973877,
      "logps/chosen": -1.078054666519165,
      "logps/rejected": -16.625701904296875,
      "loss": 3.1376,
      "nll_loss": 3.129685878753662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10780548304319382,
      "rewards/margins": 1.554764747619629,
      "rewards/rejected": -1.6625701189041138,
      "step": 1414
    },
    {
      "epoch": 0.880248833592535,
      "grad_norm": 0.5274661183357239,
      "learning_rate": 1.4625e-05,
      "log_odds_chosen": 5.400740146636963,
      "log_odds_ratio": -0.28576767444610596,
      "logits/chosen": 0.2634659707546234,
      "logits/rejected": 0.4704267382621765,
      "logps/chosen": -1.1760218143463135,
      "logps/rejected": -6.290962219238281,
      "loss": 3.1959,
      "nll_loss": 3.1673409938812256,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1176021620631218,
      "rewards/margins": 0.5114940404891968,
      "rewards/rejected": -0.6290962100028992,
      "step": 1415
    },
    {
      "epoch": 0.8808709175738725,
      "grad_norm": 0.6360839009284973,
      "learning_rate": 1.4599999999999999e-05,
      "log_odds_chosen": 18.659954071044922,
      "log_odds_ratio": -0.0560433454811573,
      "logits/chosen": 0.1060413271188736,
      "logits/rejected": 1.236191749572754,
      "logps/chosen": -1.053462266921997,
      "logps/rejected": -19.213502883911133,
      "loss": 2.3897,
      "nll_loss": 2.3841118812561035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10534623265266418,
      "rewards/margins": 1.8160041570663452,
      "rewards/rejected": -1.921350359916687,
      "step": 1416
    },
    {
      "epoch": 0.8814930015552099,
      "grad_norm": 0.5016162395477295,
      "learning_rate": 1.4575e-05,
      "log_odds_chosen": 14.632455825805664,
      "log_odds_ratio": -0.18493463099002838,
      "logits/chosen": 0.29079532623291016,
      "logits/rejected": 1.2937943935394287,
      "logps/chosen": -1.1794633865356445,
      "logps/rejected": -15.487001419067383,
      "loss": 2.7321,
      "nll_loss": 2.7136242389678955,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11794634163379669,
      "rewards/margins": 1.4307537078857422,
      "rewards/rejected": -1.548699975013733,
      "step": 1417
    },
    {
      "epoch": 0.8821150855365474,
      "grad_norm": 0.47031769156455994,
      "learning_rate": 1.455e-05,
      "log_odds_chosen": 15.577923774719238,
      "log_odds_ratio": -0.004490715451538563,
      "logits/chosen": 0.2111625373363495,
      "logits/rejected": 1.3336447477340698,
      "logps/chosen": -1.0458276271820068,
      "logps/rejected": -16.042156219482422,
      "loss": 2.5581,
      "nll_loss": 2.557619333267212,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1045827716588974,
      "rewards/margins": 1.4996328353881836,
      "rewards/rejected": -1.6042156219482422,
      "step": 1418
    },
    {
      "epoch": 0.882737169517885,
      "grad_norm": 0.6436307430267334,
      "learning_rate": 1.4524999999999999e-05,
      "log_odds_chosen": 6.930571556091309,
      "log_odds_ratio": -0.13519811630249023,
      "logits/chosen": 0.1770489364862442,
      "logits/rejected": 0.7756469249725342,
      "logps/chosen": -1.2851059436798096,
      "logps/rejected": -7.893743991851807,
      "loss": 2.6715,
      "nll_loss": 2.6580209732055664,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12851059436798096,
      "rewards/margins": 0.6608638763427734,
      "rewards/rejected": -0.7893744111061096,
      "step": 1419
    },
    {
      "epoch": 0.8833592534992224,
      "grad_norm": 0.7085882425308228,
      "learning_rate": 1.45e-05,
      "log_odds_chosen": 14.094657897949219,
      "log_odds_ratio": -0.16490913927555084,
      "logits/chosen": 0.2271670699119568,
      "logits/rejected": 1.1130411624908447,
      "logps/chosen": -1.4116222858428955,
      "logps/rejected": -15.165432929992676,
      "loss": 2.8172,
      "nll_loss": 2.8007044792175293,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1411622315645218,
      "rewards/margins": 1.3753809928894043,
      "rewards/rejected": -1.5165432691574097,
      "step": 1420
    },
    {
      "epoch": 0.8839813374805598,
      "grad_norm": 0.48786014318466187,
      "learning_rate": 1.4475e-05,
      "log_odds_chosen": 13.460868835449219,
      "log_odds_ratio": -0.013222498819231987,
      "logits/chosen": 0.34067630767822266,
      "logits/rejected": 1.1033310890197754,
      "logps/chosen": -1.1284339427947998,
      "logps/rejected": -14.182600975036621,
      "loss": 3.205,
      "nll_loss": 3.203630208969116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11284340173006058,
      "rewards/margins": 1.3054168224334717,
      "rewards/rejected": -1.418260097503662,
      "step": 1421
    },
    {
      "epoch": 0.8846034214618974,
      "grad_norm": 0.5590962767601013,
      "learning_rate": 1.4449999999999999e-05,
      "log_odds_chosen": 6.180453300476074,
      "log_odds_ratio": -0.3521609902381897,
      "logits/chosen": 0.30372947454452515,
      "logits/rejected": 0.859440803527832,
      "logps/chosen": -1.206611156463623,
      "logps/rejected": -7.186957359313965,
      "loss": 2.8295,
      "nll_loss": 2.7942943572998047,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12066111713647842,
      "rewards/margins": 0.5980346202850342,
      "rewards/rejected": -0.7186957597732544,
      "step": 1422
    },
    {
      "epoch": 0.8852255054432349,
      "grad_norm": 0.5221461057662964,
      "learning_rate": 1.4425e-05,
      "log_odds_chosen": 10.76657485961914,
      "log_odds_ratio": -0.20132054388523102,
      "logits/chosen": 0.19903302192687988,
      "logits/rejected": 0.5170689225196838,
      "logps/chosen": -1.120729923248291,
      "logps/rejected": -11.55020523071289,
      "loss": 3.1357,
      "nll_loss": 3.1155667304992676,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11207298934459686,
      "rewards/margins": 1.0429476499557495,
      "rewards/rejected": -1.1550207138061523,
      "step": 1423
    },
    {
      "epoch": 0.8858475894245723,
      "grad_norm": 0.5751407742500305,
      "learning_rate": 1.44e-05,
      "log_odds_chosen": 15.834976196289062,
      "log_odds_ratio": -0.13728603720664978,
      "logits/chosen": 0.3189513683319092,
      "logits/rejected": 1.3567453622817993,
      "logps/chosen": -1.0502710342407227,
      "logps/rejected": -16.521549224853516,
      "loss": 3.1854,
      "nll_loss": 3.1716909408569336,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10502710938453674,
      "rewards/margins": 1.5471278429031372,
      "rewards/rejected": -1.652154803276062,
      "step": 1424
    },
    {
      "epoch": 0.8864696734059098,
      "grad_norm": 0.5363021492958069,
      "learning_rate": 1.4374999999999999e-05,
      "log_odds_chosen": 19.794139862060547,
      "log_odds_ratio": -0.0894283652305603,
      "logits/chosen": 0.1790408045053482,
      "logits/rejected": 1.2223554849624634,
      "logps/chosen": -1.062679648399353,
      "logps/rejected": -20.486034393310547,
      "loss": 2.3991,
      "nll_loss": 2.390169382095337,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10626795887947083,
      "rewards/margins": 1.9423353672027588,
      "rewards/rejected": -2.0486035346984863,
      "step": 1425
    },
    {
      "epoch": 0.8870917573872473,
      "grad_norm": 0.4193936288356781,
      "learning_rate": 1.435e-05,
      "log_odds_chosen": 19.63629913330078,
      "log_odds_ratio": -0.014150548726320267,
      "logits/chosen": 0.20590832829475403,
      "logits/rejected": 1.3159328699111938,
      "logps/chosen": -1.1567211151123047,
      "logps/rejected": -20.181039810180664,
      "loss": 3.0019,
      "nll_loss": 3.0004405975341797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11567211151123047,
      "rewards/margins": 1.9024319648742676,
      "rewards/rejected": -2.018104076385498,
      "step": 1426
    },
    {
      "epoch": 0.8877138413685848,
      "grad_norm": 3.831439256668091,
      "learning_rate": 1.4325e-05,
      "log_odds_chosen": 7.438900470733643,
      "log_odds_ratio": -0.2074524164199829,
      "logits/chosen": 0.35610508918762207,
      "logits/rejected": 0.6404019594192505,
      "logps/chosen": -0.8985580205917358,
      "logps/rejected": -7.916787624359131,
      "loss": 3.1508,
      "nll_loss": 3.130077838897705,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08985579758882523,
      "rewards/margins": 0.7018229961395264,
      "rewards/rejected": -0.791678786277771,
      "step": 1427
    },
    {
      "epoch": 0.8883359253499222,
      "grad_norm": 1.092878818511963,
      "learning_rate": 1.43e-05,
      "log_odds_chosen": 18.112590789794922,
      "log_odds_ratio": -0.06469640880823135,
      "logits/chosen": 0.5067091584205627,
      "logits/rejected": 1.4463820457458496,
      "logps/chosen": -1.6346746683120728,
      "logps/rejected": -19.401138305664062,
      "loss": 3.9286,
      "nll_loss": 3.922130584716797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16346746683120728,
      "rewards/margins": 1.776646614074707,
      "rewards/rejected": -1.9401140213012695,
      "step": 1428
    },
    {
      "epoch": 0.8889580093312597,
      "grad_norm": 0.40182310342788696,
      "learning_rate": 1.4275e-05,
      "log_odds_chosen": 17.829360961914062,
      "log_odds_ratio": -4.7570942115271464e-05,
      "logits/chosen": 0.4751187562942505,
      "logits/rejected": 1.7462925910949707,
      "logps/chosen": -1.5762360095977783,
      "logps/rejected": -19.148487091064453,
      "loss": 3.6646,
      "nll_loss": 3.6646087169647217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15762358903884888,
      "rewards/margins": 1.7572250366210938,
      "rewards/rejected": -1.9148486852645874,
      "step": 1429
    },
    {
      "epoch": 0.8895800933125972,
      "grad_norm": 0.4239805340766907,
      "learning_rate": 1.4249999999999999e-05,
      "log_odds_chosen": 20.32266616821289,
      "log_odds_ratio": -0.15288175642490387,
      "logits/chosen": 0.2174326777458191,
      "logits/rejected": 1.30299711227417,
      "logps/chosen": -1.0801613330841064,
      "logps/rejected": -21.01744270324707,
      "loss": 2.8315,
      "nll_loss": 2.8161635398864746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10801612585783005,
      "rewards/margins": 1.9937282800674438,
      "rewards/rejected": -2.1017441749572754,
      "step": 1430
    },
    {
      "epoch": 0.8902021772939347,
      "grad_norm": 0.4994823634624481,
      "learning_rate": 1.4225e-05,
      "log_odds_chosen": 15.963156700134277,
      "log_odds_ratio": -0.03521762415766716,
      "logits/chosen": 0.191191628575325,
      "logits/rejected": 1.2609500885009766,
      "logps/chosen": -1.1631869077682495,
      "logps/rejected": -16.678741455078125,
      "loss": 2.6632,
      "nll_loss": 2.659717559814453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11631868779659271,
      "rewards/margins": 1.5515555143356323,
      "rewards/rejected": -1.6678742170333862,
      "step": 1431
    },
    {
      "epoch": 0.8908242612752721,
      "grad_norm": 0.5873495936393738,
      "learning_rate": 1.42e-05,
      "log_odds_chosen": 14.923879623413086,
      "log_odds_ratio": -0.00032400363124907017,
      "logits/chosen": 0.14456267654895782,
      "logits/rejected": 0.8725841045379639,
      "logps/chosen": -1.1998927593231201,
      "logps/rejected": -15.70447826385498,
      "loss": 2.725,
      "nll_loss": 2.7249717712402344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11998927593231201,
      "rewards/margins": 1.4504585266113281,
      "rewards/rejected": -1.5704479217529297,
      "step": 1432
    },
    {
      "epoch": 0.8914463452566096,
      "grad_norm": 0.6210421919822693,
      "learning_rate": 1.4174999999999999e-05,
      "log_odds_chosen": 11.426855087280273,
      "log_odds_ratio": -0.1443122774362564,
      "logits/chosen": 0.34937727451324463,
      "logits/rejected": 1.107904076576233,
      "logps/chosen": -1.5372898578643799,
      "logps/rejected": -12.699395179748535,
      "loss": 3.2649,
      "nll_loss": 3.2504353523254395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15372899174690247,
      "rewards/margins": 1.1162105798721313,
      "rewards/rejected": -1.269939661026001,
      "step": 1433
    },
    {
      "epoch": 0.8920684292379472,
      "grad_norm": 0.6488243937492371,
      "learning_rate": 1.415e-05,
      "log_odds_chosen": 11.816390037536621,
      "log_odds_ratio": -0.08364453911781311,
      "logits/chosen": 0.2236700803041458,
      "logits/rejected": 1.4047526121139526,
      "logps/chosen": -1.4130889177322388,
      "logps/rejected": -12.905828475952148,
      "loss": 2.6891,
      "nll_loss": 2.680753469467163,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14130890369415283,
      "rewards/margins": 1.1492741107940674,
      "rewards/rejected": -1.2905830144882202,
      "step": 1434
    },
    {
      "epoch": 0.8926905132192846,
      "grad_norm": 7.441209316253662,
      "learning_rate": 1.4125e-05,
      "log_odds_chosen": 9.631667137145996,
      "log_odds_ratio": -0.13277006149291992,
      "logits/chosen": 0.20559415221214294,
      "logits/rejected": 0.3517681062221527,
      "logps/chosen": -1.7672669887542725,
      "logps/rejected": -10.936025619506836,
      "loss": 2.9401,
      "nll_loss": 2.9268393516540527,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17672669887542725,
      "rewards/margins": 0.9168758988380432,
      "rewards/rejected": -1.0936025381088257,
      "step": 1435
    },
    {
      "epoch": 0.8933125972006221,
      "grad_norm": 0.5252163410186768,
      "learning_rate": 1.4099999999999999e-05,
      "log_odds_chosen": 18.1024112701416,
      "log_odds_ratio": -0.03475270792841911,
      "logits/chosen": 0.1778755635023117,
      "logits/rejected": 0.9054905772209167,
      "logps/chosen": -1.1439323425292969,
      "logps/rejected": -18.793231964111328,
      "loss": 2.8542,
      "nll_loss": 2.8507204055786133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11439324170351028,
      "rewards/margins": 1.764930248260498,
      "rewards/rejected": -1.8793234825134277,
      "step": 1436
    },
    {
      "epoch": 0.8939346811819595,
      "grad_norm": 0.4643388092517853,
      "learning_rate": 1.4075e-05,
      "log_odds_chosen": 11.433347702026367,
      "log_odds_ratio": -0.13411425054073334,
      "logits/chosen": 0.26451680064201355,
      "logits/rejected": 1.064178705215454,
      "logps/chosen": -0.990769624710083,
      "logps/rejected": -11.884706497192383,
      "loss": 2.7966,
      "nll_loss": 2.7832188606262207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09907696396112442,
      "rewards/margins": 1.0893937349319458,
      "rewards/rejected": -1.188470721244812,
      "step": 1437
    },
    {
      "epoch": 0.8945567651632971,
      "grad_norm": 0.5520986318588257,
      "learning_rate": 1.4050000000000003e-05,
      "log_odds_chosen": 10.105766296386719,
      "log_odds_ratio": -0.1801670640707016,
      "logits/chosen": 0.12006427347660065,
      "logits/rejected": 0.7376315593719482,
      "logps/chosen": -1.650545597076416,
      "logps/rejected": -11.550055503845215,
      "loss": 2.5541,
      "nll_loss": 2.5360946655273438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1650545597076416,
      "rewards/margins": 0.9899511337280273,
      "rewards/rejected": -1.155005693435669,
      "step": 1438
    },
    {
      "epoch": 0.8951788491446345,
      "grad_norm": 0.5198318362236023,
      "learning_rate": 1.4025000000000002e-05,
      "log_odds_chosen": 7.641714096069336,
      "log_odds_ratio": -0.25751960277557373,
      "logits/chosen": 0.29090097546577454,
      "logits/rejected": 0.6141082048416138,
      "logps/chosen": -1.2502827644348145,
      "logps/rejected": -8.630484580993652,
      "loss": 3.1904,
      "nll_loss": 3.164672374725342,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12502828240394592,
      "rewards/margins": 0.738020122051239,
      "rewards/rejected": -0.8630484342575073,
      "step": 1439
    },
    {
      "epoch": 0.895800933125972,
      "grad_norm": 0.4290095865726471,
      "learning_rate": 1.4000000000000001e-05,
      "log_odds_chosen": 14.732014656066895,
      "log_odds_ratio": -0.08160438388586044,
      "logits/chosen": 0.12874333560466766,
      "logits/rejected": 0.9056077003479004,
      "logps/chosen": -1.228428602218628,
      "logps/rejected": -15.536664009094238,
      "loss": 3.2002,
      "nll_loss": 3.192028045654297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12284286320209503,
      "rewards/margins": 1.4308236837387085,
      "rewards/rejected": -1.5536664724349976,
      "step": 1440
    },
    {
      "epoch": 0.8964230171073095,
      "grad_norm": 0.30635496973991394,
      "learning_rate": 1.3975000000000003e-05,
      "log_odds_chosen": 14.656373023986816,
      "log_odds_ratio": -0.0002228428202215582,
      "logits/chosen": 0.23041866719722748,
      "logits/rejected": 1.2936317920684814,
      "logps/chosen": -0.9796774387359619,
      "logps/rejected": -14.984707832336426,
      "loss": 2.9882,
      "nll_loss": 2.98820161819458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0979677364230156,
      "rewards/margins": 1.400503158569336,
      "rewards/rejected": -1.4984709024429321,
      "step": 1441
    },
    {
      "epoch": 0.897045101088647,
      "grad_norm": 0.41320523619651794,
      "learning_rate": 1.3950000000000002e-05,
      "log_odds_chosen": 10.99651050567627,
      "log_odds_ratio": -0.32272082567214966,
      "logits/chosen": 0.19428414106369019,
      "logits/rejected": 0.7030611634254456,
      "logps/chosen": -1.1404513120651245,
      "logps/rejected": -11.859764099121094,
      "loss": 3.2682,
      "nll_loss": 3.235926628112793,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11404513567686081,
      "rewards/margins": 1.0719313621520996,
      "rewards/rejected": -1.1859763860702515,
      "step": 1442
    },
    {
      "epoch": 0.8976671850699844,
      "grad_norm": 0.7050619721412659,
      "learning_rate": 1.3925000000000001e-05,
      "log_odds_chosen": 11.484809875488281,
      "log_odds_ratio": -0.15941309928894043,
      "logits/chosen": 0.35777807235717773,
      "logits/rejected": 1.1093180179595947,
      "logps/chosen": -1.2923328876495361,
      "logps/rejected": -12.462808609008789,
      "loss": 3.3092,
      "nll_loss": 3.293220043182373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12923328578472137,
      "rewards/margins": 1.1170475482940674,
      "rewards/rejected": -1.2462809085845947,
      "step": 1443
    },
    {
      "epoch": 0.8982892690513219,
      "grad_norm": 0.412223219871521,
      "learning_rate": 1.3900000000000002e-05,
      "log_odds_chosen": 18.409934997558594,
      "log_odds_ratio": -0.0002995289978571236,
      "logits/chosen": 0.2523375451564789,
      "logits/rejected": 1.2306113243103027,
      "logps/chosen": -1.0025357007980347,
      "logps/rejected": -18.848125457763672,
      "loss": 3.369,
      "nll_loss": 3.368983745574951,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10025357455015182,
      "rewards/margins": 1.7845587730407715,
      "rewards/rejected": -1.884812355041504,
      "step": 1444
    },
    {
      "epoch": 0.8989113530326595,
      "grad_norm": 0.3040986955165863,
      "learning_rate": 1.3875000000000002e-05,
      "log_odds_chosen": 17.37039566040039,
      "log_odds_ratio": -1.7285432249991572e-06,
      "logits/chosen": 0.5640264749526978,
      "logits/rejected": 1.9091010093688965,
      "logps/chosen": -0.8681442737579346,
      "logps/rejected": -17.551462173461914,
      "loss": 4.1642,
      "nll_loss": 4.164191246032715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08681442588567734,
      "rewards/margins": 1.6683318614959717,
      "rewards/rejected": -1.7551462650299072,
      "step": 1445
    },
    {
      "epoch": 0.8995334370139969,
      "grad_norm": 0.7804985046386719,
      "learning_rate": 1.3850000000000001e-05,
      "log_odds_chosen": 12.528520584106445,
      "log_odds_ratio": -0.036599528044462204,
      "logits/chosen": 0.31422773003578186,
      "logits/rejected": 1.119031310081482,
      "logps/chosen": -1.1160674095153809,
      "logps/rejected": -13.23311996459961,
      "loss": 2.9005,
      "nll_loss": 2.896796703338623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11160673946142197,
      "rewards/margins": 1.211705207824707,
      "rewards/rejected": -1.3233120441436768,
      "step": 1446
    },
    {
      "epoch": 0.9001555209953344,
      "grad_norm": 0.41449618339538574,
      "learning_rate": 1.3825000000000002e-05,
      "log_odds_chosen": 11.29874038696289,
      "log_odds_ratio": -0.19807516038417816,
      "logits/chosen": 0.07104424387216568,
      "logits/rejected": 0.4711911380290985,
      "logps/chosen": -1.0898693799972534,
      "logps/rejected": -12.04837703704834,
      "loss": 3.2759,
      "nll_loss": 3.256134271621704,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10898693650960922,
      "rewards/margins": 1.0958508253097534,
      "rewards/rejected": -1.2048377990722656,
      "step": 1447
    },
    {
      "epoch": 0.9007776049766718,
      "grad_norm": 0.42537832260131836,
      "learning_rate": 1.3800000000000002e-05,
      "log_odds_chosen": 14.00430965423584,
      "log_odds_ratio": -0.10201773792505264,
      "logits/chosen": 0.2881453037261963,
      "logits/rejected": 1.2576179504394531,
      "logps/chosen": -0.884018063545227,
      "logps/rejected": -14.224514961242676,
      "loss": 3.2637,
      "nll_loss": 3.2534520626068115,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08840180933475494,
      "rewards/margins": 1.3340497016906738,
      "rewards/rejected": -1.4224514961242676,
      "step": 1448
    },
    {
      "epoch": 0.9013996889580094,
      "grad_norm": 0.42243319749832153,
      "learning_rate": 1.3775000000000001e-05,
      "log_odds_chosen": 11.295740127563477,
      "log_odds_ratio": -0.10844042152166367,
      "logits/chosen": 0.16659203171730042,
      "logits/rejected": 0.9894870519638062,
      "logps/chosen": -1.0629875659942627,
      "logps/rejected": -11.810334205627441,
      "loss": 3.1585,
      "nll_loss": 3.1476120948791504,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10629875957965851,
      "rewards/margins": 1.0747345685958862,
      "rewards/rejected": -1.1810333728790283,
      "step": 1449
    },
    {
      "epoch": 0.9020217729393468,
      "grad_norm": 0.5875496864318848,
      "learning_rate": 1.3750000000000002e-05,
      "log_odds_chosen": 10.801142692565918,
      "log_odds_ratio": -0.17056904733181,
      "logits/chosen": 0.09089596569538116,
      "logits/rejected": 0.5544472932815552,
      "logps/chosen": -1.0837565660476685,
      "logps/rejected": -11.519688606262207,
      "loss": 2.8123,
      "nll_loss": 2.795261859893799,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1083756536245346,
      "rewards/margins": 1.043593168258667,
      "rewards/rejected": -1.1519688367843628,
      "step": 1450
    },
    {
      "epoch": 0.9026438569206843,
      "grad_norm": 0.6801373362541199,
      "learning_rate": 1.3725000000000002e-05,
      "log_odds_chosen": 11.216224670410156,
      "log_odds_ratio": -0.11024151742458344,
      "logits/chosen": 0.15584243834018707,
      "logits/rejected": 0.7682511210441589,
      "logps/chosen": -1.1805226802825928,
      "logps/rejected": -12.04312801361084,
      "loss": 3.2438,
      "nll_loss": 3.2327301502227783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11805225908756256,
      "rewards/margins": 1.0862605571746826,
      "rewards/rejected": -1.204312801361084,
      "step": 1451
    },
    {
      "epoch": 0.9032659409020217,
      "grad_norm": 1.5156279802322388,
      "learning_rate": 1.3700000000000001e-05,
      "log_odds_chosen": 11.422454833984375,
      "log_odds_ratio": -0.11768678575754166,
      "logits/chosen": 0.23056858777999878,
      "logits/rejected": 1.1006563901901245,
      "logps/chosen": -1.2227720022201538,
      "logps/rejected": -12.296416282653809,
      "loss": 3.1485,
      "nll_loss": 3.1367223262786865,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12227719277143478,
      "rewards/margins": 1.107364535331726,
      "rewards/rejected": -1.2296416759490967,
      "step": 1452
    },
    {
      "epoch": 0.9038880248833593,
      "grad_norm": 0.4438559114933014,
      "learning_rate": 1.3675000000000002e-05,
      "log_odds_chosen": 21.953859329223633,
      "log_odds_ratio": -1.4901162970204496e-08,
      "logits/chosen": 0.14678457379341125,
      "logits/rejected": 0.9667971730232239,
      "logps/chosen": -0.9780452251434326,
      "logps/rejected": -22.29878044128418,
      "loss": 2.7227,
      "nll_loss": 2.7226831912994385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09780453145503998,
      "rewards/margins": 2.132073402404785,
      "rewards/rejected": -2.2298779487609863,
      "step": 1453
    },
    {
      "epoch": 0.9045101088646967,
      "grad_norm": 2.3538661003112793,
      "learning_rate": 1.3650000000000001e-05,
      "log_odds_chosen": 12.744718551635742,
      "log_odds_ratio": -0.06637068092823029,
      "logits/chosen": 0.18035347759723663,
      "logits/rejected": 0.8537945747375488,
      "logps/chosen": -2.0617189407348633,
      "logps/rejected": -14.523418426513672,
      "loss": 3.229,
      "nll_loss": 3.2223703861236572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.206171914935112,
      "rewards/margins": 1.246169924736023,
      "rewards/rejected": -1.4523417949676514,
      "step": 1454
    },
    {
      "epoch": 0.9051321928460342,
      "grad_norm": 0.5291186571121216,
      "learning_rate": 1.3625e-05,
      "log_odds_chosen": 9.619067192077637,
      "log_odds_ratio": -0.1370728313922882,
      "logits/chosen": 0.10796351730823517,
      "logits/rejected": 0.5130314826965332,
      "logps/chosen": -0.9000706672668457,
      "logps/rejected": -9.873458862304688,
      "loss": 2.6361,
      "nll_loss": 2.622385025024414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09000707417726517,
      "rewards/margins": 0.8973388671875,
      "rewards/rejected": -0.9873458743095398,
      "step": 1455
    },
    {
      "epoch": 0.9057542768273716,
      "grad_norm": 0.5473167896270752,
      "learning_rate": 1.3600000000000002e-05,
      "log_odds_chosen": 9.86450481414795,
      "log_odds_ratio": -0.08004651963710785,
      "logits/chosen": 0.2445843517780304,
      "logits/rejected": 0.9183000326156616,
      "logps/chosen": -1.064620018005371,
      "logps/rejected": -10.451854705810547,
      "loss": 3.2033,
      "nll_loss": 3.1953063011169434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10646200180053711,
      "rewards/margins": 0.9387235641479492,
      "rewards/rejected": -1.0451855659484863,
      "step": 1456
    },
    {
      "epoch": 0.9063763608087092,
      "grad_norm": 0.4545001685619354,
      "learning_rate": 1.3575000000000001e-05,
      "log_odds_chosen": 14.232173919677734,
      "log_odds_ratio": -0.15462376177310944,
      "logits/chosen": 0.3833097517490387,
      "logits/rejected": 1.2270677089691162,
      "logps/chosen": -1.1023833751678467,
      "logps/rejected": -14.765386581420898,
      "loss": 3.3058,
      "nll_loss": 3.2903385162353516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11023833602666855,
      "rewards/margins": 1.366300344467163,
      "rewards/rejected": -1.4765386581420898,
      "step": 1457
    },
    {
      "epoch": 0.9069984447900467,
      "grad_norm": 0.41073378920555115,
      "learning_rate": 1.3550000000000002e-05,
      "log_odds_chosen": 7.300911903381348,
      "log_odds_ratio": -0.1350908875465393,
      "logits/chosen": 0.18523235619068146,
      "logits/rejected": 0.4215852618217468,
      "logps/chosen": -0.9286333918571472,
      "logps/rejected": -7.772511959075928,
      "loss": 3.1842,
      "nll_loss": 3.170691728591919,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09286333620548248,
      "rewards/margins": 0.6843879222869873,
      "rewards/rejected": -0.7772511839866638,
      "step": 1458
    },
    {
      "epoch": 0.9076205287713841,
      "grad_norm": 0.499830961227417,
      "learning_rate": 1.3525000000000002e-05,
      "log_odds_chosen": 16.171920776367188,
      "log_odds_ratio": -0.0024008448235690594,
      "logits/chosen": 0.30481821298599243,
      "logits/rejected": 1.3369604349136353,
      "logps/chosen": -0.9462791681289673,
      "logps/rejected": -16.467857360839844,
      "loss": 3.1955,
      "nll_loss": 3.19523286819458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09462791681289673,
      "rewards/margins": 1.5521578788757324,
      "rewards/rejected": -1.6467856168746948,
      "step": 1459
    },
    {
      "epoch": 0.9082426127527217,
      "grad_norm": 0.44492849707603455,
      "learning_rate": 1.3500000000000001e-05,
      "log_odds_chosen": 10.069293975830078,
      "log_odds_ratio": -0.0921773836016655,
      "logits/chosen": 0.3066140413284302,
      "logits/rejected": 0.8817154765129089,
      "logps/chosen": -1.0714751482009888,
      "logps/rejected": -10.760387420654297,
      "loss": 3.5663,
      "nll_loss": 3.5571231842041016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10714752227067947,
      "rewards/margins": 0.9688910841941833,
      "rewards/rejected": -1.0760385990142822,
      "step": 1460
    },
    {
      "epoch": 0.9088646967340591,
      "grad_norm": 0.6452760696411133,
      "learning_rate": 1.3475000000000002e-05,
      "log_odds_chosen": 12.372976303100586,
      "log_odds_ratio": -0.22037629783153534,
      "logits/chosen": 0.24817940592765808,
      "logits/rejected": 0.922889232635498,
      "logps/chosen": -1.1425799131393433,
      "logps/rejected": -13.200763702392578,
      "loss": 2.7866,
      "nll_loss": 2.764538288116455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11425799131393433,
      "rewards/margins": 1.2058184146881104,
      "rewards/rejected": -1.3200764656066895,
      "step": 1461
    },
    {
      "epoch": 0.9094867807153966,
      "grad_norm": 0.6373099684715271,
      "learning_rate": 1.3450000000000002e-05,
      "log_odds_chosen": 10.688899040222168,
      "log_odds_ratio": -0.2276204526424408,
      "logits/chosen": 0.5093823075294495,
      "logits/rejected": 1.0521379709243774,
      "logps/chosen": -1.4107290506362915,
      "logps/rejected": -11.85671615600586,
      "loss": 4.0253,
      "nll_loss": 4.002525329589844,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14107291400432587,
      "rewards/margins": 1.0445988178253174,
      "rewards/rejected": -1.1856716871261597,
      "step": 1462
    },
    {
      "epoch": 0.910108864696734,
      "grad_norm": 0.5296677947044373,
      "learning_rate": 1.3425000000000001e-05,
      "log_odds_chosen": 12.754611015319824,
      "log_odds_ratio": -0.19718533754348755,
      "logits/chosen": 0.3532048165798187,
      "logits/rejected": 1.3405009508132935,
      "logps/chosen": -1.1448729038238525,
      "logps/rejected": -13.596270561218262,
      "loss": 3.2044,
      "nll_loss": 3.1846399307250977,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11448729038238525,
      "rewards/margins": 1.2451398372650146,
      "rewards/rejected": -1.3596272468566895,
      "step": 1463
    },
    {
      "epoch": 0.9107309486780716,
      "grad_norm": 0.9786263108253479,
      "learning_rate": 1.3400000000000002e-05,
      "log_odds_chosen": 7.895408630371094,
      "log_odds_ratio": -0.18677686154842377,
      "logits/chosen": 0.09396594762802124,
      "logits/rejected": 0.8790769577026367,
      "logps/chosen": -1.0056233406066895,
      "logps/rejected": -8.473982810974121,
      "loss": 2.7747,
      "nll_loss": 2.755993366241455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10056232661008835,
      "rewards/margins": 0.7468359470367432,
      "rewards/rejected": -0.8473982810974121,
      "step": 1464
    },
    {
      "epoch": 0.911353032659409,
      "grad_norm": 2.532768726348877,
      "learning_rate": 1.3375000000000002e-05,
      "log_odds_chosen": 16.206634521484375,
      "log_odds_ratio": -0.0901508703827858,
      "logits/chosen": 0.29881221055984497,
      "logits/rejected": 1.2665278911590576,
      "logps/chosen": -1.5193629264831543,
      "logps/rejected": -17.421382904052734,
      "loss": 3.2096,
      "nll_loss": 3.200589895248413,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15193629264831543,
      "rewards/margins": 1.5902019739151,
      "rewards/rejected": -1.7421382665634155,
      "step": 1465
    },
    {
      "epoch": 0.9119751166407465,
      "grad_norm": 0.45200738310813904,
      "learning_rate": 1.3350000000000001e-05,
      "log_odds_chosen": 12.099109649658203,
      "log_odds_ratio": -0.10622071474790573,
      "logits/chosen": 0.27904149889945984,
      "logits/rejected": 1.2529817819595337,
      "logps/chosen": -1.0653222799301147,
      "logps/rejected": -12.664470672607422,
      "loss": 3.0189,
      "nll_loss": 3.0083017349243164,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10653223097324371,
      "rewards/margins": 1.1599149703979492,
      "rewards/rejected": -1.2664470672607422,
      "step": 1466
    },
    {
      "epoch": 0.9125972006220839,
      "grad_norm": 0.5577512383460999,
      "learning_rate": 1.3325000000000002e-05,
      "log_odds_chosen": 14.484386444091797,
      "log_odds_ratio": -0.007911762222647667,
      "logits/chosen": 0.1263856291770935,
      "logits/rejected": 0.8978151082992554,
      "logps/chosen": -0.9192382097244263,
      "logps/rejected": -14.804035186767578,
      "loss": 2.5146,
      "nll_loss": 2.513773202896118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09192381799221039,
      "rewards/margins": 1.3884797096252441,
      "rewards/rejected": -1.4804035425186157,
      "step": 1467
    },
    {
      "epoch": 0.9132192846034215,
      "grad_norm": 0.4941674768924713,
      "learning_rate": 1.3300000000000001e-05,
      "log_odds_chosen": 10.110100746154785,
      "log_odds_ratio": -0.03399426490068436,
      "logits/chosen": 0.22827720642089844,
      "logits/rejected": 0.8136681914329529,
      "logps/chosen": -0.9263241291046143,
      "logps/rejected": -10.246597290039062,
      "loss": 3.2691,
      "nll_loss": 3.265674352645874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09263241291046143,
      "rewards/margins": 0.932027280330658,
      "rewards/rejected": -1.0246597528457642,
      "step": 1468
    },
    {
      "epoch": 0.913841368584759,
      "grad_norm": 0.6259835362434387,
      "learning_rate": 1.3275e-05,
      "log_odds_chosen": 11.36493968963623,
      "log_odds_ratio": -0.19162523746490479,
      "logits/chosen": 0.19190458953380585,
      "logits/rejected": 0.771049976348877,
      "logps/chosen": -1.0270004272460938,
      "logps/rejected": -11.900120735168457,
      "loss": 2.7219,
      "nll_loss": 2.70273494720459,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10270003974437714,
      "rewards/margins": 1.0873119831085205,
      "rewards/rejected": -1.1900120973587036,
      "step": 1469
    },
    {
      "epoch": 0.9144634525660964,
      "grad_norm": 0.8540546298027039,
      "learning_rate": 1.3250000000000002e-05,
      "log_odds_chosen": 9.284761428833008,
      "log_odds_ratio": -0.2318931221961975,
      "logits/chosen": 0.22063320875167847,
      "logits/rejected": 0.7904773354530334,
      "logps/chosen": -1.284113883972168,
      "logps/rejected": -10.228588104248047,
      "loss": 2.8488,
      "nll_loss": 2.825575828552246,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1284113973379135,
      "rewards/margins": 0.8944474458694458,
      "rewards/rejected": -1.0228588581085205,
      "step": 1470
    },
    {
      "epoch": 0.9150855365474339,
      "grad_norm": 0.5292484760284424,
      "learning_rate": 1.3225000000000001e-05,
      "log_odds_chosen": 12.626777648925781,
      "log_odds_ratio": -0.13767273724079132,
      "logits/chosen": 0.2530801594257355,
      "logits/rejected": 1.1952565908432007,
      "logps/chosen": -0.9651281833648682,
      "logps/rejected": -13.066086769104004,
      "loss": 2.8352,
      "nll_loss": 2.821428060531616,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0965128242969513,
      "rewards/margins": 1.2100958824157715,
      "rewards/rejected": -1.3066086769104004,
      "step": 1471
    },
    {
      "epoch": 0.9157076205287714,
      "grad_norm": 0.737404465675354,
      "learning_rate": 1.32e-05,
      "log_odds_chosen": 14.638701438903809,
      "log_odds_ratio": -0.0027568021323531866,
      "logits/chosen": 0.38527604937553406,
      "logits/rejected": 1.2940354347229004,
      "logps/chosen": -1.0043842792510986,
      "logps/rejected": -15.046712875366211,
      "loss": 2.9281,
      "nll_loss": 2.9278013706207275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1004384309053421,
      "rewards/margins": 1.4042328596115112,
      "rewards/rejected": -1.5046712160110474,
      "step": 1472
    },
    {
      "epoch": 0.9163297045101089,
      "grad_norm": 0.4254592955112457,
      "learning_rate": 1.3175000000000002e-05,
      "log_odds_chosen": 18.022205352783203,
      "log_odds_ratio": -0.002449390245601535,
      "logits/chosen": 0.45590126514434814,
      "logits/rejected": 1.8061012029647827,
      "logps/chosen": -1.1624892950057983,
      "logps/rejected": -18.69601058959961,
      "loss": 3.6024,
      "nll_loss": 3.6021316051483154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11624892801046371,
      "rewards/margins": 1.753352165222168,
      "rewards/rejected": -1.8696010112762451,
      "step": 1473
    },
    {
      "epoch": 0.9169517884914463,
      "grad_norm": 0.4656004309654236,
      "learning_rate": 1.3150000000000001e-05,
      "log_odds_chosen": 11.069246292114258,
      "log_odds_ratio": -0.14720577001571655,
      "logits/chosen": 0.1729264259338379,
      "logits/rejected": 0.8423563241958618,
      "logps/chosen": -1.3754249811172485,
      "logps/rejected": -12.108327865600586,
      "loss": 2.8642,
      "nll_loss": 2.8495240211486816,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1375425010919571,
      "rewards/margins": 1.073290228843689,
      "rewards/rejected": -1.2108327150344849,
      "step": 1474
    },
    {
      "epoch": 0.9175738724727839,
      "grad_norm": 0.8528266549110413,
      "learning_rate": 1.3125e-05,
      "log_odds_chosen": 10.443950653076172,
      "log_odds_ratio": -0.059229776263237,
      "logits/chosen": 0.3430590033531189,
      "logits/rejected": 1.0942491292953491,
      "logps/chosen": -1.181295394897461,
      "logps/rejected": -11.18059253692627,
      "loss": 3.3527,
      "nll_loss": 3.3467843532562256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11812953650951385,
      "rewards/margins": 0.9999297261238098,
      "rewards/rejected": -1.1180591583251953,
      "step": 1475
    },
    {
      "epoch": 0.9181959564541213,
      "grad_norm": 0.748715877532959,
      "learning_rate": 1.3100000000000002e-05,
      "log_odds_chosen": 9.176325798034668,
      "log_odds_ratio": -0.19434534013271332,
      "logits/chosen": 0.27273157238960266,
      "logits/rejected": 0.9127519130706787,
      "logps/chosen": -1.1760427951812744,
      "logps/rejected": -9.9922513961792,
      "loss": 2.9926,
      "nll_loss": 2.973123550415039,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11760427802801132,
      "rewards/margins": 0.8816208839416504,
      "rewards/rejected": -0.9992251396179199,
      "step": 1476
    },
    {
      "epoch": 0.9188180404354588,
      "grad_norm": 0.4457939863204956,
      "learning_rate": 1.3075000000000001e-05,
      "log_odds_chosen": 17.727825164794922,
      "log_odds_ratio": -0.0003742209810297936,
      "logits/chosen": 0.489930659532547,
      "logits/rejected": 1.601247787475586,
      "logps/chosen": -1.2831439971923828,
      "logps/rejected": -18.507240295410156,
      "loss": 3.5756,
      "nll_loss": 3.575566530227661,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12831439077854156,
      "rewards/margins": 1.722409725189209,
      "rewards/rejected": -1.8507239818572998,
      "step": 1477
    },
    {
      "epoch": 0.9194401244167962,
      "grad_norm": 0.5372123718261719,
      "learning_rate": 1.305e-05,
      "log_odds_chosen": 5.822672367095947,
      "log_odds_ratio": -0.24347680807113647,
      "logits/chosen": 0.2464301735162735,
      "logits/rejected": 0.44997262954711914,
      "logps/chosen": -1.2049652338027954,
      "logps/rejected": -6.674422740936279,
      "loss": 3.2726,
      "nll_loss": 3.2482857704162598,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12049652636051178,
      "rewards/margins": 0.5469457507133484,
      "rewards/rejected": -0.6674423217773438,
      "step": 1478
    },
    {
      "epoch": 0.9200622083981338,
      "grad_norm": 0.4896005094051361,
      "learning_rate": 1.3025000000000002e-05,
      "log_odds_chosen": 12.5404052734375,
      "log_odds_ratio": -0.08044345676898956,
      "logits/chosen": 0.38424739241600037,
      "logits/rejected": 1.3379318714141846,
      "logps/chosen": -0.9839893579483032,
      "logps/rejected": -13.034258842468262,
      "loss": 3.1035,
      "nll_loss": 3.0955049991607666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09839893877506256,
      "rewards/margins": 1.2050269842147827,
      "rewards/rejected": -1.3034260272979736,
      "step": 1479
    },
    {
      "epoch": 0.9206842923794712,
      "grad_norm": 0.5063160061836243,
      "learning_rate": 1.3000000000000001e-05,
      "log_odds_chosen": 9.515791893005371,
      "log_odds_ratio": -0.16696563363075256,
      "logits/chosen": 0.36880922317504883,
      "logits/rejected": 0.9227983355522156,
      "logps/chosen": -1.0066584348678589,
      "logps/rejected": -10.174430847167969,
      "loss": 3.3245,
      "nll_loss": 3.3077735900878906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10066584497690201,
      "rewards/margins": 0.9167773127555847,
      "rewards/rejected": -1.017443060874939,
      "step": 1480
    },
    {
      "epoch": 0.9213063763608087,
      "grad_norm": 0.4576911926269531,
      "learning_rate": 1.2975e-05,
      "log_odds_chosen": 12.466102600097656,
      "log_odds_ratio": -0.18402725458145142,
      "logits/chosen": 0.13083387911319733,
      "logits/rejected": 0.53689044713974,
      "logps/chosen": -1.0055042505264282,
      "logps/rejected": -13.049692153930664,
      "loss": 2.6695,
      "nll_loss": 2.6510913372039795,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10055042803287506,
      "rewards/margins": 1.2044188976287842,
      "rewards/rejected": -1.304969310760498,
      "step": 1481
    },
    {
      "epoch": 0.9219284603421461,
      "grad_norm": 0.46238452196121216,
      "learning_rate": 1.2950000000000001e-05,
      "log_odds_chosen": 15.337545394897461,
      "log_odds_ratio": -0.03438110649585724,
      "logits/chosen": 0.4146786034107208,
      "logits/rejected": 1.442423701286316,
      "logps/chosen": -1.1937940120697021,
      "logps/rejected": -16.170305252075195,
      "loss": 3.3299,
      "nll_loss": 3.3265042304992676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11937940120697021,
      "rewards/margins": 1.4976511001586914,
      "rewards/rejected": -1.6170305013656616,
      "step": 1482
    },
    {
      "epoch": 0.9225505443234837,
      "grad_norm": 1.6496787071228027,
      "learning_rate": 1.2925e-05,
      "log_odds_chosen": 17.128173828125,
      "log_odds_ratio": -8.068131137406453e-05,
      "logits/chosen": 0.49006718397140503,
      "logits/rejected": 1.4875874519348145,
      "logps/chosen": -1.2655510902404785,
      "logps/rejected": -18.002466201782227,
      "loss": 3.4641,
      "nll_loss": 3.46408748626709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12655511498451233,
      "rewards/margins": 1.6736915111541748,
      "rewards/rejected": -1.8002467155456543,
      "step": 1483
    },
    {
      "epoch": 0.9231726283048212,
      "grad_norm": 0.839210569858551,
      "learning_rate": 1.29e-05,
      "log_odds_chosen": 16.212020874023438,
      "log_odds_ratio": -0.006270325277000666,
      "logits/chosen": 0.256135493516922,
      "logits/rejected": 1.3664355278015137,
      "logps/chosen": -1.0374066829681396,
      "logps/rejected": -16.69953155517578,
      "loss": 2.8523,
      "nll_loss": 2.851687431335449,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10374066978693008,
      "rewards/margins": 1.5662124156951904,
      "rewards/rejected": -1.6699531078338623,
      "step": 1484
    },
    {
      "epoch": 0.9237947122861586,
      "grad_norm": 0.5375116467475891,
      "learning_rate": 1.2875000000000001e-05,
      "log_odds_chosen": 15.11935043334961,
      "log_odds_ratio": -0.010710510425269604,
      "logits/chosen": 0.23992419242858887,
      "logits/rejected": 1.610540509223938,
      "logps/chosen": -1.131101131439209,
      "logps/rejected": -15.822179794311523,
      "loss": 2.3641,
      "nll_loss": 2.3630285263061523,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11311011761426926,
      "rewards/margins": 1.469107985496521,
      "rewards/rejected": -1.582218050956726,
      "step": 1485
    },
    {
      "epoch": 0.9244167962674961,
      "grad_norm": 0.6357943415641785,
      "learning_rate": 1.285e-05,
      "log_odds_chosen": 5.553439140319824,
      "log_odds_ratio": -0.39662641286849976,
      "logits/chosen": 0.21328821778297424,
      "logits/rejected": 0.5546770095825195,
      "logps/chosen": -1.0445785522460938,
      "logps/rejected": -6.2532267570495605,
      "loss": 2.487,
      "nll_loss": 2.4473471641540527,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10445785522460938,
      "rewards/margins": 0.5208648443222046,
      "rewards/rejected": -0.625322699546814,
      "step": 1486
    },
    {
      "epoch": 0.9250388802488336,
      "grad_norm": 1.35960054397583,
      "learning_rate": 1.2825000000000002e-05,
      "log_odds_chosen": 13.23503303527832,
      "log_odds_ratio": -0.026235153898596764,
      "logits/chosen": 0.04117313399910927,
      "logits/rejected": 0.4238998293876648,
      "logps/chosen": -1.3150159120559692,
      "logps/rejected": -14.009586334228516,
      "loss": 2.9444,
      "nll_loss": 2.9417426586151123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13150160014629364,
      "rewards/margins": 1.2694569826126099,
      "rewards/rejected": -1.4009586572647095,
      "step": 1487
    },
    {
      "epoch": 0.9256609642301711,
      "grad_norm": 0.8810969591140747,
      "learning_rate": 1.2800000000000001e-05,
      "log_odds_chosen": 13.211524963378906,
      "log_odds_ratio": -0.08693347871303558,
      "logits/chosen": 0.29597651958465576,
      "logits/rejected": 1.2714580297470093,
      "logps/chosen": -1.328726053237915,
      "logps/rejected": -14.09039306640625,
      "loss": 3.231,
      "nll_loss": 3.2223141193389893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1328725963830948,
      "rewards/margins": 1.2761669158935547,
      "rewards/rejected": -1.4090393781661987,
      "step": 1488
    },
    {
      "epoch": 0.9262830482115085,
      "grad_norm": 0.6031063199043274,
      "learning_rate": 1.2775e-05,
      "log_odds_chosen": 13.652146339416504,
      "log_odds_ratio": -0.0523035004734993,
      "logits/chosen": 0.3231468200683594,
      "logits/rejected": 0.9991359710693359,
      "logps/chosen": -1.0230048894882202,
      "logps/rejected": -14.191951751708984,
      "loss": 3.2951,
      "nll_loss": 3.2898614406585693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1023004874587059,
      "rewards/margins": 1.3168946504592896,
      "rewards/rejected": -1.4191951751708984,
      "step": 1489
    },
    {
      "epoch": 0.926905132192846,
      "grad_norm": 0.627905547618866,
      "learning_rate": 1.2750000000000002e-05,
      "log_odds_chosen": 10.043725967407227,
      "log_odds_ratio": -0.15319031476974487,
      "logits/chosen": 0.11043369770050049,
      "logits/rejected": 0.7919861674308777,
      "logps/chosen": -1.0833916664123535,
      "logps/rejected": -10.649333953857422,
      "loss": 2.8962,
      "nll_loss": 2.8808720111846924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10833916813135147,
      "rewards/margins": 0.9565942883491516,
      "rewards/rejected": -1.0649335384368896,
      "step": 1490
    },
    {
      "epoch": 0.9275272161741835,
      "grad_norm": 0.4083629548549652,
      "learning_rate": 1.2725000000000001e-05,
      "log_odds_chosen": 14.011482238769531,
      "log_odds_ratio": -0.10505443811416626,
      "logits/chosen": 0.3975568115711212,
      "logits/rejected": 1.282463550567627,
      "logps/chosen": -0.7236562967300415,
      "logps/rejected": -14.066780090332031,
      "loss": 2.9918,
      "nll_loss": 2.9812498092651367,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07236562669277191,
      "rewards/margins": 1.3343124389648438,
      "rewards/rejected": -1.4066780805587769,
      "step": 1491
    },
    {
      "epoch": 0.928149300155521,
      "grad_norm": 0.524310827255249,
      "learning_rate": 1.27e-05,
      "log_odds_chosen": 11.368038177490234,
      "log_odds_ratio": -0.09671325981616974,
      "logits/chosen": 0.35482871532440186,
      "logits/rejected": 1.0728152990341187,
      "logps/chosen": -1.3038160800933838,
      "logps/rejected": -12.34891128540039,
      "loss": 3.4129,
      "nll_loss": 3.403229236602783,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13038161396980286,
      "rewards/margins": 1.1045094728469849,
      "rewards/rejected": -1.2348910570144653,
      "step": 1492
    },
    {
      "epoch": 0.9287713841368584,
      "grad_norm": 0.5173730850219727,
      "learning_rate": 1.2675000000000001e-05,
      "log_odds_chosen": 12.411164283752441,
      "log_odds_ratio": -0.0007505936082452536,
      "logits/chosen": 0.20201897621154785,
      "logits/rejected": 0.9776585102081299,
      "logps/chosen": -1.2841124534606934,
      "logps/rejected": -13.26869010925293,
      "loss": 3.0104,
      "nll_loss": 3.0102906227111816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12841124832630157,
      "rewards/margins": 1.1984577178955078,
      "rewards/rejected": -1.326869010925293,
      "step": 1493
    },
    {
      "epoch": 0.929393468118196,
      "grad_norm": 1.3060460090637207,
      "learning_rate": 1.2650000000000001e-05,
      "log_odds_chosen": 8.517346382141113,
      "log_odds_ratio": -0.004937875550240278,
      "logits/chosen": 0.31674104928970337,
      "logits/rejected": 0.7935144901275635,
      "logps/chosen": -1.203063726425171,
      "logps/rejected": -9.244138717651367,
      "loss": 3.5697,
      "nll_loss": 3.5692033767700195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12030638009309769,
      "rewards/margins": 0.8041075468063354,
      "rewards/rejected": -0.9244139194488525,
      "step": 1494
    },
    {
      "epoch": 0.9300155520995335,
      "grad_norm": 0.758243978023529,
      "learning_rate": 1.2625e-05,
      "log_odds_chosen": 16.914310455322266,
      "log_odds_ratio": -5.289989076118218e-06,
      "logits/chosen": 0.4182242453098297,
      "logits/rejected": 1.7866361141204834,
      "logps/chosen": -0.9813098907470703,
      "logps/rejected": -17.347728729248047,
      "loss": 3.1011,
      "nll_loss": 3.101064920425415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09813098609447479,
      "rewards/margins": 1.6366418600082397,
      "rewards/rejected": -1.7347729206085205,
      "step": 1495
    },
    {
      "epoch": 0.9306376360808709,
      "grad_norm": 0.61381995677948,
      "learning_rate": 1.2600000000000001e-05,
      "log_odds_chosen": 4.939112186431885,
      "log_odds_ratio": -0.23473593592643738,
      "logits/chosen": 0.18245118856430054,
      "logits/rejected": 0.4139311611652374,
      "logps/chosen": -1.11427903175354,
      "logps/rejected": -5.520164966583252,
      "loss": 3.0184,
      "nll_loss": 2.9949193000793457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1114279106259346,
      "rewards/margins": 0.4405885636806488,
      "rewards/rejected": -0.5520164966583252,
      "step": 1496
    },
    {
      "epoch": 0.9312597200622084,
      "grad_norm": 0.6340899467468262,
      "learning_rate": 1.2575e-05,
      "log_odds_chosen": 5.411050796508789,
      "log_odds_ratio": -0.32791846990585327,
      "logits/chosen": 0.20967331528663635,
      "logits/rejected": 0.4660225510597229,
      "logps/chosen": -1.0537598133087158,
      "logps/rejected": -6.0242204666137695,
      "loss": 3.0802,
      "nll_loss": 3.047390937805176,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1053759828209877,
      "rewards/margins": 0.4970460534095764,
      "rewards/rejected": -0.6024219989776611,
      "step": 1497
    },
    {
      "epoch": 0.9318818040435459,
      "grad_norm": 0.5436956286430359,
      "learning_rate": 1.255e-05,
      "log_odds_chosen": 8.004091262817383,
      "log_odds_ratio": -0.1258280873298645,
      "logits/chosen": 0.11888009309768677,
      "logits/rejected": 0.5211424827575684,
      "logps/chosen": -1.5225926637649536,
      "logps/rejected": -9.303966522216797,
      "loss": 2.7858,
      "nll_loss": 2.7731993198394775,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15225926041603088,
      "rewards/margins": 0.7781373858451843,
      "rewards/rejected": -0.9303966760635376,
      "step": 1498
    },
    {
      "epoch": 0.9325038880248834,
      "grad_norm": 0.577427864074707,
      "learning_rate": 1.2525000000000001e-05,
      "log_odds_chosen": 9.300687789916992,
      "log_odds_ratio": -0.15734805166721344,
      "logits/chosen": 0.2356635332107544,
      "logits/rejected": 1.1234387159347534,
      "logps/chosen": -1.3030877113342285,
      "logps/rejected": -10.311586380004883,
      "loss": 2.6674,
      "nll_loss": 2.6516411304473877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13030877709388733,
      "rewards/margins": 0.9008498191833496,
      "rewards/rejected": -1.031158685684204,
      "step": 1499
    },
    {
      "epoch": 0.9331259720062208,
      "grad_norm": 0.4966254234313965,
      "learning_rate": 1.25e-05,
      "log_odds_chosen": 7.163057327270508,
      "log_odds_ratio": -0.11469916999340057,
      "logits/chosen": 0.2907523810863495,
      "logits/rejected": 0.8035860657691956,
      "logps/chosen": -1.073452353477478,
      "logps/rejected": -7.86415958404541,
      "loss": 3.1622,
      "nll_loss": 3.150710344314575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10734523087739944,
      "rewards/margins": 0.6790707111358643,
      "rewards/rejected": -0.7864159345626831,
      "step": 1500
    },
    {
      "epoch": 0.9337480559875583,
      "grad_norm": 0.7652081251144409,
      "learning_rate": 1.2475e-05,
      "log_odds_chosen": 8.80681324005127,
      "log_odds_ratio": -0.10107364505529404,
      "logits/chosen": 0.19027899205684662,
      "logits/rejected": 0.9780977964401245,
      "logps/chosen": -1.2092275619506836,
      "logps/rejected": -9.490230560302734,
      "loss": 2.8116,
      "nll_loss": 2.8014516830444336,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1209227591753006,
      "rewards/margins": 0.828100323677063,
      "rewards/rejected": -0.9490230679512024,
      "step": 1501
    },
    {
      "epoch": 0.9343701399688958,
      "grad_norm": 0.5554780960083008,
      "learning_rate": 1.2450000000000001e-05,
      "log_odds_chosen": 5.736096382141113,
      "log_odds_ratio": -0.10775266587734222,
      "logits/chosen": 0.21648985147476196,
      "logits/rejected": 0.7037409543991089,
      "logps/chosen": -1.4072825908660889,
      "logps/rejected": -6.900873184204102,
      "loss": 2.9205,
      "nll_loss": 2.909726858139038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14072826504707336,
      "rewards/margins": 0.549359142780304,
      "rewards/rejected": -0.6900873184204102,
      "step": 1502
    },
    {
      "epoch": 0.9349922239502333,
      "grad_norm": 0.4990847110748291,
      "learning_rate": 1.2425e-05,
      "log_odds_chosen": 6.844992160797119,
      "log_odds_ratio": -0.12505614757537842,
      "logits/chosen": 0.1515367031097412,
      "logits/rejected": 0.5935308933258057,
      "logps/chosen": -1.2481999397277832,
      "logps/rejected": -7.804927825927734,
      "loss": 2.8519,
      "nll_loss": 2.8393449783325195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12482000142335892,
      "rewards/margins": 0.6556727886199951,
      "rewards/rejected": -0.7804927825927734,
      "step": 1503
    },
    {
      "epoch": 0.9356143079315707,
      "grad_norm": 0.5404874682426453,
      "learning_rate": 1.24e-05,
      "log_odds_chosen": 12.516252517700195,
      "log_odds_ratio": -0.001035528490319848,
      "logits/chosen": 0.2302742600440979,
      "logits/rejected": 0.9458526372909546,
      "logps/chosen": -1.1124039888381958,
      "logps/rejected": -13.01448917388916,
      "loss": 2.7871,
      "nll_loss": 2.786956548690796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11124040186405182,
      "rewards/margins": 1.1902084350585938,
      "rewards/rejected": -1.3014488220214844,
      "step": 1504
    },
    {
      "epoch": 0.9362363919129082,
      "grad_norm": 0.4964461326599121,
      "learning_rate": 1.2375000000000001e-05,
      "log_odds_chosen": 7.724780559539795,
      "log_odds_ratio": -0.2758060395717621,
      "logits/chosen": 0.26488885283470154,
      "logits/rejected": 0.8047075867652893,
      "logps/chosen": -1.2019445896148682,
      "logps/rejected": -8.662567138671875,
      "loss": 3.0769,
      "nll_loss": 3.0493555068969727,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1201944649219513,
      "rewards/margins": 0.7460623383522034,
      "rewards/rejected": -0.8662567734718323,
      "step": 1505
    },
    {
      "epoch": 0.9368584758942458,
      "grad_norm": 0.6931914687156677,
      "learning_rate": 1.235e-05,
      "log_odds_chosen": 10.359264373779297,
      "log_odds_ratio": -0.2229677140712738,
      "logits/chosen": 0.22140294313430786,
      "logits/rejected": 0.8083517551422119,
      "logps/chosen": -1.0726377964019775,
      "logps/rejected": -11.14098834991455,
      "loss": 2.9101,
      "nll_loss": 2.8877878189086914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10726377367973328,
      "rewards/margins": 1.0068351030349731,
      "rewards/rejected": -1.1140987873077393,
      "step": 1506
    },
    {
      "epoch": 0.9374805598755832,
      "grad_norm": 0.5076847076416016,
      "learning_rate": 1.2325e-05,
      "log_odds_chosen": 13.57852840423584,
      "log_odds_ratio": -0.00017591772484593093,
      "logits/chosen": 0.15400853753089905,
      "logits/rejected": 1.0751701593399048,
      "logps/chosen": -1.0404856204986572,
      "logps/rejected": -14.068706512451172,
      "loss": 2.4864,
      "nll_loss": 2.486356019973755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10404857248067856,
      "rewards/margins": 1.3028221130371094,
      "rewards/rejected": -1.4068706035614014,
      "step": 1507
    },
    {
      "epoch": 0.9381026438569207,
      "grad_norm": 0.5365278720855713,
      "learning_rate": 1.23e-05,
      "log_odds_chosen": 10.816153526306152,
      "log_odds_ratio": -0.12300407141447067,
      "logits/chosen": 0.0691407173871994,
      "logits/rejected": 0.763458251953125,
      "logps/chosen": -1.367710828781128,
      "logps/rejected": -11.853177070617676,
      "loss": 2.67,
      "nll_loss": 2.6577091217041016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1367710828781128,
      "rewards/margins": 1.0485467910766602,
      "rewards/rejected": -1.1853177547454834,
      "step": 1508
    },
    {
      "epoch": 0.9387247278382581,
      "grad_norm": 1.6353999376296997,
      "learning_rate": 1.2275e-05,
      "log_odds_chosen": 6.172170639038086,
      "log_odds_ratio": -0.28141510486602783,
      "logits/chosen": 0.2383582592010498,
      "logits/rejected": 0.6284301280975342,
      "logps/chosen": -1.2594974040985107,
      "logps/rejected": -6.96533203125,
      "loss": 3.2839,
      "nll_loss": 3.2557425498962402,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12594974040985107,
      "rewards/margins": 0.5705834627151489,
      "rewards/rejected": -0.696533203125,
      "step": 1509
    },
    {
      "epoch": 0.9393468118195957,
      "grad_norm": 0.8588716387748718,
      "learning_rate": 1.225e-05,
      "log_odds_chosen": 4.53616189956665,
      "log_odds_ratio": -0.2243345081806183,
      "logits/chosen": 0.2073875516653061,
      "logits/rejected": 0.4629060626029968,
      "logps/chosen": -1.5986669063568115,
      "logps/rejected": -5.792633533477783,
      "loss": 2.6652,
      "nll_loss": 2.642760753631592,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15986669063568115,
      "rewards/margins": 0.41939669847488403,
      "rewards/rejected": -0.5792633891105652,
      "step": 1510
    },
    {
      "epoch": 0.9399688958009331,
      "grad_norm": 0.9089362621307373,
      "learning_rate": 1.2225e-05,
      "log_odds_chosen": 8.19037914276123,
      "log_odds_ratio": -0.11289818584918976,
      "logits/chosen": 0.18413937091827393,
      "logits/rejected": 0.6798813939094543,
      "logps/chosen": -0.737865686416626,
      "logps/rejected": -7.826382160186768,
      "loss": 2.6895,
      "nll_loss": 2.6782355308532715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07378657162189484,
      "rewards/margins": 0.70885169506073,
      "rewards/rejected": -0.7826382517814636,
      "step": 1511
    },
    {
      "epoch": 0.9405909797822706,
      "grad_norm": 0.5411819815635681,
      "learning_rate": 1.22e-05,
      "log_odds_chosen": 10.123170852661133,
      "log_odds_ratio": -0.16777871549129486,
      "logits/chosen": 0.19700467586517334,
      "logits/rejected": 0.7284948825836182,
      "logps/chosen": -1.0894380807876587,
      "logps/rejected": -10.83730411529541,
      "loss": 2.4656,
      "nll_loss": 2.4488000869750977,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10894380509853363,
      "rewards/margins": 0.974786639213562,
      "rewards/rejected": -1.0837304592132568,
      "step": 1512
    },
    {
      "epoch": 0.9412130637636081,
      "grad_norm": 0.5812392234802246,
      "learning_rate": 1.2175e-05,
      "log_odds_chosen": 14.376922607421875,
      "log_odds_ratio": -0.020723722875118256,
      "logits/chosen": 0.3078254163265228,
      "logits/rejected": 1.2554408311843872,
      "logps/chosen": -0.8478683233261108,
      "logps/rejected": -14.51792049407959,
      "loss": 2.8554,
      "nll_loss": 2.853318214416504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08478683233261108,
      "rewards/margins": 1.3670051097869873,
      "rewards/rejected": -1.4517920017242432,
      "step": 1513
    },
    {
      "epoch": 0.9418351477449456,
      "grad_norm": 7.773168087005615,
      "learning_rate": 1.215e-05,
      "log_odds_chosen": 10.952980041503906,
      "log_odds_ratio": -0.2803260087966919,
      "logits/chosen": 0.21798907220363617,
      "logits/rejected": 0.8301116228103638,
      "logps/chosen": -1.5885159969329834,
      "logps/rejected": -12.118048667907715,
      "loss": 3.4017,
      "nll_loss": 3.373647928237915,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15885160863399506,
      "rewards/margins": 1.0529532432556152,
      "rewards/rejected": -1.2118048667907715,
      "step": 1514
    },
    {
      "epoch": 0.942457231726283,
      "grad_norm": 0.5161781907081604,
      "learning_rate": 1.2125e-05,
      "log_odds_chosen": 10.635643005371094,
      "log_odds_ratio": -0.16009806096553802,
      "logits/chosen": 0.3003097176551819,
      "logits/rejected": 0.7898467779159546,
      "logps/chosen": -1.16111159324646,
      "logps/rejected": -11.478269577026367,
      "loss": 3.3575,
      "nll_loss": 3.3414993286132812,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.116111159324646,
      "rewards/margins": 1.0317158699035645,
      "rewards/rejected": -1.1478270292282104,
      "step": 1515
    },
    {
      "epoch": 0.9430793157076205,
      "grad_norm": 0.5859395861625671,
      "learning_rate": 1.2100000000000001e-05,
      "log_odds_chosen": 14.82717514038086,
      "log_odds_ratio": -0.0002996036200784147,
      "logits/chosen": 0.31332436203956604,
      "logits/rejected": 1.3550702333450317,
      "logps/chosen": -1.2290230989456177,
      "logps/rejected": -15.66615104675293,
      "loss": 2.9716,
      "nll_loss": 2.9715206623077393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12290230393409729,
      "rewards/margins": 1.443712830543518,
      "rewards/rejected": -1.566615104675293,
      "step": 1516
    },
    {
      "epoch": 0.943701399688958,
      "grad_norm": 0.8364419341087341,
      "learning_rate": 1.2075e-05,
      "log_odds_chosen": 7.173739910125732,
      "log_odds_ratio": -0.07395786046981812,
      "logits/chosen": 0.307338148355484,
      "logits/rejected": 0.6798582077026367,
      "logps/chosen": -0.8903729915618896,
      "logps/rejected": -7.38499641418457,
      "loss": 3.2096,
      "nll_loss": 3.2022242546081543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08903729170560837,
      "rewards/margins": 0.6494623422622681,
      "rewards/rejected": -0.738499641418457,
      "step": 1517
    },
    {
      "epoch": 0.9443234836702955,
      "grad_norm": 0.7112438678741455,
      "learning_rate": 1.205e-05,
      "log_odds_chosen": 4.342818260192871,
      "log_odds_ratio": -0.16208398342132568,
      "logits/chosen": 0.19038252532482147,
      "logits/rejected": 0.6972472071647644,
      "logps/chosen": -1.3774667978286743,
      "logps/rejected": -5.464968204498291,
      "loss": 2.9031,
      "nll_loss": 2.886842727661133,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1377466768026352,
      "rewards/margins": 0.40875014662742615,
      "rewards/rejected": -0.5464968681335449,
      "step": 1518
    },
    {
      "epoch": 0.944945567651633,
      "grad_norm": 0.5884023904800415,
      "learning_rate": 1.2025000000000001e-05,
      "log_odds_chosen": 11.338025093078613,
      "log_odds_ratio": -0.02644185721874237,
      "logits/chosen": 0.18361350893974304,
      "logits/rejected": 0.8677639365196228,
      "logps/chosen": -1.2472485303878784,
      "logps/rejected": -12.216198921203613,
      "loss": 2.9414,
      "nll_loss": 2.938786506652832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1247248575091362,
      "rewards/margins": 1.0968950986862183,
      "rewards/rejected": -1.2216198444366455,
      "step": 1519
    },
    {
      "epoch": 0.9455676516329704,
      "grad_norm": 0.416962206363678,
      "learning_rate": 1.2e-05,
      "log_odds_chosen": 12.717013359069824,
      "log_odds_ratio": -0.007837346754968166,
      "logits/chosen": 0.2448866069316864,
      "logits/rejected": 1.115902304649353,
      "logps/chosen": -1.300870418548584,
      "logps/rejected": -13.522640228271484,
      "loss": 3.1906,
      "nll_loss": 3.1898019313812256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13008704781532288,
      "rewards/margins": 1.2221770286560059,
      "rewards/rejected": -1.352264165878296,
      "step": 1520
    },
    {
      "epoch": 0.946189735614308,
      "grad_norm": 0.6918087601661682,
      "learning_rate": 1.1975e-05,
      "log_odds_chosen": 10.162681579589844,
      "log_odds_ratio": -0.16703368723392487,
      "logits/chosen": 0.1385897994041443,
      "logits/rejected": 0.8961490988731384,
      "logps/chosen": -1.1439799070358276,
      "logps/rejected": -11.012479782104492,
      "loss": 2.7063,
      "nll_loss": 2.689603090286255,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11439798772335052,
      "rewards/margins": 0.9868499040603638,
      "rewards/rejected": -1.1012479066848755,
      "step": 1521
    },
    {
      "epoch": 0.9468118195956454,
      "grad_norm": 0.849907398223877,
      "learning_rate": 1.195e-05,
      "log_odds_chosen": 13.232062339782715,
      "log_odds_ratio": -0.00012399072875268757,
      "logits/chosen": 0.3761923015117645,
      "logits/rejected": 1.0816802978515625,
      "logps/chosen": -1.2260112762451172,
      "logps/rejected": -13.891904830932617,
      "loss": 3.7704,
      "nll_loss": 3.770425796508789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12260112911462784,
      "rewards/margins": 1.2665894031524658,
      "rewards/rejected": -1.3891905546188354,
      "step": 1522
    },
    {
      "epoch": 0.9474339035769829,
      "grad_norm": 1.2110743522644043,
      "learning_rate": 1.1925e-05,
      "log_odds_chosen": 14.117300033569336,
      "log_odds_ratio": -0.07730953395366669,
      "logits/chosen": 0.18281584978103638,
      "logits/rejected": 1.4821653366088867,
      "logps/chosen": -1.16435968875885,
      "logps/rejected": -14.846202850341797,
      "loss": 2.2272,
      "nll_loss": 2.2194743156433105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1164359599351883,
      "rewards/margins": 1.3681843280792236,
      "rewards/rejected": -1.4846203327178955,
      "step": 1523
    },
    {
      "epoch": 0.9480559875583203,
      "grad_norm": 0.4888734519481659,
      "learning_rate": 1.19e-05,
      "log_odds_chosen": 12.100495338439941,
      "log_odds_ratio": -0.26559317111968994,
      "logits/chosen": 0.33780574798583984,
      "logits/rejected": 0.9818902015686035,
      "logps/chosen": -1.1453903913497925,
      "logps/rejected": -12.870171546936035,
      "loss": 3.0814,
      "nll_loss": 3.0548579692840576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11453904211521149,
      "rewards/margins": 1.1724780797958374,
      "rewards/rejected": -1.2870171070098877,
      "step": 1524
    },
    {
      "epoch": 0.9486780715396579,
      "grad_norm": 3.656186103820801,
      "learning_rate": 1.1875e-05,
      "log_odds_chosen": 12.780552864074707,
      "log_odds_ratio": -0.1394210159778595,
      "logits/chosen": 0.3146355152130127,
      "logits/rejected": 1.1511363983154297,
      "logps/chosen": -1.582490086555481,
      "logps/rejected": -14.108150482177734,
      "loss": 3.215,
      "nll_loss": 3.201098918914795,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15824902057647705,
      "rewards/margins": 1.2525660991668701,
      "rewards/rejected": -1.4108150005340576,
      "step": 1525
    },
    {
      "epoch": 0.9493001555209953,
      "grad_norm": 0.49954620003700256,
      "learning_rate": 1.185e-05,
      "log_odds_chosen": 10.924458503723145,
      "log_odds_ratio": -0.07372286170721054,
      "logits/chosen": 0.2594726085662842,
      "logits/rejected": 0.8973318338394165,
      "logps/chosen": -0.8566200733184814,
      "logps/rejected": -11.07498550415039,
      "loss": 3.4027,
      "nll_loss": 3.395279884338379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08566200733184814,
      "rewards/margins": 1.021836519241333,
      "rewards/rejected": -1.1074985265731812,
      "step": 1526
    },
    {
      "epoch": 0.9499222395023328,
      "grad_norm": 0.37401673197746277,
      "learning_rate": 1.1825e-05,
      "log_odds_chosen": 12.93400764465332,
      "log_odds_ratio": -0.10499030351638794,
      "logits/chosen": 0.04626879468560219,
      "logits/rejected": 0.5351424217224121,
      "logps/chosen": -1.0168579816818237,
      "logps/rejected": -13.485580444335938,
      "loss": 3.0433,
      "nll_loss": 3.032796859741211,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10168579965829849,
      "rewards/margins": 1.2468721866607666,
      "rewards/rejected": -1.3485580682754517,
      "step": 1527
    },
    {
      "epoch": 0.9505443234836704,
      "grad_norm": 0.49836209416389465,
      "learning_rate": 1.18e-05,
      "log_odds_chosen": 11.764126777648926,
      "log_odds_ratio": -0.043174177408218384,
      "logits/chosen": 0.18523219227790833,
      "logits/rejected": 0.8043461441993713,
      "logps/chosen": -1.0149867534637451,
      "logps/rejected": -12.14633846282959,
      "loss": 3.1922,
      "nll_loss": 3.187835931777954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10149867087602615,
      "rewards/margins": 1.1131352186203003,
      "rewards/rejected": -1.2146339416503906,
      "step": 1528
    },
    {
      "epoch": 0.9511664074650078,
      "grad_norm": 2.499340295791626,
      "learning_rate": 1.1775e-05,
      "log_odds_chosen": 14.100419998168945,
      "log_odds_ratio": -0.029255589470267296,
      "logits/chosen": 0.21268950402736664,
      "logits/rejected": 0.9389889240264893,
      "logps/chosen": -1.1754329204559326,
      "logps/rejected": -14.794118881225586,
      "loss": 3.0132,
      "nll_loss": 3.0102486610412598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1175433024764061,
      "rewards/margins": 1.3618686199188232,
      "rewards/rejected": -1.4794118404388428,
      "step": 1529
    },
    {
      "epoch": 0.9517884914463453,
      "grad_norm": 0.5740556716918945,
      "learning_rate": 1.175e-05,
      "log_odds_chosen": 9.988779067993164,
      "log_odds_ratio": -0.02954120934009552,
      "logits/chosen": 0.12227550148963928,
      "logits/rejected": 0.5920629501342773,
      "logps/chosen": -1.3234734535217285,
      "logps/rejected": -10.88769817352295,
      "loss": 3.0954,
      "nll_loss": 3.0924153327941895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13234734535217285,
      "rewards/margins": 0.9564225077629089,
      "rewards/rejected": -1.0887699127197266,
      "step": 1530
    },
    {
      "epoch": 0.9524105754276827,
      "grad_norm": 0.3808734714984894,
      "learning_rate": 1.1725e-05,
      "log_odds_chosen": 13.617674827575684,
      "log_odds_ratio": -0.0005064024007879198,
      "logits/chosen": 0.14402905106544495,
      "logits/rejected": 0.8458138704299927,
      "logps/chosen": -1.1474123001098633,
      "logps/rejected": -14.359789848327637,
      "loss": 2.9651,
      "nll_loss": 2.9650535583496094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11474122852087021,
      "rewards/margins": 1.3212378025054932,
      "rewards/rejected": -1.4359790086746216,
      "step": 1531
    },
    {
      "epoch": 0.9530326594090203,
      "grad_norm": 0.594470739364624,
      "learning_rate": 1.1700000000000001e-05,
      "log_odds_chosen": 10.063478469848633,
      "log_odds_ratio": -0.008701215498149395,
      "logits/chosen": 0.2576248049736023,
      "logits/rejected": 1.087363600730896,
      "logps/chosen": -1.083966851234436,
      "logps/rejected": -10.723621368408203,
      "loss": 2.9443,
      "nll_loss": 2.9434709548950195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10839669406414032,
      "rewards/margins": 0.9639653563499451,
      "rewards/rejected": -1.0723621845245361,
      "step": 1532
    },
    {
      "epoch": 0.9536547433903577,
      "grad_norm": 0.38040781021118164,
      "learning_rate": 1.1675000000000001e-05,
      "log_odds_chosen": 7.64688777923584,
      "log_odds_ratio": -0.33788639307022095,
      "logits/chosen": 0.09052865207195282,
      "logits/rejected": 0.3300938010215759,
      "logps/chosen": -1.3858916759490967,
      "logps/rejected": -8.915044784545898,
      "loss": 3.3453,
      "nll_loss": 3.311525344848633,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13858917355537415,
      "rewards/margins": 0.7529152631759644,
      "rewards/rejected": -0.8915044665336609,
      "step": 1533
    },
    {
      "epoch": 0.9542768273716952,
      "grad_norm": 0.540230929851532,
      "learning_rate": 1.1650000000000002e-05,
      "log_odds_chosen": 10.206024169921875,
      "log_odds_ratio": -0.10539490729570389,
      "logits/chosen": 0.17809538543224335,
      "logits/rejected": 0.9211235642433167,
      "logps/chosen": -1.1614084243774414,
      "logps/rejected": -11.00497055053711,
      "loss": 2.933,
      "nll_loss": 2.922435760498047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11614084243774414,
      "rewards/margins": 0.984356164932251,
      "rewards/rejected": -1.1004970073699951,
      "step": 1534
    },
    {
      "epoch": 0.9548989113530326,
      "grad_norm": 0.5923968553543091,
      "learning_rate": 1.1625000000000001e-05,
      "log_odds_chosen": 8.06085205078125,
      "log_odds_ratio": -0.10900537669658661,
      "logits/chosen": 0.09346047043800354,
      "logits/rejected": 0.6258691549301147,
      "logps/chosen": -1.2285250425338745,
      "logps/rejected": -8.991416931152344,
      "loss": 2.5072,
      "nll_loss": 2.4963459968566895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12285250425338745,
      "rewards/margins": 0.7762892842292786,
      "rewards/rejected": -0.899141788482666,
      "step": 1535
    },
    {
      "epoch": 0.9555209953343702,
      "grad_norm": 0.576630711555481,
      "learning_rate": 1.16e-05,
      "log_odds_chosen": 11.273231506347656,
      "log_odds_ratio": -0.01995195262134075,
      "logits/chosen": 0.1091519147157669,
      "logits/rejected": 0.8646926879882812,
      "logps/chosen": -1.2671400308609009,
      "logps/rejected": -12.117172241210938,
      "loss": 3.0024,
      "nll_loss": 3.0004241466522217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12671400606632233,
      "rewards/margins": 1.085003137588501,
      "rewards/rejected": -1.211717128753662,
      "step": 1536
    },
    {
      "epoch": 0.9561430793157076,
      "grad_norm": 0.5668447613716125,
      "learning_rate": 1.1575000000000002e-05,
      "log_odds_chosen": 8.51119327545166,
      "log_odds_ratio": -0.1863744705915451,
      "logits/chosen": 0.22063475847244263,
      "logits/rejected": 1.0275760889053345,
      "logps/chosen": -1.0437042713165283,
      "logps/rejected": -9.248046875,
      "loss": 2.7686,
      "nll_loss": 2.7500054836273193,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10437043011188507,
      "rewards/margins": 0.8204342722892761,
      "rewards/rejected": -0.9248047471046448,
      "step": 1537
    },
    {
      "epoch": 0.9567651632970451,
      "grad_norm": 0.7183383703231812,
      "learning_rate": 1.1550000000000001e-05,
      "log_odds_chosen": 9.048027038574219,
      "log_odds_ratio": -0.18893525004386902,
      "logits/chosen": 0.14682061970233917,
      "logits/rejected": 0.5823942422866821,
      "logps/chosen": -1.291245937347412,
      "logps/rejected": -10.022661209106445,
      "loss": 3.2216,
      "nll_loss": 3.2026634216308594,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12912458181381226,
      "rewards/margins": 0.8731414675712585,
      "rewards/rejected": -1.0022661685943604,
      "step": 1538
    },
    {
      "epoch": 0.9573872472783825,
      "grad_norm": 0.7190625071525574,
      "learning_rate": 1.1525e-05,
      "log_odds_chosen": 11.10485553741455,
      "log_odds_ratio": -0.03244490176439285,
      "logits/chosen": 0.24327440559864044,
      "logits/rejected": 0.8990823030471802,
      "logps/chosen": -1.2052109241485596,
      "logps/rejected": -11.896177291870117,
      "loss": 2.7229,
      "nll_loss": 2.7196097373962402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12052108347415924,
      "rewards/margins": 1.0690966844558716,
      "rewards/rejected": -1.1896177530288696,
      "step": 1539
    },
    {
      "epoch": 0.9580093312597201,
      "grad_norm": 0.5160905718803406,
      "learning_rate": 1.1500000000000002e-05,
      "log_odds_chosen": 14.448993682861328,
      "log_odds_ratio": -1.6958052583504468e-05,
      "logits/chosen": 0.31216803193092346,
      "logits/rejected": 1.264636516571045,
      "logps/chosen": -1.1925134658813477,
      "logps/rejected": -15.253753662109375,
      "loss": 3.3065,
      "nll_loss": 3.306514263153076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11925134807825089,
      "rewards/margins": 1.4061239957809448,
      "rewards/rejected": -1.5253753662109375,
      "step": 1540
    },
    {
      "epoch": 0.9586314152410575,
      "grad_norm": 0.43151193857192993,
      "learning_rate": 1.1475000000000001e-05,
      "log_odds_chosen": 10.120941162109375,
      "log_odds_ratio": -0.06145137920975685,
      "logits/chosen": 0.107419952750206,
      "logits/rejected": 0.42087522149086,
      "logps/chosen": -1.298701286315918,
      "logps/rejected": -10.87364673614502,
      "loss": 3.4746,
      "nll_loss": 3.46848464012146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12987013161182404,
      "rewards/margins": 0.9574945569038391,
      "rewards/rejected": -1.087364673614502,
      "step": 1541
    },
    {
      "epoch": 0.959253499222395,
      "grad_norm": 0.513953447341919,
      "learning_rate": 1.145e-05,
      "log_odds_chosen": 7.542330265045166,
      "log_odds_ratio": -0.3065045475959778,
      "logits/chosen": 0.23044796288013458,
      "logits/rejected": 0.6118386387825012,
      "logps/chosen": -1.169670820236206,
      "logps/rejected": -8.212411880493164,
      "loss": 3.0168,
      "nll_loss": 2.9861817359924316,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11696707457304001,
      "rewards/margins": 0.7042741179466248,
      "rewards/rejected": -0.8212412595748901,
      "step": 1542
    },
    {
      "epoch": 0.9598755832037325,
      "grad_norm": 0.4111732840538025,
      "learning_rate": 1.1425000000000002e-05,
      "log_odds_chosen": 11.027361869812012,
      "log_odds_ratio": -0.09346923232078552,
      "logits/chosen": 0.35561326146125793,
      "logits/rejected": 1.1297205686569214,
      "logps/chosen": -1.0780565738677979,
      "logps/rejected": -11.643434524536133,
      "loss": 3.6231,
      "nll_loss": 3.613704204559326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10780565440654755,
      "rewards/margins": 1.0565377473831177,
      "rewards/rejected": -1.1643434762954712,
      "step": 1543
    },
    {
      "epoch": 0.96049766718507,
      "grad_norm": 0.5090989470481873,
      "learning_rate": 1.1400000000000001e-05,
      "log_odds_chosen": 4.659493446350098,
      "log_odds_ratio": -0.24689598381519318,
      "logits/chosen": 0.09714489430189133,
      "logits/rejected": 0.3079416751861572,
      "logps/chosen": -1.0464566946029663,
      "logps/rejected": -5.345902442932129,
      "loss": 2.9323,
      "nll_loss": 2.9075896739959717,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10464566946029663,
      "rewards/margins": 0.42994457483291626,
      "rewards/rejected": -0.5345902442932129,
      "step": 1544
    },
    {
      "epoch": 0.9611197511664075,
      "grad_norm": 0.6913397312164307,
      "learning_rate": 1.1375e-05,
      "log_odds_chosen": 12.19678020477295,
      "log_odds_ratio": -0.07039018720388412,
      "logits/chosen": 0.13496407866477966,
      "logits/rejected": 0.8670356273651123,
      "logps/chosen": -0.9649935960769653,
      "logps/rejected": -12.593484878540039,
      "loss": 2.7044,
      "nll_loss": 2.6974096298217773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09649936109781265,
      "rewards/margins": 1.1628491878509521,
      "rewards/rejected": -1.2593486309051514,
      "step": 1545
    },
    {
      "epoch": 0.9617418351477449,
      "grad_norm": 0.6813112497329712,
      "learning_rate": 1.1350000000000001e-05,
      "log_odds_chosen": 11.80770492553711,
      "log_odds_ratio": -0.06004830822348595,
      "logits/chosen": 0.1646079421043396,
      "logits/rejected": 0.7866946458816528,
      "logps/chosen": -1.2272096872329712,
      "logps/rejected": -12.65283203125,
      "loss": 3.0013,
      "nll_loss": 2.9952850341796875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12272097170352936,
      "rewards/margins": 1.1425621509552002,
      "rewards/rejected": -1.265283226966858,
      "step": 1546
    },
    {
      "epoch": 0.9623639191290825,
      "grad_norm": 0.5108241438865662,
      "learning_rate": 1.1325e-05,
      "log_odds_chosen": 5.958439826965332,
      "log_odds_ratio": -0.4140869081020355,
      "logits/chosen": 0.26702117919921875,
      "logits/rejected": 0.5210127234458923,
      "logps/chosen": -1.2928879261016846,
      "logps/rejected": -6.994579315185547,
      "loss": 3.3799,
      "nll_loss": 3.3384571075439453,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12928879261016846,
      "rewards/margins": 0.5701691508293152,
      "rewards/rejected": -0.6994579434394836,
      "step": 1547
    },
    {
      "epoch": 0.9629860031104199,
      "grad_norm": 0.5286838412284851,
      "learning_rate": 1.13e-05,
      "log_odds_chosen": 14.024574279785156,
      "log_odds_ratio": -0.00039988019852899015,
      "logits/chosen": 0.3011244535446167,
      "logits/rejected": 1.2122119665145874,
      "logps/chosen": -1.0042245388031006,
      "logps/rejected": -14.499638557434082,
      "loss": 3.0475,
      "nll_loss": 3.0474681854248047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1004224568605423,
      "rewards/margins": 1.349541425704956,
      "rewards/rejected": -1.4499640464782715,
      "step": 1548
    },
    {
      "epoch": 0.9636080870917574,
      "grad_norm": 0.4677494168281555,
      "learning_rate": 1.1275000000000001e-05,
      "log_odds_chosen": 9.42393970489502,
      "log_odds_ratio": -0.16956661641597748,
      "logits/chosen": 0.11604233086109161,
      "logits/rejected": 0.7357518076896667,
      "logps/chosen": -1.2022645473480225,
      "logps/rejected": -10.252437591552734,
      "loss": 2.8647,
      "nll_loss": 2.847747802734375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12022645771503448,
      "rewards/margins": 0.9050173759460449,
      "rewards/rejected": -1.0252437591552734,
      "step": 1549
    },
    {
      "epoch": 0.9642301710730948,
      "grad_norm": 0.5168280601501465,
      "learning_rate": 1.125e-05,
      "log_odds_chosen": 11.79471206665039,
      "log_odds_ratio": -0.08363199234008789,
      "logits/chosen": 0.3251190185546875,
      "logits/rejected": 1.2006672620773315,
      "logps/chosen": -1.3963546752929688,
      "logps/rejected": -12.919671058654785,
      "loss": 3.4929,
      "nll_loss": 3.484511137008667,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13963547348976135,
      "rewards/margins": 1.1523315906524658,
      "rewards/rejected": -1.2919670343399048,
      "step": 1550
    },
    {
      "epoch": 0.9648522550544324,
      "grad_norm": 0.6232737898826599,
      "learning_rate": 1.1225e-05,
      "log_odds_chosen": 12.68300724029541,
      "log_odds_ratio": -0.0002065250009763986,
      "logits/chosen": 0.12053397297859192,
      "logits/rejected": 0.9833630323410034,
      "logps/chosen": -1.0209791660308838,
      "logps/rejected": -13.13711166381836,
      "loss": 2.3546,
      "nll_loss": 2.3545565605163574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10209790617227554,
      "rewards/margins": 1.2116132974624634,
      "rewards/rejected": -1.313711166381836,
      "step": 1551
    },
    {
      "epoch": 0.9654743390357698,
      "grad_norm": 1.0204579830169678,
      "learning_rate": 1.1200000000000001e-05,
      "log_odds_chosen": 13.328527450561523,
      "log_odds_ratio": -0.2011832892894745,
      "logits/chosen": 0.3819844722747803,
      "logits/rejected": 1.1638801097869873,
      "logps/chosen": -1.0102115869522095,
      "logps/rejected": -13.841870307922363,
      "loss": 3.1434,
      "nll_loss": 3.12326717376709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10102115571498871,
      "rewards/margins": 1.2831659317016602,
      "rewards/rejected": -1.3841872215270996,
      "step": 1552
    },
    {
      "epoch": 0.9660964230171073,
      "grad_norm": 3.3021206855773926,
      "learning_rate": 1.1175e-05,
      "log_odds_chosen": 14.08477783203125,
      "log_odds_ratio": -0.26379579305648804,
      "logits/chosen": 0.2694425880908966,
      "logits/rejected": 1.4157636165618896,
      "logps/chosen": -1.4726241827011108,
      "logps/rejected": -15.221651077270508,
      "loss": 2.3611,
      "nll_loss": 2.3346829414367676,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14726242423057556,
      "rewards/margins": 1.3749027252197266,
      "rewards/rejected": -1.5221651792526245,
      "step": 1553
    },
    {
      "epoch": 0.9667185069984447,
      "grad_norm": 0.6348780393600464,
      "learning_rate": 1.115e-05,
      "log_odds_chosen": 11.649886131286621,
      "log_odds_ratio": -0.09239879250526428,
      "logits/chosen": 0.21169120073318481,
      "logits/rejected": 0.7421719431877136,
      "logps/chosen": -1.2626230716705322,
      "logps/rejected": -12.575725555419922,
      "loss": 2.7963,
      "nll_loss": 2.7870941162109375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12626230716705322,
      "rewards/margins": 1.131310224533081,
      "rewards/rejected": -1.2575724124908447,
      "step": 1554
    },
    {
      "epoch": 0.9673405909797823,
      "grad_norm": 0.572074830532074,
      "learning_rate": 1.1125000000000001e-05,
      "log_odds_chosen": 10.015185356140137,
      "log_odds_ratio": -0.017020372673869133,
      "logits/chosen": 0.1622290313243866,
      "logits/rejected": 0.8837634325027466,
      "logps/chosen": -1.1480190753936768,
      "logps/rejected": -10.764152526855469,
      "loss": 2.786,
      "nll_loss": 2.7842512130737305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11480191349983215,
      "rewards/margins": 0.9616133570671082,
      "rewards/rejected": -1.0764153003692627,
      "step": 1555
    },
    {
      "epoch": 0.9679626749611198,
      "grad_norm": 0.5456128120422363,
      "learning_rate": 1.11e-05,
      "log_odds_chosen": 15.245088577270508,
      "log_odds_ratio": -0.00036605659988708794,
      "logits/chosen": 0.27020353078842163,
      "logits/rejected": 1.1084659099578857,
      "logps/chosen": -1.1513752937316895,
      "logps/rejected": -15.948476791381836,
      "loss": 2.9191,
      "nll_loss": 2.9190452098846436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11513753235340118,
      "rewards/margins": 1.4797101020812988,
      "rewards/rejected": -1.5948476791381836,
      "step": 1556
    },
    {
      "epoch": 0.9685847589424572,
      "grad_norm": 0.6812815070152283,
      "learning_rate": 1.1075e-05,
      "log_odds_chosen": 9.595189094543457,
      "log_odds_ratio": -0.07183755934238434,
      "logits/chosen": 0.3203434646129608,
      "logits/rejected": 0.9297034740447998,
      "logps/chosen": -1.0930052995681763,
      "logps/rejected": -10.203285217285156,
      "loss": 3.3591,
      "nll_loss": 3.3519504070281982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10930053144693375,
      "rewards/margins": 0.9110280275344849,
      "rewards/rejected": -1.0203286409378052,
      "step": 1557
    },
    {
      "epoch": 0.9692068429237947,
      "grad_norm": 0.4690392315387726,
      "learning_rate": 1.1050000000000001e-05,
      "log_odds_chosen": 5.883632659912109,
      "log_odds_ratio": -0.3575781583786011,
      "logits/chosen": 0.06924352049827576,
      "logits/rejected": 0.5914300680160522,
      "logps/chosen": -1.129926323890686,
      "logps/rejected": -6.685489177703857,
      "loss": 2.4541,
      "nll_loss": 2.4183237552642822,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11299263685941696,
      "rewards/margins": 0.5555562973022461,
      "rewards/rejected": -0.6685488820075989,
      "step": 1558
    },
    {
      "epoch": 0.9698289269051322,
      "grad_norm": 0.4620547890663147,
      "learning_rate": 1.1025e-05,
      "log_odds_chosen": 15.391680717468262,
      "log_odds_ratio": -0.07981463521718979,
      "logits/chosen": 0.2869001030921936,
      "logits/rejected": 1.361629843711853,
      "logps/chosen": -1.096318006515503,
      "logps/rejected": -16.13008689880371,
      "loss": 3.1539,
      "nll_loss": 3.145958185195923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10963179171085358,
      "rewards/margins": 1.5033769607543945,
      "rewards/rejected": -1.6130086183547974,
      "step": 1559
    },
    {
      "epoch": 0.9704510108864697,
      "grad_norm": 0.517099916934967,
      "learning_rate": 1.1000000000000001e-05,
      "log_odds_chosen": 11.813902854919434,
      "log_odds_ratio": -0.07760776579380035,
      "logits/chosen": 0.2125309407711029,
      "logits/rejected": 1.057563304901123,
      "logps/chosen": -1.2820537090301514,
      "logps/rejected": -12.755212783813477,
      "loss": 2.7792,
      "nll_loss": 2.7714877128601074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12820537388324738,
      "rewards/margins": 1.1473159790039062,
      "rewards/rejected": -1.2755212783813477,
      "step": 1560
    },
    {
      "epoch": 0.9710730948678071,
      "grad_norm": 0.5570861101150513,
      "learning_rate": 1.0975e-05,
      "log_odds_chosen": 8.667329788208008,
      "log_odds_ratio": -0.28669866919517517,
      "logits/chosen": 0.26910680532455444,
      "logits/rejected": 0.5824862718582153,
      "logps/chosen": -1.1450307369232178,
      "logps/rejected": -9.458759307861328,
      "loss": 3.2066,
      "nll_loss": 3.177957057952881,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11450307071208954,
      "rewards/margins": 0.831372857093811,
      "rewards/rejected": -0.945875883102417,
      "step": 1561
    },
    {
      "epoch": 0.9716951788491446,
      "grad_norm": 0.6233094930648804,
      "learning_rate": 1.095e-05,
      "log_odds_chosen": 12.726680755615234,
      "log_odds_ratio": -0.0005210311501286924,
      "logits/chosen": 0.2549572288990021,
      "logits/rejected": 1.0368318557739258,
      "logps/chosen": -1.2574390172958374,
      "logps/rejected": -13.563409805297852,
      "loss": 2.9092,
      "nll_loss": 2.909170627593994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12574389576911926,
      "rewards/margins": 1.2305971384048462,
      "rewards/rejected": -1.356341004371643,
      "step": 1562
    },
    {
      "epoch": 0.9723172628304821,
      "grad_norm": 7.476173400878906,
      "learning_rate": 1.0925000000000001e-05,
      "log_odds_chosen": 7.137988567352295,
      "log_odds_ratio": -0.15849797427654266,
      "logits/chosen": 0.4429250657558441,
      "logits/rejected": 1.059818983078003,
      "logps/chosen": -1.839691400527954,
      "logps/rejected": -8.487317085266113,
      "loss": 3.1695,
      "nll_loss": 3.1536335945129395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1839691400527954,
      "rewards/margins": 0.6647626161575317,
      "rewards/rejected": -0.8487317562103271,
      "step": 1563
    },
    {
      "epoch": 0.9729393468118196,
      "grad_norm": 1.7156760692596436,
      "learning_rate": 1.09e-05,
      "log_odds_chosen": 10.09739875793457,
      "log_odds_ratio": -0.14037665724754333,
      "logits/chosen": 0.16592925786972046,
      "logits/rejected": 0.832611083984375,
      "logps/chosen": -0.9634124636650085,
      "logps/rejected": -10.481101036071777,
      "loss": 2.9348,
      "nll_loss": 2.920729637145996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09634124487638474,
      "rewards/margins": 0.9517688751220703,
      "rewards/rejected": -1.0481101274490356,
      "step": 1564
    },
    {
      "epoch": 0.973561430793157,
      "grad_norm": 0.5390524864196777,
      "learning_rate": 1.0875e-05,
      "log_odds_chosen": 9.242631912231445,
      "log_odds_ratio": -0.10577996075153351,
      "logits/chosen": 0.3823816776275635,
      "logits/rejected": 0.9803093671798706,
      "logps/chosen": -1.0878280401229858,
      "logps/rejected": -9.865617752075195,
      "loss": 3.7807,
      "nll_loss": 3.7701072692871094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1087828055024147,
      "rewards/margins": 0.8777790069580078,
      "rewards/rejected": -0.9865618348121643,
      "step": 1565
    },
    {
      "epoch": 0.9741835147744946,
      "grad_norm": 0.5683154463768005,
      "learning_rate": 1.0850000000000001e-05,
      "log_odds_chosen": 10.887935638427734,
      "log_odds_ratio": -0.12982094287872314,
      "logits/chosen": 0.12701228260993958,
      "logits/rejected": 0.630905270576477,
      "logps/chosen": -0.969216525554657,
      "logps/rejected": -11.376995086669922,
      "loss": 2.6706,
      "nll_loss": 2.657618284225464,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0969216525554657,
      "rewards/margins": 1.0407779216766357,
      "rewards/rejected": -1.1376994848251343,
      "step": 1566
    },
    {
      "epoch": 0.9748055987558321,
      "grad_norm": 0.5054221749305725,
      "learning_rate": 1.0825e-05,
      "log_odds_chosen": 11.538697242736816,
      "log_odds_ratio": -0.2012937068939209,
      "logits/chosen": 0.23306933045387268,
      "logits/rejected": 1.1021910905838013,
      "logps/chosen": -1.2251908779144287,
      "logps/rejected": -12.437653541564941,
      "loss": 2.932,
      "nll_loss": 2.911904811859131,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1225190982222557,
      "rewards/margins": 1.121246099472046,
      "rewards/rejected": -1.2437653541564941,
      "step": 1567
    },
    {
      "epoch": 0.9754276827371695,
      "grad_norm": 0.3712461590766907,
      "learning_rate": 1.08e-05,
      "log_odds_chosen": 10.93287181854248,
      "log_odds_ratio": -0.0026042910758405924,
      "logits/chosen": 0.28485745191574097,
      "logits/rejected": 0.8299000263214111,
      "logps/chosen": -1.1558926105499268,
      "logps/rejected": -11.693628311157227,
      "loss": 3.2769,
      "nll_loss": 3.276618480682373,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11558926105499268,
      "rewards/margins": 1.0537735223770142,
      "rewards/rejected": -1.1693627834320068,
      "step": 1568
    },
    {
      "epoch": 0.976049766718507,
      "grad_norm": 0.468656986951828,
      "learning_rate": 1.0775000000000001e-05,
      "log_odds_chosen": 8.820526123046875,
      "log_odds_ratio": -0.1020168736577034,
      "logits/chosen": 0.27156370878219604,
      "logits/rejected": 0.9939742684364319,
      "logps/chosen": -1.0126519203186035,
      "logps/rejected": -9.478734016418457,
      "loss": 3.0389,
      "nll_loss": 3.0287251472473145,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10126519948244095,
      "rewards/margins": 0.8466081619262695,
      "rewards/rejected": -0.9478734135627747,
      "step": 1569
    },
    {
      "epoch": 0.9766718506998445,
      "grad_norm": 0.5117287635803223,
      "learning_rate": 1.075e-05,
      "log_odds_chosen": 6.064488410949707,
      "log_odds_ratio": -0.2216080129146576,
      "logits/chosen": 0.32926154136657715,
      "logits/rejected": 0.6299622654914856,
      "logps/chosen": -0.9756649136543274,
      "logps/rejected": -6.43527364730835,
      "loss": 3.2928,
      "nll_loss": 3.2706661224365234,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09756650030612946,
      "rewards/margins": 0.5459609031677246,
      "rewards/rejected": -0.6435273885726929,
      "step": 1570
    },
    {
      "epoch": 0.977293934681182,
      "grad_norm": 0.4976355731487274,
      "learning_rate": 1.0725e-05,
      "log_odds_chosen": 8.256917953491211,
      "log_odds_ratio": -0.1852550506591797,
      "logits/chosen": 0.40325844287872314,
      "logits/rejected": 0.9676214456558228,
      "logps/chosen": -1.2901231050491333,
      "logps/rejected": -9.301626205444336,
      "loss": 3.2676,
      "nll_loss": 3.249091386795044,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1290123164653778,
      "rewards/margins": 0.8011503219604492,
      "rewards/rejected": -0.9301626682281494,
      "step": 1571
    },
    {
      "epoch": 0.9779160186625194,
      "grad_norm": 0.8282569646835327,
      "learning_rate": 1.0700000000000001e-05,
      "log_odds_chosen": 9.258617401123047,
      "log_odds_ratio": -0.026124443858861923,
      "logits/chosen": 0.21136893332004547,
      "logits/rejected": 0.524401068687439,
      "logps/chosen": -0.92659592628479,
      "logps/rejected": -9.56185245513916,
      "loss": 2.7815,
      "nll_loss": 2.77886700630188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0926596000790596,
      "rewards/margins": 0.8635256886482239,
      "rewards/rejected": -0.9561852812767029,
      "step": 1572
    },
    {
      "epoch": 0.9785381026438569,
      "grad_norm": 0.4185151159763336,
      "learning_rate": 1.0675e-05,
      "log_odds_chosen": 13.6192045211792,
      "log_odds_ratio": -0.1079566702246666,
      "logits/chosen": 0.3801066279411316,
      "logits/rejected": 1.0820777416229248,
      "logps/chosen": -1.1457598209381104,
      "logps/rejected": -14.35219669342041,
      "loss": 3.5128,
      "nll_loss": 3.5020086765289307,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11457598209381104,
      "rewards/margins": 1.320643663406372,
      "rewards/rejected": -1.4352197647094727,
      "step": 1573
    },
    {
      "epoch": 0.9791601866251944,
      "grad_norm": 0.5058881640434265,
      "learning_rate": 1.065e-05,
      "log_odds_chosen": 10.972328186035156,
      "log_odds_ratio": -0.002450345316901803,
      "logits/chosen": 0.11459196358919144,
      "logits/rejected": 0.7418378591537476,
      "logps/chosen": -1.306053638458252,
      "logps/rejected": -11.894512176513672,
      "loss": 2.7975,
      "nll_loss": 2.7972939014434814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13060535490512848,
      "rewards/margins": 1.0588459968566895,
      "rewards/rejected": -1.1894512176513672,
      "step": 1574
    },
    {
      "epoch": 0.9797822706065319,
      "grad_norm": 0.5295330882072449,
      "learning_rate": 1.0625e-05,
      "log_odds_chosen": 10.171019554138184,
      "log_odds_ratio": -0.1479872316122055,
      "logits/chosen": 0.3536169230937958,
      "logits/rejected": 0.7485109567642212,
      "logps/chosen": -1.1546906232833862,
      "logps/rejected": -10.923442840576172,
      "loss": 3.8514,
      "nll_loss": 3.8366503715515137,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1154690533876419,
      "rewards/margins": 0.9768753051757812,
      "rewards/rejected": -1.0923444032669067,
      "step": 1575
    },
    {
      "epoch": 0.9804043545878693,
      "grad_norm": 0.4251069724559784,
      "learning_rate": 1.06e-05,
      "log_odds_chosen": 12.614507675170898,
      "log_odds_ratio": -0.007318359334021807,
      "logits/chosen": 0.33328309655189514,
      "logits/rejected": 1.0371602773666382,
      "logps/chosen": -1.0035243034362793,
      "logps/rejected": -13.143770217895508,
      "loss": 3.0478,
      "nll_loss": 3.047060489654541,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.100352443754673,
      "rewards/margins": 1.214024543762207,
      "rewards/rejected": -1.3143770694732666,
      "step": 1576
    },
    {
      "epoch": 0.9810264385692068,
      "grad_norm": 0.7450897693634033,
      "learning_rate": 1.0575e-05,
      "log_odds_chosen": 12.139310836791992,
      "log_odds_ratio": -0.014527878724038601,
      "logits/chosen": 0.2485947608947754,
      "logits/rejected": 0.7911143898963928,
      "logps/chosen": -1.5825238227844238,
      "logps/rejected": -13.46920394897461,
      "loss": 3.0352,
      "nll_loss": 3.0337603092193604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15825238823890686,
      "rewards/margins": 1.1886680126190186,
      "rewards/rejected": -1.346920371055603,
      "step": 1577
    },
    {
      "epoch": 0.9816485225505444,
      "grad_norm": 0.4823170602321625,
      "learning_rate": 1.055e-05,
      "log_odds_chosen": 10.599369049072266,
      "log_odds_ratio": -0.01910674385726452,
      "logits/chosen": 0.18403150141239166,
      "logits/rejected": 0.7430771589279175,
      "logps/chosen": -0.907859742641449,
      "logps/rejected": -10.833913803100586,
      "loss": 2.9747,
      "nll_loss": 2.972804546356201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09078596532344818,
      "rewards/margins": 0.9926053881645203,
      "rewards/rejected": -1.0833913087844849,
      "step": 1578
    },
    {
      "epoch": 0.9822706065318818,
      "grad_norm": 0.5032903552055359,
      "learning_rate": 1.0525e-05,
      "log_odds_chosen": 11.860549926757812,
      "log_odds_ratio": -0.000851090473588556,
      "logits/chosen": 0.2981140613555908,
      "logits/rejected": 1.1328045129776,
      "logps/chosen": -1.4232183694839478,
      "logps/rejected": -12.965503692626953,
      "loss": 3.3613,
      "nll_loss": 3.3611791133880615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14232183992862701,
      "rewards/margins": 1.1542284488677979,
      "rewards/rejected": -1.2965502738952637,
      "step": 1579
    },
    {
      "epoch": 0.9828926905132193,
      "grad_norm": 0.42979589104652405,
      "learning_rate": 1.05e-05,
      "log_odds_chosen": 11.368293762207031,
      "log_odds_ratio": -0.0016658775275573134,
      "logits/chosen": 0.2528938949108124,
      "logits/rejected": 0.9410756826400757,
      "logps/chosen": -1.0155589580535889,
      "logps/rejected": -11.831025123596191,
      "loss": 3.2585,
      "nll_loss": 3.258355140686035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10155589878559113,
      "rewards/margins": 1.081546664237976,
      "rewards/rejected": -1.1831024885177612,
      "step": 1580
    },
    {
      "epoch": 0.9835147744945568,
      "grad_norm": 0.7552335858345032,
      "learning_rate": 1.0475e-05,
      "log_odds_chosen": 8.832204818725586,
      "log_odds_ratio": -0.027881065383553505,
      "logits/chosen": 0.3224768042564392,
      "logits/rejected": 0.8275183439254761,
      "logps/chosen": -1.3035496473312378,
      "logps/rejected": -9.811277389526367,
      "loss": 3.2479,
      "nll_loss": 3.245144844055176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13035497069358826,
      "rewards/margins": 0.8507728576660156,
      "rewards/rejected": -0.9811277389526367,
      "step": 1581
    },
    {
      "epoch": 0.9841368584758943,
      "grad_norm": 0.4854262173175812,
      "learning_rate": 1.045e-05,
      "log_odds_chosen": 7.1624650955200195,
      "log_odds_ratio": -0.15542778372764587,
      "logits/chosen": 0.3523561358451843,
      "logits/rejected": 0.9019551277160645,
      "logps/chosen": -0.9577276110649109,
      "logps/rejected": -7.694332599639893,
      "loss": 2.7196,
      "nll_loss": 2.7040822505950928,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09577276557683945,
      "rewards/margins": 0.6736605763435364,
      "rewards/rejected": -0.7694332599639893,
      "step": 1582
    },
    {
      "epoch": 0.9847589424572317,
      "grad_norm": 0.48357895016670227,
      "learning_rate": 1.0425e-05,
      "log_odds_chosen": 11.84089469909668,
      "log_odds_ratio": -0.08281046897172928,
      "logits/chosen": 0.2278730422258377,
      "logits/rejected": 1.1716654300689697,
      "logps/chosen": -1.1539627313613892,
      "logps/rejected": -12.60505485534668,
      "loss": 2.3173,
      "nll_loss": 2.3089795112609863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11539626121520996,
      "rewards/margins": 1.1451092958450317,
      "rewards/rejected": -1.2605055570602417,
      "step": 1583
    },
    {
      "epoch": 0.9853810264385692,
      "grad_norm": 0.5850421190261841,
      "learning_rate": 1.04e-05,
      "log_odds_chosen": 11.96889591217041,
      "log_odds_ratio": -0.26381194591522217,
      "logits/chosen": 0.2205938845872879,
      "logits/rejected": 1.0089349746704102,
      "logps/chosen": -1.2542933225631714,
      "logps/rejected": -12.704479217529297,
      "loss": 2.6266,
      "nll_loss": 2.600229501724243,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12542933225631714,
      "rewards/margins": 1.1450186967849731,
      "rewards/rejected": -1.2704479694366455,
      "step": 1584
    },
    {
      "epoch": 0.9860031104199067,
      "grad_norm": 0.5196158289909363,
      "learning_rate": 1.0375e-05,
      "log_odds_chosen": 13.077522277832031,
      "log_odds_ratio": -0.0018111845711246133,
      "logits/chosen": 0.3096368610858917,
      "logits/rejected": 1.080397129058838,
      "logps/chosen": -1.0235284566879272,
      "logps/rejected": -13.350191116333008,
      "loss": 2.9722,
      "nll_loss": 2.972025156021118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10235285013914108,
      "rewards/margins": 1.232666254043579,
      "rewards/rejected": -1.3350191116333008,
      "step": 1585
    },
    {
      "epoch": 0.9866251944012442,
      "grad_norm": 0.5081434845924377,
      "learning_rate": 1.035e-05,
      "log_odds_chosen": 10.091049194335938,
      "log_odds_ratio": -0.05019140988588333,
      "logits/chosen": 0.17959453165531158,
      "logits/rejected": 0.8621308207511902,
      "logps/chosen": -1.0767017602920532,
      "logps/rejected": -10.69836139678955,
      "loss": 2.9533,
      "nll_loss": 2.9482641220092773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10767018049955368,
      "rewards/margins": 0.9621660113334656,
      "rewards/rejected": -1.069836139678955,
      "step": 1586
    },
    {
      "epoch": 0.9872472783825816,
      "grad_norm": 1.4921205043792725,
      "learning_rate": 1.0325e-05,
      "log_odds_chosen": 15.034688949584961,
      "log_odds_ratio": -5.1558799896156415e-06,
      "logits/chosen": 0.35980913043022156,
      "logits/rejected": 1.0580722093582153,
      "logps/chosen": -1.868407130241394,
      "logps/rejected": -16.496301651000977,
      "loss": 3.3692,
      "nll_loss": 3.3691837787628174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1868406981229782,
      "rewards/margins": 1.462789535522461,
      "rewards/rejected": -1.6496303081512451,
      "step": 1587
    },
    {
      "epoch": 0.9878693623639191,
      "grad_norm": 1.2328771352767944,
      "learning_rate": 1.03e-05,
      "log_odds_chosen": 12.517577171325684,
      "log_odds_ratio": -0.020710619166493416,
      "logits/chosen": 0.340537965297699,
      "logits/rejected": 1.3173099756240845,
      "logps/chosen": -0.9290058016777039,
      "logps/rejected": -12.619564056396484,
      "loss": 3.0733,
      "nll_loss": 3.0711874961853027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0929005891084671,
      "rewards/margins": 1.1690558195114136,
      "rewards/rejected": -1.2619564533233643,
      "step": 1588
    },
    {
      "epoch": 0.9884914463452567,
      "grad_norm": 0.639462411403656,
      "learning_rate": 1.0275e-05,
      "log_odds_chosen": 14.48855209350586,
      "log_odds_ratio": -3.144158654322382e-06,
      "logits/chosen": 0.29837891459465027,
      "logits/rejected": 1.3534728288650513,
      "logps/chosen": -0.8880980014801025,
      "logps/rejected": -14.57780933380127,
      "loss": 2.6912,
      "nll_loss": 2.691150188446045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08880980312824249,
      "rewards/margins": 1.3689711093902588,
      "rewards/rejected": -1.4577808380126953,
      "step": 1589
    },
    {
      "epoch": 0.9891135303265941,
      "grad_norm": 0.5558017492294312,
      "learning_rate": 1.025e-05,
      "log_odds_chosen": 11.420799255371094,
      "log_odds_ratio": -0.1718229502439499,
      "logits/chosen": 0.4150314927101135,
      "logits/rejected": 1.6155272722244263,
      "logps/chosen": -1.1975841522216797,
      "logps/rejected": -12.336982727050781,
      "loss": 2.9496,
      "nll_loss": 2.932443380355835,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11975841224193573,
      "rewards/margins": 1.1139397621154785,
      "rewards/rejected": -1.2336982488632202,
      "step": 1590
    },
    {
      "epoch": 0.9897356143079316,
      "grad_norm": 0.4650667905807495,
      "learning_rate": 1.0225e-05,
      "log_odds_chosen": 11.10346508026123,
      "log_odds_ratio": -0.0070058186538517475,
      "logits/chosen": 0.4163108170032501,
      "logits/rejected": 1.0532922744750977,
      "logps/chosen": -1.1275982856750488,
      "logps/rejected": -11.808972358703613,
      "loss": 3.3,
      "nll_loss": 3.2992758750915527,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11275982111692429,
      "rewards/margins": 1.0681374073028564,
      "rewards/rejected": -1.1808972358703613,
      "step": 1591
    },
    {
      "epoch": 0.990357698289269,
      "grad_norm": 0.486665278673172,
      "learning_rate": 1.02e-05,
      "log_odds_chosen": 11.23182487487793,
      "log_odds_ratio": -0.010925328359007835,
      "logits/chosen": 0.1485457718372345,
      "logits/rejected": 0.7938356995582581,
      "logps/chosen": -1.0545248985290527,
      "logps/rejected": -11.765100479125977,
      "loss": 2.9406,
      "nll_loss": 2.9395089149475098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10545249283313751,
      "rewards/margins": 1.0710575580596924,
      "rewards/rejected": -1.1765100955963135,
      "step": 1592
    },
    {
      "epoch": 0.9909797822706066,
      "grad_norm": 0.7251661419868469,
      "learning_rate": 1.0175e-05,
      "log_odds_chosen": 9.512269020080566,
      "log_odds_ratio": -0.2862478494644165,
      "logits/chosen": 0.3005995452404022,
      "logits/rejected": 0.9812291264533997,
      "logps/chosen": -0.9364309310913086,
      "logps/rejected": -10.132664680480957,
      "loss": 2.3115,
      "nll_loss": 2.282909631729126,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09364309161901474,
      "rewards/margins": 0.9196233749389648,
      "rewards/rejected": -1.0132665634155273,
      "step": 1593
    },
    {
      "epoch": 0.991601866251944,
      "grad_norm": 0.5410097241401672,
      "learning_rate": 1.0150000000000001e-05,
      "log_odds_chosen": 9.208051681518555,
      "log_odds_ratio": -0.18747776746749878,
      "logits/chosen": 0.2776232957839966,
      "logits/rejected": 1.036055564880371,
      "logps/chosen": -1.1870697736740112,
      "logps/rejected": -9.9979887008667,
      "loss": 2.9676,
      "nll_loss": 2.948817729949951,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11870698630809784,
      "rewards/margins": 0.8810918927192688,
      "rewards/rejected": -0.9997988939285278,
      "step": 1594
    },
    {
      "epoch": 0.9922239502332815,
      "grad_norm": 0.6100337505340576,
      "learning_rate": 1.0125e-05,
      "log_odds_chosen": 11.178662300109863,
      "log_odds_ratio": -0.02708502486348152,
      "logits/chosen": 0.26386797428131104,
      "logits/rejected": 0.9037798047065735,
      "logps/chosen": -0.9145346879959106,
      "logps/rejected": -11.334199905395508,
      "loss": 3.0249,
      "nll_loss": 3.0221993923187256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09145347774028778,
      "rewards/margins": 1.0419665575027466,
      "rewards/rejected": -1.1334201097488403,
      "step": 1595
    },
    {
      "epoch": 0.9928460342146189,
      "grad_norm": 2.953352451324463,
      "learning_rate": 1.0100000000000002e-05,
      "log_odds_chosen": 9.112907409667969,
      "log_odds_ratio": -0.1708381026983261,
      "logits/chosen": 0.17483511567115784,
      "logits/rejected": 0.6510817408561707,
      "logps/chosen": -1.2610094547271729,
      "logps/rejected": -10.078710556030273,
      "loss": 2.7612,
      "nll_loss": 2.744128465652466,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12610095739364624,
      "rewards/margins": 0.881770133972168,
      "rewards/rejected": -1.007871150970459,
      "step": 1596
    },
    {
      "epoch": 0.9934681181959565,
      "grad_norm": 0.48717230558395386,
      "learning_rate": 1.0075000000000001e-05,
      "log_odds_chosen": 14.488679885864258,
      "log_odds_ratio": -0.0001398213644279167,
      "logits/chosen": 0.22382810711860657,
      "logits/rejected": 1.1981775760650635,
      "logps/chosen": -1.0615004301071167,
      "logps/rejected": -14.9384765625,
      "loss": 2.774,
      "nll_loss": 2.7739906311035156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10615004599094391,
      "rewards/margins": 1.387697696685791,
      "rewards/rejected": -1.4938478469848633,
      "step": 1597
    },
    {
      "epoch": 0.9940902021772939,
      "grad_norm": 0.47258996963500977,
      "learning_rate": 1.005e-05,
      "log_odds_chosen": 9.835807800292969,
      "log_odds_ratio": -0.24436575174331665,
      "logits/chosen": 0.30663585662841797,
      "logits/rejected": 0.9482704401016235,
      "logps/chosen": -1.2437496185302734,
      "logps/rejected": -10.829856872558594,
      "loss": 2.9497,
      "nll_loss": 2.9252965450286865,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12437495589256287,
      "rewards/margins": 0.9586107730865479,
      "rewards/rejected": -1.082985758781433,
      "step": 1598
    },
    {
      "epoch": 0.9947122861586314,
      "grad_norm": 0.5687262415885925,
      "learning_rate": 1.0025000000000001e-05,
      "log_odds_chosen": 7.890524864196777,
      "log_odds_ratio": -0.17712527513504028,
      "logits/chosen": 0.2682409882545471,
      "logits/rejected": 0.8357047438621521,
      "logps/chosen": -1.0649604797363281,
      "logps/rejected": -8.51159954071045,
      "loss": 2.6346,
      "nll_loss": 2.6168601512908936,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10649605840444565,
      "rewards/margins": 0.7446639537811279,
      "rewards/rejected": -0.8511600494384766,
      "step": 1599
    },
    {
      "epoch": 0.995334370139969,
      "grad_norm": 0.6656171679496765,
      "learning_rate": 1e-05,
      "log_odds_chosen": 10.36136531829834,
      "log_odds_ratio": -0.1039852499961853,
      "logits/chosen": 0.3594803810119629,
      "logits/rejected": 0.7507673501968384,
      "logps/chosen": -0.8331475853919983,
      "logps/rejected": -10.611698150634766,
      "loss": 3.0173,
      "nll_loss": 3.0068767070770264,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08331475406885147,
      "rewards/margins": 0.9778551459312439,
      "rewards/rejected": -1.0611698627471924,
      "step": 1600
    },
    {
      "epoch": 0.9959564541213064,
      "grad_norm": 0.7820709943771362,
      "learning_rate": 9.975e-06,
      "log_odds_chosen": 8.559907913208008,
      "log_odds_ratio": -0.19893547892570496,
      "logits/chosen": 0.32968413829803467,
      "logits/rejected": 0.9315425157546997,
      "logps/chosen": -1.1777303218841553,
      "logps/rejected": -9.3145112991333,
      "loss": 3.0425,
      "nll_loss": 3.0226306915283203,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11777302622795105,
      "rewards/margins": 0.8136780261993408,
      "rewards/rejected": -0.931451141834259,
      "step": 1601
    },
    {
      "epoch": 0.9965785381026439,
      "grad_norm": 0.5365874767303467,
      "learning_rate": 9.950000000000001e-06,
      "log_odds_chosen": 4.336380481719971,
      "log_odds_ratio": -0.31836003065109253,
      "logits/chosen": -0.07907748222351074,
      "logits/rejected": 0.1741887927055359,
      "logps/chosen": -1.5812636613845825,
      "logps/rejected": -5.768222808837891,
      "loss": 2.5041,
      "nll_loss": 2.4723105430603027,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1581263691186905,
      "rewards/margins": 0.4186958968639374,
      "rewards/rejected": -0.5768222808837891,
      "step": 1602
    },
    {
      "epoch": 0.9972006220839813,
      "grad_norm": 0.6687927842140198,
      "learning_rate": 9.925e-06,
      "log_odds_chosen": 6.069998264312744,
      "log_odds_ratio": -0.13015542924404144,
      "logits/chosen": 0.20325936377048492,
      "logits/rejected": 0.6469485759735107,
      "logps/chosen": -1.2751343250274658,
      "logps/rejected": -6.99819803237915,
      "loss": 2.9466,
      "nll_loss": 2.9335503578186035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12751343846321106,
      "rewards/margins": 0.5723063945770264,
      "rewards/rejected": -0.699819803237915,
      "step": 1603
    },
    {
      "epoch": 0.9978227060653189,
      "grad_norm": 0.4991750419139862,
      "learning_rate": 9.900000000000002e-06,
      "log_odds_chosen": 8.110227584838867,
      "log_odds_ratio": -0.10920145362615585,
      "logits/chosen": 0.1598237156867981,
      "logits/rejected": 0.7098408341407776,
      "logps/chosen": -1.1333831548690796,
      "logps/rejected": -8.875372886657715,
      "loss": 3.0029,
      "nll_loss": 2.991992950439453,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11333832144737244,
      "rewards/margins": 0.7741988897323608,
      "rewards/rejected": -0.8875373005867004,
      "step": 1604
    },
    {
      "epoch": 0.9984447900466563,
      "grad_norm": 0.4689289629459381,
      "learning_rate": 9.875000000000001e-06,
      "log_odds_chosen": 7.205236911773682,
      "log_odds_ratio": -0.10720035433769226,
      "logits/chosen": 0.09387461841106415,
      "logits/rejected": 0.5457128882408142,
      "logps/chosen": -1.4958709478378296,
      "logps/rejected": -8.467511177062988,
      "loss": 3.0375,
      "nll_loss": 3.0267648696899414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14958709478378296,
      "rewards/margins": 0.697163999080658,
      "rewards/rejected": -0.8467510938644409,
      "step": 1605
    },
    {
      "epoch": 0.9990668740279938,
      "grad_norm": 0.5157873034477234,
      "learning_rate": 9.85e-06,
      "log_odds_chosen": 10.404451370239258,
      "log_odds_ratio": -0.004838836379349232,
      "logits/chosen": 0.23798084259033203,
      "logits/rejected": 0.7781890034675598,
      "logps/chosen": -0.9072043299674988,
      "logps/rejected": -10.63504409790039,
      "loss": 2.5904,
      "nll_loss": 2.5898895263671875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09072043001651764,
      "rewards/margins": 0.9727840423583984,
      "rewards/rejected": -1.0635044574737549,
      "step": 1606
    },
    {
      "epoch": 0.9996889580093312,
      "grad_norm": 0.45538413524627686,
      "learning_rate": 9.825000000000002e-06,
      "log_odds_chosen": 13.454652786254883,
      "log_odds_ratio": -0.007265198975801468,
      "logits/chosen": 0.23318925499916077,
      "logits/rejected": 0.9881614446640015,
      "logps/chosen": -0.8240679502487183,
      "logps/rejected": -13.559259414672852,
      "loss": 3.0339,
      "nll_loss": 3.033188819885254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08240679651498795,
      "rewards/margins": 1.2735192775726318,
      "rewards/rejected": -1.3559260368347168,
      "step": 1607
    },
    {
      "epoch": 1.0003110419906687,
      "grad_norm": 1.1448850631713867,
      "learning_rate": 9.800000000000001e-06,
      "log_odds_chosen": 12.285888671875,
      "log_odds_ratio": -0.0008950508199632168,
      "logits/chosen": 0.1304558366537094,
      "logits/rejected": 0.48669731616973877,
      "logps/chosen": -1.2000110149383545,
      "logps/rejected": -12.767168045043945,
      "loss": 2.9909,
      "nll_loss": 2.9908194541931152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12000110745429993,
      "rewards/margins": 1.1567156314849854,
      "rewards/rejected": -1.2767168283462524,
      "step": 1608
    },
    {
      "epoch": 1.0009331259720062,
      "grad_norm": 6.079453945159912,
      "learning_rate": 9.775e-06,
      "log_odds_chosen": 11.781765937805176,
      "log_odds_ratio": -0.006075817160308361,
      "logits/chosen": 0.31462544202804565,
      "logits/rejected": 0.9736188650131226,
      "logps/chosen": -1.4540257453918457,
      "logps/rejected": -12.950994491577148,
      "loss": 3.3169,
      "nll_loss": 3.316250801086426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14540256559848785,
      "rewards/margins": 1.1496968269348145,
      "rewards/rejected": -1.2950994968414307,
      "step": 1609
    },
    {
      "epoch": 1.0015552099533438,
      "grad_norm": 0.5772498846054077,
      "learning_rate": 9.750000000000002e-06,
      "log_odds_chosen": 9.219070434570312,
      "log_odds_ratio": -0.03137453272938728,
      "logits/chosen": 0.2179422527551651,
      "logits/rejected": 0.3960098624229431,
      "logps/chosen": -0.7287307977676392,
      "logps/rejected": -8.83579158782959,
      "loss": 2.9345,
      "nll_loss": 2.9313859939575195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0728730782866478,
      "rewards/margins": 0.8107061386108398,
      "rewards/rejected": -0.8835792541503906,
      "step": 1610
    },
    {
      "epoch": 1.0021772939346811,
      "grad_norm": 0.5609044432640076,
      "learning_rate": 9.725000000000001e-06,
      "log_odds_chosen": 11.617526054382324,
      "log_odds_ratio": -0.17162840068340302,
      "logits/chosen": 0.2871357798576355,
      "logits/rejected": 1.2228782176971436,
      "logps/chosen": -1.0836530923843384,
      "logps/rejected": -12.375179290771484,
      "loss": 2.9787,
      "nll_loss": 2.96148943901062,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10836531966924667,
      "rewards/margins": 1.129152536392212,
      "rewards/rejected": -1.2375179529190063,
      "step": 1611
    },
    {
      "epoch": 1.0027993779160187,
      "grad_norm": 0.5066659450531006,
      "learning_rate": 9.7e-06,
      "log_odds_chosen": 6.8533430099487305,
      "log_odds_ratio": -0.28324294090270996,
      "logits/chosen": 0.35354509949684143,
      "logits/rejected": 0.6426160335540771,
      "logps/chosen": -1.046383023262024,
      "logps/rejected": -7.608695983886719,
      "loss": 3.3284,
      "nll_loss": 3.3001086711883545,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10463830083608627,
      "rewards/margins": 0.6562313437461853,
      "rewards/rejected": -0.7608696222305298,
      "step": 1612
    },
    {
      "epoch": 1.003421461897356,
      "grad_norm": 0.5004162192344666,
      "learning_rate": 9.675000000000001e-06,
      "log_odds_chosen": 11.484716415405273,
      "log_odds_ratio": -0.09374061226844788,
      "logits/chosen": 0.2518472671508789,
      "logits/rejected": 0.6501962542533875,
      "logps/chosen": -0.8462555408477783,
      "logps/rejected": -11.751243591308594,
      "loss": 3.2331,
      "nll_loss": 3.223677158355713,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08462555706501007,
      "rewards/margins": 1.090498685836792,
      "rewards/rejected": -1.1751242876052856,
      "step": 1613
    },
    {
      "epoch": 1.0040435458786936,
      "grad_norm": 0.5626034736633301,
      "learning_rate": 9.65e-06,
      "log_odds_chosen": 7.137520790100098,
      "log_odds_ratio": -0.25156933069229126,
      "logits/chosen": 0.30671074986457825,
      "logits/rejected": 0.664474606513977,
      "logps/chosen": -1.1866908073425293,
      "logps/rejected": -8.073833465576172,
      "loss": 3.3325,
      "nll_loss": 3.3073184490203857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11866907775402069,
      "rewards/margins": 0.6887142658233643,
      "rewards/rejected": -0.8073834180831909,
      "step": 1614
    },
    {
      "epoch": 1.0046656298600312,
      "grad_norm": 0.6039891242980957,
      "learning_rate": 9.625e-06,
      "log_odds_chosen": 10.04780387878418,
      "log_odds_ratio": -0.017218245193362236,
      "logits/chosen": 0.2603004276752472,
      "logits/rejected": 0.9685711860656738,
      "logps/chosen": -1.0688201189041138,
      "logps/rejected": -10.475671768188477,
      "loss": 2.7024,
      "nll_loss": 2.7006781101226807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1068820059299469,
      "rewards/margins": 0.9406852722167969,
      "rewards/rejected": -1.0475672483444214,
      "step": 1615
    },
    {
      "epoch": 1.0052877138413685,
      "grad_norm": 0.5885332226753235,
      "learning_rate": 9.600000000000001e-06,
      "log_odds_chosen": 14.77972412109375,
      "log_odds_ratio": -8.65639085532166e-05,
      "logits/chosen": 0.2089068740606308,
      "logits/rejected": 1.0688587427139282,
      "logps/chosen": -1.0595755577087402,
      "logps/rejected": -15.397711753845215,
      "loss": 2.7162,
      "nll_loss": 2.7161991596221924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10595756024122238,
      "rewards/margins": 1.4338136911392212,
      "rewards/rejected": -1.5397711992263794,
      "step": 1616
    },
    {
      "epoch": 1.005909797822706,
      "grad_norm": 0.6334537267684937,
      "learning_rate": 9.575e-06,
      "log_odds_chosen": 8.620237350463867,
      "log_odds_ratio": -0.10149388015270233,
      "logits/chosen": 0.31568121910095215,
      "logits/rejected": 0.9553278088569641,
      "logps/chosen": -1.2086679935455322,
      "logps/rejected": -9.4962797164917,
      "loss": 2.6799,
      "nll_loss": 2.669752597808838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1208668053150177,
      "rewards/margins": 0.8287612199783325,
      "rewards/rejected": -0.9496279954910278,
      "step": 1617
    },
    {
      "epoch": 1.0065318818040436,
      "grad_norm": 0.4512050449848175,
      "learning_rate": 9.55e-06,
      "log_odds_chosen": 13.334916114807129,
      "log_odds_ratio": -0.01727379858493805,
      "logits/chosen": 0.27413877844810486,
      "logits/rejected": 1.246893048286438,
      "logps/chosen": -1.2072069644927979,
      "logps/rejected": -14.180831909179688,
      "loss": 3.3728,
      "nll_loss": 3.3710615634918213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12072070688009262,
      "rewards/margins": 1.2973624467849731,
      "rewards/rejected": -1.4180831909179688,
      "step": 1618
    },
    {
      "epoch": 1.007153965785381,
      "grad_norm": 0.4606066048145294,
      "learning_rate": 9.525000000000001e-06,
      "log_odds_chosen": 9.140962600708008,
      "log_odds_ratio": -0.22496956586837769,
      "logits/chosen": 0.26907601952552795,
      "logits/rejected": 0.7347286939620972,
      "logps/chosen": -1.0311001539230347,
      "logps/rejected": -9.772761344909668,
      "loss": 3.3197,
      "nll_loss": 3.2971792221069336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10311002284288406,
      "rewards/margins": 0.8741660714149475,
      "rewards/rejected": -0.9772761464118958,
      "step": 1619
    },
    {
      "epoch": 1.0077760497667185,
      "grad_norm": 0.5686437487602234,
      "learning_rate": 9.5e-06,
      "log_odds_chosen": 11.098395347595215,
      "log_odds_ratio": -0.0006429707864299417,
      "logits/chosen": 0.2674759030342102,
      "logits/rejected": 0.975322961807251,
      "logps/chosen": -0.975488543510437,
      "logps/rejected": -11.32666015625,
      "loss": 2.7744,
      "nll_loss": 2.7743349075317383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09754885733127594,
      "rewards/margins": 1.0351171493530273,
      "rewards/rejected": -1.1326658725738525,
      "step": 1620
    },
    {
      "epoch": 1.008398133748056,
      "grad_norm": 0.5224622488021851,
      "learning_rate": 9.475e-06,
      "log_odds_chosen": 11.144240379333496,
      "log_odds_ratio": -0.020334865897893906,
      "logits/chosen": 0.16083793342113495,
      "logits/rejected": 0.7533071041107178,
      "logps/chosen": -1.4969186782836914,
      "logps/rejected": -12.29748821258545,
      "loss": 3.0359,
      "nll_loss": 3.03383207321167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1496918797492981,
      "rewards/margins": 1.0800570249557495,
      "rewards/rejected": -1.2297488451004028,
      "step": 1621
    },
    {
      "epoch": 1.0090202177293934,
      "grad_norm": 0.5410659313201904,
      "learning_rate": 9.450000000000001e-06,
      "log_odds_chosen": 8.919403076171875,
      "log_odds_ratio": -0.04832353815436363,
      "logits/chosen": 0.11468247324228287,
      "logits/rejected": 0.5462947487831116,
      "logps/chosen": -0.9372066259384155,
      "logps/rejected": -9.317145347595215,
      "loss": 2.7276,
      "nll_loss": 2.7227399349212646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09372066706418991,
      "rewards/margins": 0.8379939198493958,
      "rewards/rejected": -0.931714653968811,
      "step": 1622
    },
    {
      "epoch": 1.009642301710731,
      "grad_norm": 0.5279932022094727,
      "learning_rate": 9.425e-06,
      "log_odds_chosen": 9.845748901367188,
      "log_odds_ratio": -0.22621092200279236,
      "logits/chosen": 0.14059889316558838,
      "logits/rejected": 0.6940497159957886,
      "logps/chosen": -1.0277276039123535,
      "logps/rejected": -10.441768646240234,
      "loss": 2.6895,
      "nll_loss": 2.666898727416992,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10277276486158371,
      "rewards/margins": 0.9414041638374329,
      "rewards/rejected": -1.0441769361495972,
      "step": 1623
    },
    {
      "epoch": 1.0102643856920683,
      "grad_norm": 0.45512518286705017,
      "learning_rate": 9.4e-06,
      "log_odds_chosen": 8.7213716506958,
      "log_odds_ratio": -0.22801584005355835,
      "logits/chosen": 0.18221153318881989,
      "logits/rejected": 0.6221237778663635,
      "logps/chosen": -0.9007062911987305,
      "logps/rejected": -9.128543853759766,
      "loss": 3.0911,
      "nll_loss": 3.0683000087738037,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09007062762975693,
      "rewards/margins": 0.8227838277816772,
      "rewards/rejected": -0.9128544330596924,
      "step": 1624
    },
    {
      "epoch": 1.010886469673406,
      "grad_norm": 0.5499163269996643,
      "learning_rate": 9.375000000000001e-06,
      "log_odds_chosen": 13.863089561462402,
      "log_odds_ratio": -0.00355136557482183,
      "logits/chosen": 0.313961386680603,
      "logits/rejected": 1.2714308500289917,
      "logps/chosen": -1.0769063234329224,
      "logps/rejected": -14.27375602722168,
      "loss": 3.1019,
      "nll_loss": 3.101551055908203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10769063234329224,
      "rewards/margins": 1.3196849822998047,
      "rewards/rejected": -1.4273755550384521,
      "step": 1625
    },
    {
      "epoch": 1.0115085536547435,
      "grad_norm": 10.339038848876953,
      "learning_rate": 9.35e-06,
      "log_odds_chosen": 12.912702560424805,
      "log_odds_ratio": -0.0035233048256486654,
      "logits/chosen": 0.3977145552635193,
      "logits/rejected": 1.5618952512741089,
      "logps/chosen": -1.3644357919692993,
      "logps/rejected": -13.928369522094727,
      "loss": 2.8382,
      "nll_loss": 2.8378021717071533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1364435851573944,
      "rewards/margins": 1.2563934326171875,
      "rewards/rejected": -1.3928370475769043,
      "step": 1626
    },
    {
      "epoch": 1.0121306376360808,
      "grad_norm": 0.6409747004508972,
      "learning_rate": 9.325e-06,
      "log_odds_chosen": 7.80657958984375,
      "log_odds_ratio": -0.14597001671791077,
      "logits/chosen": 0.3135356307029724,
      "logits/rejected": 0.9694831371307373,
      "logps/chosen": -1.0715206861495972,
      "logps/rejected": -8.544049263000488,
      "loss": 3.0718,
      "nll_loss": 3.057161331176758,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1071520671248436,
      "rewards/margins": 0.7472528219223022,
      "rewards/rejected": -0.8544048070907593,
      "step": 1627
    },
    {
      "epoch": 1.0127527216174184,
      "grad_norm": 0.49594753980636597,
      "learning_rate": 9.3e-06,
      "log_odds_chosen": 9.20004653930664,
      "log_odds_ratio": -0.16701725125312805,
      "logits/chosen": 0.205534428358078,
      "logits/rejected": 0.8317206501960754,
      "logps/chosen": -1.0844453573226929,
      "logps/rejected": -9.976225852966309,
      "loss": 2.8505,
      "nll_loss": 2.8337974548339844,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10844454169273376,
      "rewards/margins": 0.8891780972480774,
      "rewards/rejected": -0.9976226687431335,
      "step": 1628
    },
    {
      "epoch": 1.013374805598756,
      "grad_norm": 0.5218719840049744,
      "learning_rate": 9.275e-06,
      "log_odds_chosen": 12.281805038452148,
      "log_odds_ratio": -0.002139680553227663,
      "logits/chosen": 0.30369701981544495,
      "logits/rejected": 1.1859633922576904,
      "logps/chosen": -0.9268889427185059,
      "logps/rejected": -12.533408164978027,
      "loss": 2.9504,
      "nll_loss": 2.950138568878174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0926888957619667,
      "rewards/margins": 1.1606519222259521,
      "rewards/rejected": -1.2533408403396606,
      "step": 1629
    },
    {
      "epoch": 1.0139968895800933,
      "grad_norm": 0.4695424437522888,
      "learning_rate": 9.25e-06,
      "log_odds_chosen": 11.708459854125977,
      "log_odds_ratio": -0.17082315683364868,
      "logits/chosen": 0.063386932015419,
      "logits/rejected": 0.5708033442497253,
      "logps/chosen": -1.0007423162460327,
      "logps/rejected": -12.331064224243164,
      "loss": 3.1128,
      "nll_loss": 3.0956997871398926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10007423907518387,
      "rewards/margins": 1.133032202720642,
      "rewards/rejected": -1.2331064939498901,
      "step": 1630
    },
    {
      "epoch": 1.0146189735614308,
      "grad_norm": 0.704513669013977,
      "learning_rate": 9.225e-06,
      "log_odds_chosen": 3.0890965461730957,
      "log_odds_ratio": -0.2591760456562042,
      "logits/chosen": 0.28655803203582764,
      "logits/rejected": 0.2705000638961792,
      "logps/chosen": -0.9935261011123657,
      "logps/rejected": -3.74479079246521,
      "loss": 3.3577,
      "nll_loss": 3.331785202026367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09935261309146881,
      "rewards/margins": 0.27512648701667786,
      "rewards/rejected": -0.37447911500930786,
      "step": 1631
    },
    {
      "epoch": 1.0152410575427682,
      "grad_norm": 0.49560216069221497,
      "learning_rate": 9.2e-06,
      "log_odds_chosen": 11.836051940917969,
      "log_odds_ratio": -0.0024351109750568867,
      "logits/chosen": 0.23947438597679138,
      "logits/rejected": 1.1317088603973389,
      "logps/chosen": -0.7412389516830444,
      "logps/rejected": -11.846299171447754,
      "loss": 2.8303,
      "nll_loss": 2.8300487995147705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07412389665842056,
      "rewards/margins": 1.1105060577392578,
      "rewards/rejected": -1.1846299171447754,
      "step": 1632
    },
    {
      "epoch": 1.0158631415241057,
      "grad_norm": 0.6829708218574524,
      "learning_rate": 9.175000000000001e-06,
      "log_odds_chosen": 13.079815864562988,
      "log_odds_ratio": -0.011433121748268604,
      "logits/chosen": 0.2853188216686249,
      "logits/rejected": 1.2370648384094238,
      "logps/chosen": -1.119140386581421,
      "logps/rejected": -13.742539405822754,
      "loss": 2.8621,
      "nll_loss": 2.860992908477783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11191404610872269,
      "rewards/margins": 1.2623398303985596,
      "rewards/rejected": -1.3742539882659912,
      "step": 1633
    },
    {
      "epoch": 1.0164852255054433,
      "grad_norm": 0.47538915276527405,
      "learning_rate": 9.15e-06,
      "log_odds_chosen": 13.191617965698242,
      "log_odds_ratio": -0.1977131962776184,
      "logits/chosen": 0.2846056818962097,
      "logits/rejected": 0.9628204107284546,
      "logps/chosen": -1.1906006336212158,
      "logps/rejected": -14.044559478759766,
      "loss": 3.3493,
      "nll_loss": 3.3294787406921387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11906006187200546,
      "rewards/margins": 1.285395860671997,
      "rewards/rejected": -1.4044559001922607,
      "step": 1634
    },
    {
      "epoch": 1.0171073094867806,
      "grad_norm": 0.6789840459823608,
      "learning_rate": 9.125e-06,
      "log_odds_chosen": 12.73701000213623,
      "log_odds_ratio": -0.07127546519041061,
      "logits/chosen": 0.19782181084156036,
      "logits/rejected": 1.3850300312042236,
      "logps/chosen": -1.0893549919128418,
      "logps/rejected": -13.426777839660645,
      "loss": 2.5798,
      "nll_loss": 2.572679281234741,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10893549770116806,
      "rewards/margins": 1.233742356300354,
      "rewards/rejected": -1.3426778316497803,
      "step": 1635
    },
    {
      "epoch": 1.0177293934681182,
      "grad_norm": 0.46701884269714355,
      "learning_rate": 9.100000000000001e-06,
      "log_odds_chosen": 12.932587623596191,
      "log_odds_ratio": -0.0002816337801050395,
      "logits/chosen": 0.3475741147994995,
      "logits/rejected": 1.3140875101089478,
      "logps/chosen": -1.0825169086456299,
      "logps/rejected": -13.484697341918945,
      "loss": 3.4196,
      "nll_loss": 3.419621467590332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10825169086456299,
      "rewards/margins": 1.240217924118042,
      "rewards/rejected": -1.3484697341918945,
      "step": 1636
    },
    {
      "epoch": 1.0183514774494558,
      "grad_norm": 0.5956992506980896,
      "learning_rate": 9.075e-06,
      "log_odds_chosen": 9.46751594543457,
      "log_odds_ratio": -0.12005427479743958,
      "logits/chosen": 0.16276614367961884,
      "logits/rejected": 0.7512280344963074,
      "logps/chosen": -0.8385735154151917,
      "logps/rejected": -9.679712295532227,
      "loss": 2.728,
      "nll_loss": 2.7160072326660156,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08385735750198364,
      "rewards/margins": 0.8841139078140259,
      "rewards/rejected": -0.9679712653160095,
      "step": 1637
    },
    {
      "epoch": 1.018973561430793,
      "grad_norm": 0.44732701778411865,
      "learning_rate": 9.05e-06,
      "log_odds_chosen": 10.955498695373535,
      "log_odds_ratio": -0.1099550873041153,
      "logits/chosen": 0.13595137000083923,
      "logits/rejected": 1.1490558385849,
      "logps/chosen": -1.2426047325134277,
      "logps/rejected": -11.894643783569336,
      "loss": 2.7433,
      "nll_loss": 2.7323482036590576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12426047027111053,
      "rewards/margins": 1.0652040243148804,
      "rewards/rejected": -1.1894644498825073,
      "step": 1638
    },
    {
      "epoch": 1.0195956454121307,
      "grad_norm": 0.8864432573318481,
      "learning_rate": 9.025e-06,
      "log_odds_chosen": 10.31212329864502,
      "log_odds_ratio": -0.17925220727920532,
      "logits/chosen": 0.18999649584293365,
      "logits/rejected": 0.9962109327316284,
      "logps/chosen": -1.1252002716064453,
      "logps/rejected": -10.985133171081543,
      "loss": 3.2673,
      "nll_loss": 3.2493417263031006,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11252002418041229,
      "rewards/margins": 0.9859933257102966,
      "rewards/rejected": -1.0985133647918701,
      "step": 1639
    },
    {
      "epoch": 1.0202177293934682,
      "grad_norm": 0.5043367743492126,
      "learning_rate": 9e-06,
      "log_odds_chosen": 14.583747863769531,
      "log_odds_ratio": -0.005502515938133001,
      "logits/chosen": 0.2616561949253082,
      "logits/rejected": 1.3393410444259644,
      "logps/chosen": -1.0964058637619019,
      "logps/rejected": -15.243389129638672,
      "loss": 3.0131,
      "nll_loss": 3.0125889778137207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10964059084653854,
      "rewards/margins": 1.4146982431411743,
      "rewards/rejected": -1.5243388414382935,
      "step": 1640
    },
    {
      "epoch": 1.0208398133748056,
      "grad_norm": 0.4772394299507141,
      "learning_rate": 8.975e-06,
      "log_odds_chosen": 13.822057723999023,
      "log_odds_ratio": -0.003005790524184704,
      "logits/chosen": 0.2907000184059143,
      "logits/rejected": 1.3935949802398682,
      "logps/chosen": -1.2302254438400269,
      "logps/rejected": -14.66832447052002,
      "loss": 2.9518,
      "nll_loss": 2.9514856338500977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12302252650260925,
      "rewards/margins": 1.3438100814819336,
      "rewards/rejected": -1.4668323993682861,
      "step": 1641
    },
    {
      "epoch": 1.0214618973561431,
      "grad_norm": 0.5077674984931946,
      "learning_rate": 8.95e-06,
      "log_odds_chosen": 10.747535705566406,
      "log_odds_ratio": -0.11549379676580429,
      "logits/chosen": 0.23091387748718262,
      "logits/rejected": 1.3234198093414307,
      "logps/chosen": -1.0908149480819702,
      "logps/rejected": -11.471622467041016,
      "loss": 2.7741,
      "nll_loss": 2.7625174522399902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10908149182796478,
      "rewards/margins": 1.0380808115005493,
      "rewards/rejected": -1.1471623182296753,
      "step": 1642
    },
    {
      "epoch": 1.0220839813374805,
      "grad_norm": 0.4699145555496216,
      "learning_rate": 8.925e-06,
      "log_odds_chosen": 11.556468963623047,
      "log_odds_ratio": -0.0014705258654430509,
      "logits/chosen": 0.30578139424324036,
      "logits/rejected": 1.0413280725479126,
      "logps/chosen": -0.8892867565155029,
      "logps/rejected": -11.754606246948242,
      "loss": 3.3391,
      "nll_loss": 3.338972806930542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08892867714166641,
      "rewards/margins": 1.0865321159362793,
      "rewards/rejected": -1.175460696220398,
      "step": 1643
    },
    {
      "epoch": 1.022706065318818,
      "grad_norm": 0.5027690529823303,
      "learning_rate": 8.9e-06,
      "log_odds_chosen": 10.694028854370117,
      "log_odds_ratio": -0.10222534090280533,
      "logits/chosen": 0.2960459589958191,
      "logits/rejected": 1.1127769947052002,
      "logps/chosen": -1.3843754529953003,
      "logps/rejected": -11.760503768920898,
      "loss": 3.3106,
      "nll_loss": 3.3003430366516113,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13843755424022675,
      "rewards/margins": 1.037612795829773,
      "rewards/rejected": -1.1760503053665161,
      "step": 1644
    },
    {
      "epoch": 1.0233281493001556,
      "grad_norm": 0.4155937135219574,
      "learning_rate": 8.875e-06,
      "log_odds_chosen": 11.716170310974121,
      "log_odds_ratio": -0.0036328420974314213,
      "logits/chosen": 0.22692963480949402,
      "logits/rejected": 1.210436463356018,
      "logps/chosen": -1.059539794921875,
      "logps/rejected": -12.310782432556152,
      "loss": 2.9858,
      "nll_loss": 2.985433340072632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10595397651195526,
      "rewards/margins": 1.125124216079712,
      "rewards/rejected": -1.2310782670974731,
      "step": 1645
    },
    {
      "epoch": 1.023950233281493,
      "grad_norm": 0.5141968131065369,
      "learning_rate": 8.85e-06,
      "log_odds_chosen": 14.782506942749023,
      "log_odds_ratio": -3.0398521175811766e-06,
      "logits/chosen": 0.2925991714000702,
      "logits/rejected": 1.2928005456924438,
      "logps/chosen": -1.1867799758911133,
      "logps/rejected": -15.458690643310547,
      "loss": 3.3069,
      "nll_loss": 3.3069450855255127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11867799609899521,
      "rewards/margins": 1.427191138267517,
      "rewards/rejected": -1.5458691120147705,
      "step": 1646
    },
    {
      "epoch": 1.0245723172628305,
      "grad_norm": 0.46805500984191895,
      "learning_rate": 8.825e-06,
      "log_odds_chosen": 11.715710639953613,
      "log_odds_ratio": -0.05991049483418465,
      "logits/chosen": 0.1678595393896103,
      "logits/rejected": 0.6166034936904907,
      "logps/chosen": -1.2212872505187988,
      "logps/rejected": -12.560037612915039,
      "loss": 2.9821,
      "nll_loss": 2.976140022277832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12212873995304108,
      "rewards/margins": 1.1338750123977661,
      "rewards/rejected": -1.256003737449646,
      "step": 1647
    },
    {
      "epoch": 1.025194401244168,
      "grad_norm": 0.5412681102752686,
      "learning_rate": 8.8e-06,
      "log_odds_chosen": 6.542932510375977,
      "log_odds_ratio": -0.17082630097866058,
      "logits/chosen": 0.08377973735332489,
      "logits/rejected": 0.2964077889919281,
      "logps/chosen": -1.022648572921753,
      "logps/rejected": -7.137678623199463,
      "loss": 2.8866,
      "nll_loss": 2.869472026824951,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10226485878229141,
      "rewards/margins": 0.611503005027771,
      "rewards/rejected": -0.7137678861618042,
      "step": 1648
    },
    {
      "epoch": 1.0258164852255054,
      "grad_norm": 0.568443238735199,
      "learning_rate": 8.775e-06,
      "log_odds_chosen": 13.49286937713623,
      "log_odds_ratio": -0.11019520461559296,
      "logits/chosen": 0.21617048978805542,
      "logits/rejected": 1.1071817874908447,
      "logps/chosen": -1.297656774520874,
      "logps/rejected": -14.47043228149414,
      "loss": 2.8539,
      "nll_loss": 2.8428421020507812,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12976567447185516,
      "rewards/margins": 1.3172776699066162,
      "rewards/rejected": -1.4470432996749878,
      "step": 1649
    },
    {
      "epoch": 1.026438569206843,
      "grad_norm": 0.8365088105201721,
      "learning_rate": 8.75e-06,
      "log_odds_chosen": 8.931707382202148,
      "log_odds_ratio": -0.054758794605731964,
      "logits/chosen": 0.1608840823173523,
      "logits/rejected": 0.92148357629776,
      "logps/chosen": -1.4928791522979736,
      "logps/rejected": -10.121048927307129,
      "loss": 2.7444,
      "nll_loss": 2.738970994949341,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14928790926933289,
      "rewards/margins": 0.8628170490264893,
      "rewards/rejected": -1.0121049880981445,
      "step": 1650
    },
    {
      "epoch": 1.0270606531881805,
      "grad_norm": 0.593108594417572,
      "learning_rate": 8.725e-06,
      "log_odds_chosen": 8.8706636428833,
      "log_odds_ratio": -0.1627025157213211,
      "logits/chosen": 0.2817043960094452,
      "logits/rejected": 0.9622519612312317,
      "logps/chosen": -1.1062893867492676,
      "logps/rejected": -9.600112915039062,
      "loss": 2.8041,
      "nll_loss": 2.7878713607788086,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11062893271446228,
      "rewards/margins": 0.8493824005126953,
      "rewards/rejected": -0.9600112438201904,
      "step": 1651
    },
    {
      "epoch": 1.0276827371695179,
      "grad_norm": 0.527895987033844,
      "learning_rate": 8.7e-06,
      "log_odds_chosen": 10.27220630645752,
      "log_odds_ratio": -0.0012983853230252862,
      "logits/chosen": 0.32650476694107056,
      "logits/rejected": 0.947507381439209,
      "logps/chosen": -1.0816434621810913,
      "logps/rejected": -10.885249137878418,
      "loss": 3.1771,
      "nll_loss": 3.1769237518310547,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10816435515880585,
      "rewards/margins": 0.9803605079650879,
      "rewards/rejected": -1.0885248184204102,
      "step": 1652
    },
    {
      "epoch": 1.0283048211508554,
      "grad_norm": 0.5855411887168884,
      "learning_rate": 8.674999999999999e-06,
      "log_odds_chosen": 10.549446105957031,
      "log_odds_ratio": -0.1294151395559311,
      "logits/chosen": 0.24553564190864563,
      "logits/rejected": 0.9942394495010376,
      "logps/chosen": -1.028377890586853,
      "logps/rejected": -11.156509399414062,
      "loss": 3.0605,
      "nll_loss": 3.0475263595581055,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10283779352903366,
      "rewards/margins": 1.0128130912780762,
      "rewards/rejected": -1.11565101146698,
      "step": 1653
    },
    {
      "epoch": 1.0289269051321928,
      "grad_norm": 0.48511913418769836,
      "learning_rate": 8.65e-06,
      "log_odds_chosen": 11.069501876831055,
      "log_odds_ratio": -0.2245204597711563,
      "logits/chosen": 0.2477826178073883,
      "logits/rejected": 1.0707584619522095,
      "logps/chosen": -1.2009871006011963,
      "logps/rejected": -12.028597831726074,
      "loss": 3.0148,
      "nll_loss": 2.9923830032348633,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12009871006011963,
      "rewards/margins": 1.0827610492706299,
      "rewards/rejected": -1.2028597593307495,
      "step": 1654
    },
    {
      "epoch": 1.0295489891135303,
      "grad_norm": 0.4419611096382141,
      "learning_rate": 8.625e-06,
      "log_odds_chosen": 6.270127773284912,
      "log_odds_ratio": -0.284975528717041,
      "logits/chosen": 0.2656201422214508,
      "logits/rejected": 0.5505940914154053,
      "logps/chosen": -0.9445799589157104,
      "logps/rejected": -6.642779350280762,
      "loss": 3.4913,
      "nll_loss": 3.4627652168273926,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09445799887180328,
      "rewards/margins": 0.5698199272155762,
      "rewards/rejected": -0.664277970790863,
      "step": 1655
    },
    {
      "epoch": 1.0301710730948679,
      "grad_norm": 0.651220977306366,
      "learning_rate": 8.599999999999999e-06,
      "log_odds_chosen": 10.812170028686523,
      "log_odds_ratio": -0.11016173660755157,
      "logits/chosen": 0.4329462945461273,
      "logits/rejected": 0.958261251449585,
      "logps/chosen": -0.9240531921386719,
      "logps/rejected": -11.261800765991211,
      "loss": 3.3339,
      "nll_loss": 3.3228507041931152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09240531921386719,
      "rewards/margins": 1.033774733543396,
      "rewards/rejected": -1.1261800527572632,
      "step": 1656
    },
    {
      "epoch": 1.0307931570762052,
      "grad_norm": 0.4697381556034088,
      "learning_rate": 8.575000000000002e-06,
      "log_odds_chosen": 13.0179443359375,
      "log_odds_ratio": -0.04233914241194725,
      "logits/chosen": 0.40351903438568115,
      "logits/rejected": 1.2205008268356323,
      "logps/chosen": -0.8437970280647278,
      "logps/rejected": -13.180481910705566,
      "loss": 3.3443,
      "nll_loss": 3.340068817138672,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08437971025705338,
      "rewards/margins": 1.2336684465408325,
      "rewards/rejected": -1.3180482387542725,
      "step": 1657
    },
    {
      "epoch": 1.0314152410575428,
      "grad_norm": 0.4366108775138855,
      "learning_rate": 8.550000000000001e-06,
      "log_odds_chosen": 14.342367172241211,
      "log_odds_ratio": -0.0002054653305094689,
      "logits/chosen": 0.38981306552886963,
      "logits/rejected": 1.3780097961425781,
      "logps/chosen": -1.0400646924972534,
      "logps/rejected": -14.886777877807617,
      "loss": 3.5205,
      "nll_loss": 3.520503520965576,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10400646924972534,
      "rewards/margins": 1.3846713304519653,
      "rewards/rejected": -1.4886778593063354,
      "step": 1658
    },
    {
      "epoch": 1.0320373250388803,
      "grad_norm": 0.6422800421714783,
      "learning_rate": 8.525e-06,
      "log_odds_chosen": 18.632835388183594,
      "log_odds_ratio": -3.8743075947422767e-07,
      "logits/chosen": 0.290744811296463,
      "logits/rejected": 1.8333404064178467,
      "logps/chosen": -1.349289894104004,
      "logps/rejected": -19.636199951171875,
      "loss": 2.8319,
      "nll_loss": 2.8319075107574463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13492897152900696,
      "rewards/margins": 1.8286911249160767,
      "rewards/rejected": -1.9636200666427612,
      "step": 1659
    },
    {
      "epoch": 1.0326594090202177,
      "grad_norm": 0.5107365846633911,
      "learning_rate": 8.500000000000002e-06,
      "log_odds_chosen": 9.844313621520996,
      "log_odds_ratio": -0.11742329597473145,
      "logits/chosen": 0.2040708363056183,
      "logits/rejected": 0.6333246231079102,
      "logps/chosen": -1.2048256397247314,
      "logps/rejected": -10.693470001220703,
      "loss": 2.9243,
      "nll_loss": 2.912597179412842,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12048256397247314,
      "rewards/margins": 0.9488645195960999,
      "rewards/rejected": -1.0693471431732178,
      "step": 1660
    },
    {
      "epoch": 1.0332814930015553,
      "grad_norm": 0.4743788540363312,
      "learning_rate": 8.475000000000001e-06,
      "log_odds_chosen": 14.308911323547363,
      "log_odds_ratio": -0.08154227584600449,
      "logits/chosen": 0.16491791605949402,
      "logits/rejected": 1.0353258848190308,
      "logps/chosen": -1.0096592903137207,
      "logps/rejected": -14.768461227416992,
      "loss": 2.7397,
      "nll_loss": 2.731551170349121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1009659394621849,
      "rewards/margins": 1.375880241394043,
      "rewards/rejected": -1.4768463373184204,
      "step": 1661
    },
    {
      "epoch": 1.0339035769828926,
      "grad_norm": 0.5415398478507996,
      "learning_rate": 8.45e-06,
      "log_odds_chosen": 12.673973083496094,
      "log_odds_ratio": -0.0749824270606041,
      "logits/chosen": 0.27741706371307373,
      "logits/rejected": 1.1964267492294312,
      "logps/chosen": -0.8574082851409912,
      "logps/rejected": -13.011079788208008,
      "loss": 2.7563,
      "nll_loss": 2.7487845420837402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0857408344745636,
      "rewards/margins": 1.2153671979904175,
      "rewards/rejected": -1.3011078834533691,
      "step": 1662
    },
    {
      "epoch": 1.0345256609642302,
      "grad_norm": 2.5805556774139404,
      "learning_rate": 8.425000000000001e-06,
      "log_odds_chosen": 12.792621612548828,
      "log_odds_ratio": -0.09287473559379578,
      "logits/chosen": 0.3072134852409363,
      "logits/rejected": 1.4438683986663818,
      "logps/chosen": -1.1244754791259766,
      "logps/rejected": -13.274697303771973,
      "loss": 2.7362,
      "nll_loss": 2.7268877029418945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11244755238294601,
      "rewards/margins": 1.2150222063064575,
      "rewards/rejected": -1.327469825744629,
      "step": 1663
    },
    {
      "epoch": 1.0351477449455677,
      "grad_norm": 0.5692621469497681,
      "learning_rate": 8.400000000000001e-06,
      "log_odds_chosen": 12.14323902130127,
      "log_odds_ratio": -0.08631030470132828,
      "logits/chosen": 0.2854219377040863,
      "logits/rejected": 1.3667171001434326,
      "logps/chosen": -1.0392296314239502,
      "logps/rejected": -12.711332321166992,
      "loss": 2.9046,
      "nll_loss": 2.895948648452759,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10392297059297562,
      "rewards/margins": 1.1672102212905884,
      "rewards/rejected": -1.2711331844329834,
      "step": 1664
    },
    {
      "epoch": 1.035769828926905,
      "grad_norm": 0.6746039986610413,
      "learning_rate": 8.375e-06,
      "log_odds_chosen": 10.063125610351562,
      "log_odds_ratio": -0.1466355323791504,
      "logits/chosen": 0.371590256690979,
      "logits/rejected": 0.8608337044715881,
      "logps/chosen": -0.8012755513191223,
      "logps/rejected": -10.27314281463623,
      "loss": 3.3805,
      "nll_loss": 3.3658719062805176,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08012756705284119,
      "rewards/margins": 0.9471867084503174,
      "rewards/rejected": -1.027314305305481,
      "step": 1665
    },
    {
      "epoch": 1.0363919129082426,
      "grad_norm": 1.6298115253448486,
      "learning_rate": 8.350000000000001e-06,
      "log_odds_chosen": 12.227920532226562,
      "log_odds_ratio": -0.013340647332370281,
      "logits/chosen": 0.1919909566640854,
      "logits/rejected": 0.9904528856277466,
      "logps/chosen": -1.073433756828308,
      "logps/rejected": -12.796059608459473,
      "loss": 2.9285,
      "nll_loss": 2.927194595336914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10734338313341141,
      "rewards/margins": 1.1722626686096191,
      "rewards/rejected": -1.2796061038970947,
      "step": 1666
    },
    {
      "epoch": 1.0370139968895802,
      "grad_norm": 0.6101058125495911,
      "learning_rate": 8.325e-06,
      "log_odds_chosen": 10.183680534362793,
      "log_odds_ratio": -0.03945440798997879,
      "logits/chosen": 0.19264741241931915,
      "logits/rejected": 1.0039281845092773,
      "logps/chosen": -0.9766156077384949,
      "logps/rejected": -10.625382423400879,
      "loss": 2.5488,
      "nll_loss": 2.544872999191284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0976615697145462,
      "rewards/margins": 0.9648767113685608,
      "rewards/rejected": -1.0625382661819458,
      "step": 1667
    },
    {
      "epoch": 1.0376360808709175,
      "grad_norm": 0.5250122547149658,
      "learning_rate": 8.3e-06,
      "log_odds_chosen": 9.799418449401855,
      "log_odds_ratio": -0.010854106396436691,
      "logits/chosen": 0.27762043476104736,
      "logits/rejected": 0.7463703155517578,
      "logps/chosen": -1.1591010093688965,
      "logps/rejected": -10.477595329284668,
      "loss": 3.3777,
      "nll_loss": 3.3766472339630127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.115910105407238,
      "rewards/margins": 0.931849479675293,
      "rewards/rejected": -1.0477595329284668,
      "step": 1668
    },
    {
      "epoch": 1.038258164852255,
      "grad_norm": 0.6709027886390686,
      "learning_rate": 8.275000000000001e-06,
      "log_odds_chosen": 11.033429145812988,
      "log_odds_ratio": -0.17414605617523193,
      "logits/chosen": 0.4432452619075775,
      "logits/rejected": 1.211717128753662,
      "logps/chosen": -0.9460048675537109,
      "logps/rejected": -11.531136512756348,
      "loss": 3.2111,
      "nll_loss": 3.1937031745910645,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09460049867630005,
      "rewards/margins": 1.0585131645202637,
      "rewards/rejected": -1.1531137228012085,
      "step": 1669
    },
    {
      "epoch": 1.0388802488335926,
      "grad_norm": 2.698802947998047,
      "learning_rate": 8.25e-06,
      "log_odds_chosen": 13.314163208007812,
      "log_odds_ratio": -0.07637029886245728,
      "logits/chosen": 0.20780786871910095,
      "logits/rejected": 0.8780178427696228,
      "logps/chosen": -1.4755923748016357,
      "logps/rejected": -14.065767288208008,
      "loss": 2.9374,
      "nll_loss": 2.929812431335449,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1475592404603958,
      "rewards/margins": 1.2590174674987793,
      "rewards/rejected": -1.4065766334533691,
      "step": 1670
    },
    {
      "epoch": 1.03950233281493,
      "grad_norm": 0.4637618660926819,
      "learning_rate": 8.225e-06,
      "log_odds_chosen": 14.614171981811523,
      "log_odds_ratio": -0.061297375708818436,
      "logits/chosen": 0.36924469470977783,
      "logits/rejected": 1.271532654762268,
      "logps/chosen": -0.9310328364372253,
      "logps/rejected": -15.078804969787598,
      "loss": 3.3165,
      "nll_loss": 3.310403347015381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09310328215360641,
      "rewards/margins": 1.4147772789001465,
      "rewards/rejected": -1.507880449295044,
      "step": 1671
    },
    {
      "epoch": 1.0401244167962675,
      "grad_norm": 0.5099697709083557,
      "learning_rate": 8.200000000000001e-06,
      "log_odds_chosen": 10.016826629638672,
      "log_odds_ratio": -0.10711626708507538,
      "logits/chosen": 0.01908724009990692,
      "logits/rejected": 0.46566832065582275,
      "logps/chosen": -0.7953150272369385,
      "logps/rejected": -10.18687915802002,
      "loss": 2.4798,
      "nll_loss": 2.469115972518921,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07953150570392609,
      "rewards/margins": 0.9391564130783081,
      "rewards/rejected": -1.0186879634857178,
      "step": 1672
    },
    {
      "epoch": 1.0407465007776049,
      "grad_norm": 0.49379605054855347,
      "learning_rate": 8.175e-06,
      "log_odds_chosen": 12.284273147583008,
      "log_odds_ratio": -0.05759778618812561,
      "logits/chosen": 0.3830791413784027,
      "logits/rejected": 1.3384016752243042,
      "logps/chosen": -0.8844354748725891,
      "logps/rejected": -12.67086410522461,
      "loss": 2.7113,
      "nll_loss": 2.7055153846740723,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08844354748725891,
      "rewards/margins": 1.178642988204956,
      "rewards/rejected": -1.2670865058898926,
      "step": 1673
    },
    {
      "epoch": 1.0413685847589425,
      "grad_norm": 1.1188862323760986,
      "learning_rate": 8.15e-06,
      "log_odds_chosen": 9.94202709197998,
      "log_odds_ratio": -0.09657282382249832,
      "logits/chosen": 0.293910950422287,
      "logits/rejected": 0.9388896226882935,
      "logps/chosen": -1.2859009504318237,
      "logps/rejected": -10.89704704284668,
      "loss": 2.8781,
      "nll_loss": 2.86840558052063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12859009206295013,
      "rewards/margins": 0.9611146450042725,
      "rewards/rejected": -1.0897047519683838,
      "step": 1674
    },
    {
      "epoch": 1.04199066874028,
      "grad_norm": 0.5678275227546692,
      "learning_rate": 8.125000000000001e-06,
      "log_odds_chosen": 13.660589218139648,
      "log_odds_ratio": -0.0012313383631408215,
      "logits/chosen": 0.20735794305801392,
      "logits/rejected": 1.7398574352264404,
      "logps/chosen": -1.1810181140899658,
      "logps/rejected": -14.4437255859375,
      "loss": 2.6848,
      "nll_loss": 2.684652328491211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1181018203496933,
      "rewards/margins": 1.326270580291748,
      "rewards/rejected": -1.4443724155426025,
      "step": 1675
    },
    {
      "epoch": 1.0426127527216174,
      "grad_norm": 0.5290229916572571,
      "learning_rate": 8.1e-06,
      "log_odds_chosen": 4.564973831176758,
      "log_odds_ratio": -0.2650543451309204,
      "logits/chosen": 0.25153490900993347,
      "logits/rejected": 0.5440673828125,
      "logps/chosen": -1.191606044769287,
      "logps/rejected": -5.467530250549316,
      "loss": 3.2536,
      "nll_loss": 3.2270967960357666,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11916061490774155,
      "rewards/margins": 0.427592396736145,
      "rewards/rejected": -0.5467530488967896,
      "step": 1676
    },
    {
      "epoch": 1.043234836702955,
      "grad_norm": 0.39739978313446045,
      "learning_rate": 8.075000000000001e-06,
      "log_odds_chosen": 10.400979042053223,
      "log_odds_ratio": -0.2506236732006073,
      "logits/chosen": 0.16898976266384125,
      "logits/rejected": 0.5718826055526733,
      "logps/chosen": -1.1356854438781738,
      "logps/rejected": -11.066657066345215,
      "loss": 3.3225,
      "nll_loss": 3.297466993331909,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11356855928897858,
      "rewards/margins": 0.993097186088562,
      "rewards/rejected": -1.106665849685669,
      "step": 1677
    },
    {
      "epoch": 1.0438569206842925,
      "grad_norm": 0.5288723111152649,
      "learning_rate": 8.050000000000001e-06,
      "log_odds_chosen": 11.958285331726074,
      "log_odds_ratio": -0.07352694123983383,
      "logits/chosen": 0.2259128987789154,
      "logits/rejected": 1.3740233182907104,
      "logps/chosen": -1.3853117227554321,
      "logps/rejected": -13.046581268310547,
      "loss": 2.9374,
      "nll_loss": 2.930062770843506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1385311782360077,
      "rewards/margins": 1.1661269664764404,
      "rewards/rejected": -1.3046581745147705,
      "step": 1678
    },
    {
      "epoch": 1.0444790046656298,
      "grad_norm": 2.1460111141204834,
      "learning_rate": 8.025e-06,
      "log_odds_chosen": 10.04307746887207,
      "log_odds_ratio": -0.23023003339767456,
      "logits/chosen": 0.18102434277534485,
      "logits/rejected": 0.9223688840866089,
      "logps/chosen": -1.1232291460037231,
      "logps/rejected": -10.89987564086914,
      "loss": 2.447,
      "nll_loss": 2.423985481262207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11232291162014008,
      "rewards/margins": 0.9776646494865417,
      "rewards/rejected": -1.0899875164031982,
      "step": 1679
    },
    {
      "epoch": 1.0451010886469674,
      "grad_norm": 0.5670279264450073,
      "learning_rate": 8.000000000000001e-06,
      "log_odds_chosen": 7.932676315307617,
      "log_odds_ratio": -0.1673051118850708,
      "logits/chosen": 0.3382868766784668,
      "logits/rejected": 0.7066163420677185,
      "logps/chosen": -0.8997178077697754,
      "logps/rejected": -8.353787422180176,
      "loss": 3.5031,
      "nll_loss": 3.486398935317993,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08997179567813873,
      "rewards/margins": 0.745406985282898,
      "rewards/rejected": -0.8353787660598755,
      "step": 1680
    },
    {
      "epoch": 1.0457231726283047,
      "grad_norm": 0.5133948922157288,
      "learning_rate": 7.975e-06,
      "log_odds_chosen": 6.900932788848877,
      "log_odds_ratio": -0.2566588222980499,
      "logits/chosen": 0.30161863565444946,
      "logits/rejected": 0.512200117111206,
      "logps/chosen": -1.108983039855957,
      "logps/rejected": -7.704831123352051,
      "loss": 3.0651,
      "nll_loss": 3.0394201278686523,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11089830845594406,
      "rewards/margins": 0.6595847606658936,
      "rewards/rejected": -0.770483136177063,
      "step": 1681
    },
    {
      "epoch": 1.0463452566096423,
      "grad_norm": 1.7218198776245117,
      "learning_rate": 7.95e-06,
      "log_odds_chosen": 10.679391860961914,
      "log_odds_ratio": -0.2528221607208252,
      "logits/chosen": 0.16948404908180237,
      "logits/rejected": 0.7292956709861755,
      "logps/chosen": -1.3855730295181274,
      "logps/rejected": -11.849591255187988,
      "loss": 3.0227,
      "nll_loss": 2.997460126876831,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1385573148727417,
      "rewards/margins": 1.046401858329773,
      "rewards/rejected": -1.184959053993225,
      "step": 1682
    },
    {
      "epoch": 1.0469673405909798,
      "grad_norm": 0.543219804763794,
      "learning_rate": 7.925000000000001e-06,
      "log_odds_chosen": 11.195889472961426,
      "log_odds_ratio": -0.05250005051493645,
      "logits/chosen": 0.43830806016921997,
      "logits/rejected": 1.0013113021850586,
      "logps/chosen": -0.8631675243377686,
      "logps/rejected": -11.426183700561523,
      "loss": 3.1089,
      "nll_loss": 3.103605270385742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08631675690412521,
      "rewards/margins": 1.0563015937805176,
      "rewards/rejected": -1.1426184177398682,
      "step": 1683
    },
    {
      "epoch": 1.0475894245723172,
      "grad_norm": 0.6750419735908508,
      "learning_rate": 7.9e-06,
      "log_odds_chosen": 11.189361572265625,
      "log_odds_ratio": -0.07531260699033737,
      "logits/chosen": 0.3269883394241333,
      "logits/rejected": 1.2064783573150635,
      "logps/chosen": -0.7809778451919556,
      "logps/rejected": -11.376087188720703,
      "loss": 2.3984,
      "nll_loss": 2.3908650875091553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07809779047966003,
      "rewards/margins": 1.0595110654830933,
      "rewards/rejected": -1.1376087665557861,
      "step": 1684
    },
    {
      "epoch": 1.0482115085536547,
      "grad_norm": 0.4348224997520447,
      "learning_rate": 7.875e-06,
      "log_odds_chosen": 13.801148414611816,
      "log_odds_ratio": -0.08034741878509521,
      "logits/chosen": 0.3412080705165863,
      "logits/rejected": 1.4113205671310425,
      "logps/chosen": -1.0024620294570923,
      "logps/rejected": -14.352299690246582,
      "loss": 2.7022,
      "nll_loss": 2.6942036151885986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10024620592594147,
      "rewards/margins": 1.3349838256835938,
      "rewards/rejected": -1.435230016708374,
      "step": 1685
    },
    {
      "epoch": 1.0488335925349923,
      "grad_norm": 0.6142764687538147,
      "learning_rate": 7.850000000000001e-06,
      "log_odds_chosen": 10.19394302368164,
      "log_odds_ratio": -0.004159958567470312,
      "logits/chosen": 0.2460165172815323,
      "logits/rejected": 1.1073980331420898,
      "logps/chosen": -0.8783658742904663,
      "logps/rejected": -10.400548934936523,
      "loss": 2.5735,
      "nll_loss": 2.573049545288086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08783659338951111,
      "rewards/margins": 0.952218234539032,
      "rewards/rejected": -1.0400547981262207,
      "step": 1686
    },
    {
      "epoch": 1.0494556765163296,
      "grad_norm": 0.6551894545555115,
      "learning_rate": 7.825e-06,
      "log_odds_chosen": 10.567734718322754,
      "log_odds_ratio": -0.010052100755274296,
      "logits/chosen": 0.32441431283950806,
      "logits/rejected": 0.8118258714675903,
      "logps/chosen": -1.1712175607681274,
      "logps/rejected": -10.970331192016602,
      "loss": 3.2066,
      "nll_loss": 3.2055797576904297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11712174862623215,
      "rewards/margins": 0.9799113273620605,
      "rewards/rejected": -1.097033143043518,
      "step": 1687
    },
    {
      "epoch": 1.0500777604976672,
      "grad_norm": 1.7960695028305054,
      "learning_rate": 7.8e-06,
      "log_odds_chosen": 8.924139022827148,
      "log_odds_ratio": -0.19967907667160034,
      "logits/chosen": 0.18073192238807678,
      "logits/rejected": 0.5033700466156006,
      "logps/chosen": -0.9381715059280396,
      "logps/rejected": -9.414775848388672,
      "loss": 2.5335,
      "nll_loss": 2.513486862182617,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09381714463233948,
      "rewards/margins": 0.8476604223251343,
      "rewards/rejected": -0.9414775371551514,
      "step": 1688
    },
    {
      "epoch": 1.0506998444790048,
      "grad_norm": 0.657584011554718,
      "learning_rate": 7.775000000000001e-06,
      "log_odds_chosen": 8.136945724487305,
      "log_odds_ratio": -0.18999497592449188,
      "logits/chosen": 0.2717944085597992,
      "logits/rejected": 0.5513604283332825,
      "logps/chosen": -1.0118772983551025,
      "logps/rejected": -8.757270812988281,
      "loss": 2.5468,
      "nll_loss": 2.5278446674346924,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10118772834539413,
      "rewards/margins": 0.7745393514633179,
      "rewards/rejected": -0.8757270574569702,
      "step": 1689
    },
    {
      "epoch": 1.0513219284603421,
      "grad_norm": 0.7142519950866699,
      "learning_rate": 7.75e-06,
      "log_odds_chosen": 13.280343055725098,
      "log_odds_ratio": -0.0804281011223793,
      "logits/chosen": 0.40052324533462524,
      "logits/rejected": 1.358069896697998,
      "logps/chosen": -0.9685522317886353,
      "logps/rejected": -13.64314079284668,
      "loss": 2.8942,
      "nll_loss": 2.8861801624298096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09685522317886353,
      "rewards/margins": 1.2674589157104492,
      "rewards/rejected": -1.364314079284668,
      "step": 1690
    },
    {
      "epoch": 1.0519440124416797,
      "grad_norm": 0.5332468748092651,
      "learning_rate": 7.725e-06,
      "log_odds_chosen": 8.306745529174805,
      "log_odds_ratio": -0.25401604175567627,
      "logits/chosen": 0.29576265811920166,
      "logits/rejected": 0.6935917735099792,
      "logps/chosen": -1.3476217985153198,
      "logps/rejected": -9.482418060302734,
      "loss": 3.489,
      "nll_loss": 3.463611125946045,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13476218283176422,
      "rewards/margins": 0.8134796619415283,
      "rewards/rejected": -0.9482417702674866,
      "step": 1691
    },
    {
      "epoch": 1.052566096423017,
      "grad_norm": 0.5067847967147827,
      "learning_rate": 7.7e-06,
      "log_odds_chosen": 11.327463150024414,
      "log_odds_ratio": -0.09150741249322891,
      "logits/chosen": 0.3421590328216553,
      "logits/rejected": 0.8589677810668945,
      "logps/chosen": -1.1448135375976562,
      "logps/rejected": -12.03975772857666,
      "loss": 3.1256,
      "nll_loss": 3.116487979888916,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1144813597202301,
      "rewards/margins": 1.0894944667816162,
      "rewards/rejected": -1.2039759159088135,
      "step": 1692
    },
    {
      "epoch": 1.0531881804043546,
      "grad_norm": 0.45336204767227173,
      "learning_rate": 7.675e-06,
      "log_odds_chosen": 7.9443793296813965,
      "log_odds_ratio": -0.08496344089508057,
      "logits/chosen": 0.3444047272205353,
      "logits/rejected": 0.8683087229728699,
      "logps/chosen": -1.3129093647003174,
      "logps/rejected": -8.973153114318848,
      "loss": 3.3978,
      "nll_loss": 3.3893444538116455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13129094243049622,
      "rewards/margins": 0.7660244107246399,
      "rewards/rejected": -0.8973153233528137,
      "step": 1693
    },
    {
      "epoch": 1.0538102643856921,
      "grad_norm": 10.097654342651367,
      "learning_rate": 7.65e-06,
      "log_odds_chosen": 9.851930618286133,
      "log_odds_ratio": -0.30158087611198425,
      "logits/chosen": 0.27799010276794434,
      "logits/rejected": 0.9372337460517883,
      "logps/chosen": -2.7150497436523438,
      "logps/rejected": -12.091753005981445,
      "loss": 3.1269,
      "nll_loss": 3.0967228412628174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2715049684047699,
      "rewards/margins": 0.9376704096794128,
      "rewards/rejected": -1.2091753482818604,
      "step": 1694
    },
    {
      "epoch": 1.0544323483670295,
      "grad_norm": 0.6404469609260559,
      "learning_rate": 7.625e-06,
      "log_odds_chosen": 11.932069778442383,
      "log_odds_ratio": -0.016780562698841095,
      "logits/chosen": 0.2930798828601837,
      "logits/rejected": 0.8748061656951904,
      "logps/chosen": -0.9199110269546509,
      "logps/rejected": -12.094552993774414,
      "loss": 2.7043,
      "nll_loss": 2.7026607990264893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09199110418558121,
      "rewards/margins": 1.117464303970337,
      "rewards/rejected": -1.2094552516937256,
      "step": 1695
    },
    {
      "epoch": 1.055054432348367,
      "grad_norm": 0.672562301158905,
      "learning_rate": 7.6e-06,
      "log_odds_chosen": 12.345985412597656,
      "log_odds_ratio": -0.01769334264099598,
      "logits/chosen": 0.16025392711162567,
      "logits/rejected": 0.7841507792472839,
      "logps/chosen": -0.9052579998970032,
      "logps/rejected": -12.281292915344238,
      "loss": 2.5574,
      "nll_loss": 2.5555901527404785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0905258059501648,
      "rewards/margins": 1.137603521347046,
      "rewards/rejected": -1.2281293869018555,
      "step": 1696
    },
    {
      "epoch": 1.0556765163297046,
      "grad_norm": 0.7864720225334167,
      "learning_rate": 7.575e-06,
      "log_odds_chosen": 6.648435115814209,
      "log_odds_ratio": -0.21117697656154633,
      "logits/chosen": 0.25911492109298706,
      "logits/rejected": 0.46124711632728577,
      "logps/chosen": -1.0256266593933105,
      "logps/rejected": -7.215675354003906,
      "loss": 3.2426,
      "nll_loss": 3.2214407920837402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10256266593933105,
      "rewards/margins": 0.6190049052238464,
      "rewards/rejected": -0.7215675115585327,
      "step": 1697
    },
    {
      "epoch": 1.056298600311042,
      "grad_norm": 0.5210472345352173,
      "learning_rate": 7.55e-06,
      "log_odds_chosen": 6.7975172996521,
      "log_odds_ratio": -0.24025194346904755,
      "logits/chosen": 0.10606539249420166,
      "logits/rejected": 0.5300160050392151,
      "logps/chosen": -1.0408447980880737,
      "logps/rejected": -7.56657600402832,
      "loss": 2.8331,
      "nll_loss": 2.8090970516204834,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10408448427915573,
      "rewards/margins": 0.6525731086730957,
      "rewards/rejected": -0.7566576600074768,
      "step": 1698
    },
    {
      "epoch": 1.0569206842923795,
      "grad_norm": 1.3864878416061401,
      "learning_rate": 7.525e-06,
      "log_odds_chosen": 5.305638313293457,
      "log_odds_ratio": -0.3785892724990845,
      "logits/chosen": 0.11386309564113617,
      "logits/rejected": 0.3513365089893341,
      "logps/chosen": -1.1088687181472778,
      "logps/rejected": -6.139446258544922,
      "loss": 2.6211,
      "nll_loss": 2.5832529067993164,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11088687181472778,
      "rewards/margins": 0.5030577182769775,
      "rewards/rejected": -0.6139446496963501,
      "step": 1699
    },
    {
      "epoch": 1.0575427682737168,
      "grad_norm": 0.5979633927345276,
      "learning_rate": 7.5e-06,
      "log_odds_chosen": 8.369409561157227,
      "log_odds_ratio": -0.044509388506412506,
      "logits/chosen": 0.23924407362937927,
      "logits/rejected": 0.7259291410446167,
      "logps/chosen": -0.8297702670097351,
      "logps/rejected": -8.520917892456055,
      "loss": 2.8939,
      "nll_loss": 2.889413595199585,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08297702670097351,
      "rewards/margins": 0.7691147923469543,
      "rewards/rejected": -0.8520918488502502,
      "step": 1700
    },
    {
      "epoch": 1.0581648522550544,
      "grad_norm": 0.4679238796234131,
      "learning_rate": 7.4750000000000004e-06,
      "log_odds_chosen": 10.700567245483398,
      "log_odds_ratio": -0.007277395576238632,
      "logits/chosen": 0.2523863613605499,
      "logits/rejected": 0.8801952600479126,
      "logps/chosen": -1.0980184078216553,
      "logps/rejected": -11.33210563659668,
      "loss": 3.0063,
      "nll_loss": 3.005614757537842,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10980185121297836,
      "rewards/margins": 1.0234086513519287,
      "rewards/rejected": -1.1332106590270996,
      "step": 1701
    },
    {
      "epoch": 1.058786936236392,
      "grad_norm": 0.5518056154251099,
      "learning_rate": 7.45e-06,
      "log_odds_chosen": 8.354429244995117,
      "log_odds_ratio": -0.1485568732023239,
      "logits/chosen": 0.1999821811914444,
      "logits/rejected": 0.6293431520462036,
      "logps/chosen": -0.9266054034233093,
      "logps/rejected": -8.75076961517334,
      "loss": 2.8133,
      "nll_loss": 2.79842472076416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09266054630279541,
      "rewards/margins": 0.7824164628982544,
      "rewards/rejected": -0.8750770092010498,
      "step": 1702
    },
    {
      "epoch": 1.0594090202177293,
      "grad_norm": 0.48427486419677734,
      "learning_rate": 7.425e-06,
      "log_odds_chosen": 9.299489974975586,
      "log_odds_ratio": -0.12962597608566284,
      "logits/chosen": 0.3212866485118866,
      "logits/rejected": 0.7380868196487427,
      "logps/chosen": -0.8832244277000427,
      "logps/rejected": -9.699041366577148,
      "loss": 3.5157,
      "nll_loss": 3.5027828216552734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08832244575023651,
      "rewards/margins": 0.8815817832946777,
      "rewards/rejected": -0.9699042439460754,
      "step": 1703
    },
    {
      "epoch": 1.0600311041990669,
      "grad_norm": 0.5466374754905701,
      "learning_rate": 7.4e-06,
      "log_odds_chosen": 12.918460845947266,
      "log_odds_ratio": -0.004811026621609926,
      "logits/chosen": 0.23254846036434174,
      "logits/rejected": 1.191979169845581,
      "logps/chosen": -0.8864061832427979,
      "logps/rejected": -13.163785934448242,
      "loss": 2.6641,
      "nll_loss": 2.663635015487671,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08864062279462814,
      "rewards/margins": 1.2277379035949707,
      "rewards/rejected": -1.3163785934448242,
      "step": 1704
    },
    {
      "epoch": 1.0606531881804044,
      "grad_norm": 0.6290574669837952,
      "learning_rate": 7.375e-06,
      "log_odds_chosen": 10.326766967773438,
      "log_odds_ratio": -0.010386736132204533,
      "logits/chosen": 0.3104044198989868,
      "logits/rejected": 1.3550554513931274,
      "logps/chosen": -1.036363124847412,
      "logps/rejected": -10.91328239440918,
      "loss": 2.2605,
      "nll_loss": 2.25947642326355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10363630950450897,
      "rewards/margins": 0.9876919984817505,
      "rewards/rejected": -1.0913283824920654,
      "step": 1705
    },
    {
      "epoch": 1.0612752721617418,
      "grad_norm": 0.6645586490631104,
      "learning_rate": 7.35e-06,
      "log_odds_chosen": 4.954842567443848,
      "log_odds_ratio": -0.29886144399642944,
      "logits/chosen": 0.36008620262145996,
      "logits/rejected": 0.6080492734909058,
      "logps/chosen": -0.9095826745033264,
      "logps/rejected": -5.422713279724121,
      "loss": 3.1555,
      "nll_loss": 3.1255886554718018,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09095826745033264,
      "rewards/margins": 0.45131298899650574,
      "rewards/rejected": -0.5422712564468384,
      "step": 1706
    },
    {
      "epoch": 1.0618973561430793,
      "grad_norm": 0.5843470096588135,
      "learning_rate": 7.325e-06,
      "log_odds_chosen": 8.344114303588867,
      "log_odds_ratio": -0.15374304354190826,
      "logits/chosen": 0.26468321681022644,
      "logits/rejected": 0.9158927202224731,
      "logps/chosen": -1.0647417306900024,
      "logps/rejected": -9.070698738098145,
      "loss": 2.8046,
      "nll_loss": 2.789252996444702,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10647417604923248,
      "rewards/margins": 0.800595760345459,
      "rewards/rejected": -0.9070699214935303,
      "step": 1707
    },
    {
      "epoch": 1.062519440124417,
      "grad_norm": 0.5199341773986816,
      "learning_rate": 7.2999999999999996e-06,
      "log_odds_chosen": 11.067071914672852,
      "log_odds_ratio": -0.00585298053920269,
      "logits/chosen": 0.23164747655391693,
      "logits/rejected": 0.9034661650657654,
      "logps/chosen": -1.055686116218567,
      "logps/rejected": -11.569263458251953,
      "loss": 2.746,
      "nll_loss": 2.7453653812408447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10556861758232117,
      "rewards/margins": 1.0513577461242676,
      "rewards/rejected": -1.1569262742996216,
      "step": 1708
    },
    {
      "epoch": 1.0631415241057542,
      "grad_norm": 0.5655837655067444,
      "learning_rate": 7.275e-06,
      "log_odds_chosen": 11.287644386291504,
      "log_odds_ratio": -0.0015687721315771341,
      "logits/chosen": 0.289143830537796,
      "logits/rejected": 0.7014958262443542,
      "logps/chosen": -1.0149272680282593,
      "logps/rejected": -11.732308387756348,
      "loss": 3.1581,
      "nll_loss": 3.157970905303955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.101492740213871,
      "rewards/margins": 1.0717381238937378,
      "rewards/rejected": -1.1732308864593506,
      "step": 1709
    },
    {
      "epoch": 1.0637636080870918,
      "grad_norm": 0.5841426849365234,
      "learning_rate": 7.25e-06,
      "log_odds_chosen": 13.971756935119629,
      "log_odds_ratio": -0.0001850881817517802,
      "logits/chosen": 0.5528026819229126,
      "logits/rejected": 1.5337713956832886,
      "logps/chosen": -1.308842420578003,
      "logps/rejected": -14.919255256652832,
      "loss": 3.5475,
      "nll_loss": 3.547466993331909,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13088424503803253,
      "rewards/margins": 1.3610413074493408,
      "rewards/rejected": -1.491925597190857,
      "step": 1710
    },
    {
      "epoch": 1.0643856920684291,
      "grad_norm": 0.602485179901123,
      "learning_rate": 7.2249999999999994e-06,
      "log_odds_chosen": 10.762537956237793,
      "log_odds_ratio": -0.018083693459630013,
      "logits/chosen": 0.3396477699279785,
      "logits/rejected": 1.0870765447616577,
      "logps/chosen": -0.8417822122573853,
      "logps/rejected": -10.953207015991211,
      "loss": 3.0177,
      "nll_loss": 3.015902042388916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08417822420597076,
      "rewards/margins": 1.0111424922943115,
      "rewards/rejected": -1.0953208208084106,
      "step": 1711
    },
    {
      "epoch": 1.0650077760497667,
      "grad_norm": 1.0022923946380615,
      "learning_rate": 7.2e-06,
      "log_odds_chosen": 9.395156860351562,
      "log_odds_ratio": -0.17347288131713867,
      "logits/chosen": 0.23961828649044037,
      "logits/rejected": 1.10887610912323,
      "logps/chosen": -1.5142661333084106,
      "logps/rejected": -10.611567497253418,
      "loss": 2.423,
      "nll_loss": 2.4056031703948975,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15142662823200226,
      "rewards/margins": 0.909730076789856,
      "rewards/rejected": -1.0611567497253418,
      "step": 1712
    },
    {
      "epoch": 1.0656298600311043,
      "grad_norm": 0.7605722546577454,
      "learning_rate": 7.175e-06,
      "log_odds_chosen": 9.183586120605469,
      "log_odds_ratio": -0.20111922919750214,
      "logits/chosen": 0.30907413363456726,
      "logits/rejected": 0.8606806993484497,
      "logps/chosen": -1.424899697303772,
      "logps/rejected": -10.347264289855957,
      "loss": 3.4657,
      "nll_loss": 3.4456028938293457,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1424899697303772,
      "rewards/margins": 0.8922364711761475,
      "rewards/rejected": -1.0347263813018799,
      "step": 1713
    },
    {
      "epoch": 1.0662519440124416,
      "grad_norm": 0.3997776508331299,
      "learning_rate": 7.15e-06,
      "log_odds_chosen": 11.275423049926758,
      "log_odds_ratio": -0.009868920780718327,
      "logits/chosen": 0.34685301780700684,
      "logits/rejected": 1.0028785467147827,
      "logps/chosen": -1.0552277565002441,
      "logps/rejected": -11.810014724731445,
      "loss": 3.5462,
      "nll_loss": 3.5452332496643066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10552278161048889,
      "rewards/margins": 1.0754787921905518,
      "rewards/rejected": -1.1810015439987183,
      "step": 1714
    },
    {
      "epoch": 1.0668740279937792,
      "grad_norm": 0.4696863889694214,
      "learning_rate": 7.1249999999999995e-06,
      "log_odds_chosen": 10.42939567565918,
      "log_odds_ratio": -0.0563388392329216,
      "logits/chosen": 0.36971479654312134,
      "logits/rejected": 1.1650364398956299,
      "logps/chosen": -1.241980791091919,
      "logps/rejected": -11.253006935119629,
      "loss": 3.3682,
      "nll_loss": 3.3625478744506836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1241980716586113,
      "rewards/margins": 1.0011026859283447,
      "rewards/rejected": -1.1253007650375366,
      "step": 1715
    },
    {
      "epoch": 1.0674961119751167,
      "grad_norm": 0.8792657256126404,
      "learning_rate": 7.1e-06,
      "log_odds_chosen": 9.294808387756348,
      "log_odds_ratio": -0.03534835949540138,
      "logits/chosen": 0.25297093391418457,
      "logits/rejected": 0.5003412365913391,
      "logps/chosen": -0.8526089191436768,
      "logps/rejected": -9.356329917907715,
      "loss": 2.3576,
      "nll_loss": 2.354109287261963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08526089787483215,
      "rewards/margins": 0.8503720760345459,
      "rewards/rejected": -0.9356330037117004,
      "step": 1716
    },
    {
      "epoch": 1.068118195956454,
      "grad_norm": 0.521109402179718,
      "learning_rate": 7.075e-06,
      "log_odds_chosen": 14.6859712600708,
      "log_odds_ratio": -0.00011647976498352364,
      "logits/chosen": 0.1722133308649063,
      "logits/rejected": 0.9801969528198242,
      "logps/chosen": -1.2429990768432617,
      "logps/rejected": -15.523116111755371,
      "loss": 3.2801,
      "nll_loss": 3.2801103591918945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12429991364479065,
      "rewards/margins": 1.4280118942260742,
      "rewards/rejected": -1.552311658859253,
      "step": 1717
    },
    {
      "epoch": 1.0687402799377916,
      "grad_norm": 0.5586221218109131,
      "learning_rate": 7.049999999999999e-06,
      "log_odds_chosen": 7.838666915893555,
      "log_odds_ratio": -0.08945189416408539,
      "logits/chosen": 0.2482565939426422,
      "logits/rejected": 0.694401741027832,
      "logps/chosen": -1.0721337795257568,
      "logps/rejected": -8.161520004272461,
      "loss": 2.4886,
      "nll_loss": 2.479682445526123,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10721337050199509,
      "rewards/margins": 0.7089386582374573,
      "rewards/rejected": -0.8161520957946777,
      "step": 1718
    },
    {
      "epoch": 1.069362363919129,
      "grad_norm": 0.5216894745826721,
      "learning_rate": 7.025000000000001e-06,
      "log_odds_chosen": 11.799904823303223,
      "log_odds_ratio": -0.002399260178208351,
      "logits/chosen": 0.2988235354423523,
      "logits/rejected": 1.5455994606018066,
      "logps/chosen": -1.177297592163086,
      "logps/rejected": -12.587946891784668,
      "loss": 2.7591,
      "nll_loss": 2.7588682174682617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11772976815700531,
      "rewards/margins": 1.141065001487732,
      "rewards/rejected": -1.2587947845458984,
      "step": 1719
    },
    {
      "epoch": 1.0699844479004665,
      "grad_norm": 0.6003878712654114,
      "learning_rate": 7.000000000000001e-06,
      "log_odds_chosen": 8.443526268005371,
      "log_odds_ratio": -0.12604990601539612,
      "logits/chosen": 0.3089401125907898,
      "logits/rejected": 1.1080322265625,
      "logps/chosen": -1.531690239906311,
      "logps/rejected": -9.769110679626465,
      "loss": 3.1147,
      "nll_loss": 3.1020569801330566,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15316902101039886,
      "rewards/margins": 0.8237420320510864,
      "rewards/rejected": -0.9769110679626465,
      "step": 1720
    },
    {
      "epoch": 1.070606531881804,
      "grad_norm": 0.5633937120437622,
      "learning_rate": 6.975000000000001e-06,
      "log_odds_chosen": 12.794734001159668,
      "log_odds_ratio": -0.0027537632267922163,
      "logits/chosen": 0.21119673550128937,
      "logits/rejected": 1.5868358612060547,
      "logps/chosen": -1.1883983612060547,
      "logps/rejected": -13.517034530639648,
      "loss": 2.517,
      "nll_loss": 2.5166783332824707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11883984506130219,
      "rewards/margins": 1.2328636646270752,
      "rewards/rejected": -1.351703405380249,
      "step": 1721
    },
    {
      "epoch": 1.0712286158631414,
      "grad_norm": 0.45499175786972046,
      "learning_rate": 6.950000000000001e-06,
      "log_odds_chosen": 16.033639907836914,
      "log_odds_ratio": -0.00026129043544642627,
      "logits/chosen": 0.09925095736980438,
      "logits/rejected": 0.7012900114059448,
      "logps/chosen": -1.0431275367736816,
      "logps/rejected": -16.531070709228516,
      "loss": 2.851,
      "nll_loss": 2.8509817123413086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10431274771690369,
      "rewards/margins": 1.5487942695617676,
      "rewards/rejected": -1.653106927871704,
      "step": 1722
    },
    {
      "epoch": 1.071850699844479,
      "grad_norm": 0.5466725826263428,
      "learning_rate": 6.925000000000001e-06,
      "log_odds_chosen": 11.074979782104492,
      "log_odds_ratio": -0.04315614700317383,
      "logits/chosen": 0.14270518720149994,
      "logits/rejected": 0.8532730340957642,
      "logps/chosen": -0.9979894161224365,
      "logps/rejected": -11.582146644592285,
      "loss": 2.9812,
      "nll_loss": 2.976850986480713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09979893267154694,
      "rewards/margins": 1.0584158897399902,
      "rewards/rejected": -1.158214807510376,
      "step": 1723
    },
    {
      "epoch": 1.0724727838258166,
      "grad_norm": 0.4190523326396942,
      "learning_rate": 6.900000000000001e-06,
      "log_odds_chosen": 13.74249267578125,
      "log_odds_ratio": -0.01285548135638237,
      "logits/chosen": 0.2155817151069641,
      "logits/rejected": 0.9092065691947937,
      "logps/chosen": -0.9522172212600708,
      "logps/rejected": -14.18885612487793,
      "loss": 3.0457,
      "nll_loss": 3.0444235801696777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09522172808647156,
      "rewards/margins": 1.3236639499664307,
      "rewards/rejected": -1.4188857078552246,
      "step": 1724
    },
    {
      "epoch": 1.073094867807154,
      "grad_norm": 1.1821922063827515,
      "learning_rate": 6.875000000000001e-06,
      "log_odds_chosen": 13.672684669494629,
      "log_odds_ratio": -0.08438721299171448,
      "logits/chosen": 0.347389817237854,
      "logits/rejected": 1.5360487699508667,
      "logps/chosen": -1.1511101722717285,
      "logps/rejected": -14.482137680053711,
      "loss": 2.6498,
      "nll_loss": 2.6413512229919434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11511103063821793,
      "rewards/margins": 1.333102822303772,
      "rewards/rejected": -1.448213815689087,
      "step": 1725
    },
    {
      "epoch": 1.0737169517884915,
      "grad_norm": 0.5827188491821289,
      "learning_rate": 6.8500000000000005e-06,
      "log_odds_chosen": 13.990209579467773,
      "log_odds_ratio": -0.07931892573833466,
      "logits/chosen": 0.3765639662742615,
      "logits/rejected": 1.4914324283599854,
      "logps/chosen": -0.8575527667999268,
      "logps/rejected": -14.12916374206543,
      "loss": 3.0849,
      "nll_loss": 3.076958179473877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08575528115034103,
      "rewards/margins": 1.3271610736846924,
      "rewards/rejected": -1.4129164218902588,
      "step": 1726
    },
    {
      "epoch": 1.074339035769829,
      "grad_norm": 0.6240293383598328,
      "learning_rate": 6.825000000000001e-06,
      "log_odds_chosen": 15.126672744750977,
      "log_odds_ratio": -0.0863954946398735,
      "logits/chosen": 0.1938783824443817,
      "logits/rejected": 1.443101167678833,
      "logps/chosen": -1.1526148319244385,
      "logps/rejected": -15.915487289428711,
      "loss": 2.5008,
      "nll_loss": 2.4921345710754395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11526148021221161,
      "rewards/margins": 1.4762872457504272,
      "rewards/rejected": -1.5915489196777344,
      "step": 1727
    },
    {
      "epoch": 1.0749611197511664,
      "grad_norm": 9.35839557647705,
      "learning_rate": 6.800000000000001e-06,
      "log_odds_chosen": 12.475717544555664,
      "log_odds_ratio": -0.039579086005687714,
      "logits/chosen": 0.3538586497306824,
      "logits/rejected": 1.454434871673584,
      "logps/chosen": -1.897247552871704,
      "logps/rejected": -14.156525611877441,
      "loss": 3.1319,
      "nll_loss": 3.1279637813568115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18972477316856384,
      "rewards/margins": 1.225927710533142,
      "rewards/rejected": -1.4156526327133179,
      "step": 1728
    },
    {
      "epoch": 1.075583203732504,
      "grad_norm": 1.0323598384857178,
      "learning_rate": 6.775000000000001e-06,
      "log_odds_chosen": 16.044645309448242,
      "log_odds_ratio": -0.0008165925974026322,
      "logits/chosen": 0.22339317202568054,
      "logits/rejected": 1.895038366317749,
      "logps/chosen": -1.209045648574829,
      "logps/rejected": -16.88253402709961,
      "loss": 2.3606,
      "nll_loss": 2.3605496883392334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12090456485748291,
      "rewards/margins": 1.5673489570617676,
      "rewards/rejected": -1.6882535219192505,
      "step": 1729
    },
    {
      "epoch": 1.0762052877138413,
      "grad_norm": 0.4422605335712433,
      "learning_rate": 6.750000000000001e-06,
      "log_odds_chosen": 13.798027992248535,
      "log_odds_ratio": -6.737739749951288e-05,
      "logits/chosen": 0.34486210346221924,
      "logits/rejected": 1.331770420074463,
      "logps/chosen": -1.1659202575683594,
      "logps/rejected": -14.552068710327148,
      "loss": 3.3028,
      "nll_loss": 3.302793502807617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11659201979637146,
      "rewards/margins": 1.3386149406433105,
      "rewards/rejected": -1.4552068710327148,
      "step": 1730
    },
    {
      "epoch": 1.0768273716951788,
      "grad_norm": 0.6502666473388672,
      "learning_rate": 6.725000000000001e-06,
      "log_odds_chosen": 12.585850715637207,
      "log_odds_ratio": -0.11541411280632019,
      "logits/chosen": 0.11757244169712067,
      "logits/rejected": 0.8546798229217529,
      "logps/chosen": -1.0822380781173706,
      "logps/rejected": -13.226578712463379,
      "loss": 2.4227,
      "nll_loss": 2.411139965057373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1082238182425499,
      "rewards/margins": 1.2144341468811035,
      "rewards/rejected": -1.3226580619812012,
      "step": 1731
    },
    {
      "epoch": 1.0774494556765164,
      "grad_norm": 0.5105128288269043,
      "learning_rate": 6.700000000000001e-06,
      "log_odds_chosen": 15.417603492736816,
      "log_odds_ratio": -0.000615728902630508,
      "logits/chosen": 0.3644036054611206,
      "logits/rejected": 1.0982917547225952,
      "logps/chosen": -1.0642772912979126,
      "logps/rejected": -15.99770450592041,
      "loss": 3.3102,
      "nll_loss": 3.3101632595062256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10642772912979126,
      "rewards/margins": 1.493342638015747,
      "rewards/rejected": -1.5997705459594727,
      "step": 1732
    },
    {
      "epoch": 1.0780715396578537,
      "grad_norm": 0.5010104179382324,
      "learning_rate": 6.6750000000000005e-06,
      "log_odds_chosen": 10.825895309448242,
      "log_odds_ratio": -0.10436728596687317,
      "logits/chosen": 0.10802976787090302,
      "logits/rejected": 0.7807521820068359,
      "logps/chosen": -1.1920297145843506,
      "logps/rejected": -11.657849311828613,
      "loss": 3.0259,
      "nll_loss": 3.0154356956481934,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11920297890901566,
      "rewards/margins": 1.0465819835662842,
      "rewards/rejected": -1.1657850742340088,
      "step": 1733
    },
    {
      "epoch": 1.0786936236391913,
      "grad_norm": 0.5057395100593567,
      "learning_rate": 6.650000000000001e-06,
      "log_odds_chosen": 11.468727111816406,
      "log_odds_ratio": -0.12807990610599518,
      "logits/chosen": 0.13424773514270782,
      "logits/rejected": 0.758084237575531,
      "logps/chosen": -0.8967731595039368,
      "logps/rejected": -11.590532302856445,
      "loss": 2.8589,
      "nll_loss": 2.846139430999756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08967731893062592,
      "rewards/margins": 1.0693758726119995,
      "rewards/rejected": -1.159053087234497,
      "step": 1734
    },
    {
      "epoch": 1.0793157076205289,
      "grad_norm": 0.6509506702423096,
      "learning_rate": 6.625000000000001e-06,
      "log_odds_chosen": 10.882062911987305,
      "log_odds_ratio": -0.012730767950415611,
      "logits/chosen": 0.44164884090423584,
      "logits/rejected": 1.0146621465682983,
      "logps/chosen": -1.0620293617248535,
      "logps/rejected": -11.419351577758789,
      "loss": 3.15,
      "nll_loss": 3.14876651763916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10620293766260147,
      "rewards/margins": 1.0357321500778198,
      "rewards/rejected": -1.141935110092163,
      "step": 1735
    },
    {
      "epoch": 1.0799377916018662,
      "grad_norm": 10.22143840789795,
      "learning_rate": 6.6e-06,
      "log_odds_chosen": 12.94996452331543,
      "log_odds_ratio": -0.32687464356422424,
      "logits/chosen": 0.186806783080101,
      "logits/rejected": 0.8277633190155029,
      "logps/chosen": -1.3755271434783936,
      "logps/rejected": -14.012967109680176,
      "loss": 2.9523,
      "nll_loss": 2.9196386337280273,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13755272328853607,
      "rewards/margins": 1.2637439966201782,
      "rewards/rejected": -1.4012967348098755,
      "step": 1736
    },
    {
      "epoch": 1.0805598755832038,
      "grad_norm": 0.6058777570724487,
      "learning_rate": 6.5750000000000006e-06,
      "log_odds_chosen": 10.573338508605957,
      "log_odds_ratio": -0.22641825675964355,
      "logits/chosen": 0.17677393555641174,
      "logits/rejected": 0.6988618969917297,
      "logps/chosen": -1.1388049125671387,
      "logps/rejected": -11.44548225402832,
      "loss": 2.9047,
      "nll_loss": 2.882017135620117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11388048529624939,
      "rewards/margins": 1.030667781829834,
      "rewards/rejected": -1.1445482969284058,
      "step": 1737
    },
    {
      "epoch": 1.081181959564541,
      "grad_norm": 0.55973881483078,
      "learning_rate": 6.550000000000001e-06,
      "log_odds_chosen": 11.361783981323242,
      "log_odds_ratio": -0.025486905127763748,
      "logits/chosen": 0.2761267125606537,
      "logits/rejected": 1.423222303390503,
      "logps/chosen": -1.079453468322754,
      "logps/rejected": -12.022642135620117,
      "loss": 2.6204,
      "nll_loss": 2.6178572177886963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10794536024332047,
      "rewards/margins": 1.0943188667297363,
      "rewards/rejected": -1.2022643089294434,
      "step": 1738
    },
    {
      "epoch": 1.0818040435458787,
      "grad_norm": 0.41827815771102905,
      "learning_rate": 6.525e-06,
      "log_odds_chosen": 11.45641040802002,
      "log_odds_ratio": -0.04113250598311424,
      "logits/chosen": 0.28797030448913574,
      "logits/rejected": 0.9413681030273438,
      "logps/chosen": -1.1444040536880493,
      "logps/rejected": -12.222957611083984,
      "loss": 3.2833,
      "nll_loss": 3.279160261154175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11444041132926941,
      "rewards/margins": 1.1078553199768066,
      "rewards/rejected": -1.2222957611083984,
      "step": 1739
    },
    {
      "epoch": 1.0824261275272162,
      "grad_norm": 0.5239333510398865,
      "learning_rate": 6.5000000000000004e-06,
      "log_odds_chosen": 10.370500564575195,
      "log_odds_ratio": -0.010260947048664093,
      "logits/chosen": 0.1700163036584854,
      "logits/rejected": 0.7396705150604248,
      "logps/chosen": -0.9648128151893616,
      "logps/rejected": -10.788455963134766,
      "loss": 2.8972,
      "nll_loss": 2.8962013721466064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09648127853870392,
      "rewards/margins": 0.9823644161224365,
      "rewards/rejected": -1.078845739364624,
      "step": 1740
    },
    {
      "epoch": 1.0830482115085536,
      "grad_norm": 0.5762307643890381,
      "learning_rate": 6.475000000000001e-06,
      "log_odds_chosen": 6.935182571411133,
      "log_odds_ratio": -0.08945510536432266,
      "logits/chosen": 0.16429941356182098,
      "logits/rejected": 0.6121139526367188,
      "logps/chosen": -0.9953298568725586,
      "logps/rejected": -7.245857238769531,
      "loss": 2.8649,
      "nll_loss": 2.8559913635253906,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09953299164772034,
      "rewards/margins": 0.625052809715271,
      "rewards/rejected": -0.7245857119560242,
      "step": 1741
    },
    {
      "epoch": 1.0836702954898911,
      "grad_norm": 0.7302781343460083,
      "learning_rate": 6.45e-06,
      "log_odds_chosen": 13.287824630737305,
      "log_odds_ratio": -0.07792376726865768,
      "logits/chosen": 0.07993796467781067,
      "logits/rejected": 1.3684005737304688,
      "logps/chosen": -1.4768130779266357,
      "logps/rejected": -14.53031063079834,
      "loss": 2.8254,
      "nll_loss": 2.8176429271698,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1476813107728958,
      "rewards/margins": 1.3053498268127441,
      "rewards/rejected": -1.453031063079834,
      "step": 1742
    },
    {
      "epoch": 1.0842923794712287,
      "grad_norm": 0.5578170418739319,
      "learning_rate": 6.425e-06,
      "log_odds_chosen": 13.632564544677734,
      "log_odds_ratio": -0.06082217022776604,
      "logits/chosen": 0.33293527364730835,
      "logits/rejected": 1.096616506576538,
      "logps/chosen": -0.7652254700660706,
      "logps/rejected": -13.640064239501953,
      "loss": 3.1935,
      "nll_loss": 3.1874160766601562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07652254402637482,
      "rewards/margins": 1.2874839305877686,
      "rewards/rejected": -1.364006519317627,
      "step": 1743
    },
    {
      "epoch": 1.084914463452566,
      "grad_norm": 0.7817727327346802,
      "learning_rate": 6.4000000000000006e-06,
      "log_odds_chosen": 13.825263977050781,
      "log_odds_ratio": -0.10829272121191025,
      "logits/chosen": 0.18306320905685425,
      "logits/rejected": 1.2139906883239746,
      "logps/chosen": -1.0914957523345947,
      "logps/rejected": -14.526753425598145,
      "loss": 2.5395,
      "nll_loss": 2.5286498069763184,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10914956778287888,
      "rewards/margins": 1.343525767326355,
      "rewards/rejected": -1.452675223350525,
      "step": 1744
    },
    {
      "epoch": 1.0855365474339036,
      "grad_norm": 0.6816715598106384,
      "learning_rate": 6.375000000000001e-06,
      "log_odds_chosen": 8.512453079223633,
      "log_odds_ratio": -0.23665356636047363,
      "logits/chosen": 0.20159471035003662,
      "logits/rejected": 0.7449344396591187,
      "logps/chosen": -1.1736276149749756,
      "logps/rejected": -9.346839904785156,
      "loss": 2.8251,
      "nll_loss": 2.8014824390411377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11736276000738144,
      "rewards/margins": 0.817321240901947,
      "rewards/rejected": -0.9346840381622314,
      "step": 1745
    },
    {
      "epoch": 1.0861586314152412,
      "grad_norm": 0.6403155326843262,
      "learning_rate": 6.35e-06,
      "log_odds_chosen": 14.663322448730469,
      "log_odds_ratio": -0.0037884623743593693,
      "logits/chosen": 0.279615193605423,
      "logits/rejected": 1.2483148574829102,
      "logps/chosen": -0.8240023851394653,
      "logps/rejected": -14.657256126403809,
      "loss": 2.7768,
      "nll_loss": 2.7764101028442383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08240024000406265,
      "rewards/margins": 1.3833253383636475,
      "rewards/rejected": -1.4657256603240967,
      "step": 1746
    },
    {
      "epoch": 1.0867807153965785,
      "grad_norm": 0.5294352769851685,
      "learning_rate": 6.3250000000000004e-06,
      "log_odds_chosen": 7.447414398193359,
      "log_odds_ratio": -0.08214883506298065,
      "logits/chosen": 0.04716228321194649,
      "logits/rejected": 0.14632737636566162,
      "logps/chosen": -0.844534158706665,
      "logps/rejected": -7.672712326049805,
      "loss": 2.7177,
      "nll_loss": 2.709484338760376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08445341885089874,
      "rewards/margins": 0.682817816734314,
      "rewards/rejected": -0.7672712802886963,
      "step": 1747
    },
    {
      "epoch": 1.087402799377916,
      "grad_norm": 0.5357289910316467,
      "learning_rate": 6.300000000000001e-06,
      "log_odds_chosen": 9.260400772094727,
      "log_odds_ratio": -0.21293191611766815,
      "logits/chosen": 0.44084519147872925,
      "logits/rejected": 1.00531005859375,
      "logps/chosen": -1.097980260848999,
      "logps/rejected": -9.951017379760742,
      "loss": 3.4195,
      "nll_loss": 3.3982417583465576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10979802906513214,
      "rewards/margins": 0.8853037357330322,
      "rewards/rejected": -0.9951016902923584,
      "step": 1748
    },
    {
      "epoch": 1.0880248833592534,
      "grad_norm": 0.5099205374717712,
      "learning_rate": 6.275e-06,
      "log_odds_chosen": 10.247150421142578,
      "log_odds_ratio": -0.09444563835859299,
      "logits/chosen": 0.3867758512496948,
      "logits/rejected": 1.0003869533538818,
      "logps/chosen": -1.117705225944519,
      "logps/rejected": -10.993670463562012,
      "loss": 3.3419,
      "nll_loss": 3.332432985305786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11177052557468414,
      "rewards/margins": 0.9875965714454651,
      "rewards/rejected": -1.0993670225143433,
      "step": 1749
    },
    {
      "epoch": 1.088646967340591,
      "grad_norm": 0.7787434458732605,
      "learning_rate": 6.25e-06,
      "log_odds_chosen": 9.081072807312012,
      "log_odds_ratio": -0.27854081988334656,
      "logits/chosen": 0.20520712435245514,
      "logits/rejected": 0.7103989124298096,
      "logps/chosen": -1.0034761428833008,
      "logps/rejected": -9.524426460266113,
      "loss": 2.9065,
      "nll_loss": 2.8786349296569824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1003476157784462,
      "rewards/margins": 0.8520950675010681,
      "rewards/rejected": -0.9524427056312561,
      "step": 1750
    },
    {
      "epoch": 1.0892690513219285,
      "grad_norm": 0.5483114719390869,
      "learning_rate": 6.2250000000000005e-06,
      "log_odds_chosen": 14.3975191116333,
      "log_odds_ratio": -0.0014387951232492924,
      "logits/chosen": 0.19464844465255737,
      "logits/rejected": 0.9625054597854614,
      "logps/chosen": -0.960172176361084,
      "logps/rejected": -14.856963157653809,
      "loss": 2.7555,
      "nll_loss": 2.7553367614746094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09601722657680511,
      "rewards/margins": 1.389679193496704,
      "rewards/rejected": -1.4856964349746704,
      "step": 1751
    },
    {
      "epoch": 1.0898911353032659,
      "grad_norm": 0.7367234230041504,
      "learning_rate": 6.2e-06,
      "log_odds_chosen": 11.96536922454834,
      "log_odds_ratio": -0.005536233074963093,
      "logits/chosen": 0.3735648989677429,
      "logits/rejected": 0.7411990761756897,
      "logps/chosen": -1.0733413696289062,
      "logps/rejected": -12.566356658935547,
      "loss": 2.6859,
      "nll_loss": 2.6853647232055664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10733415186405182,
      "rewards/margins": 1.1493016481399536,
      "rewards/rejected": -1.2566356658935547,
      "step": 1752
    },
    {
      "epoch": 1.0905132192846034,
      "grad_norm": 0.5547901391983032,
      "learning_rate": 6.175e-06,
      "log_odds_chosen": 14.394362449645996,
      "log_odds_ratio": -0.08104939758777618,
      "logits/chosen": 0.31781625747680664,
      "logits/rejected": 1.255645751953125,
      "logps/chosen": -1.1181135177612305,
      "logps/rejected": -15.128317832946777,
      "loss": 2.5999,
      "nll_loss": 2.5917768478393555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11181135475635529,
      "rewards/margins": 1.4010204076766968,
      "rewards/rejected": -1.512831687927246,
      "step": 1753
    },
    {
      "epoch": 1.091135303265941,
      "grad_norm": 0.7209290862083435,
      "learning_rate": 6.15e-06,
      "log_odds_chosen": 13.295998573303223,
      "log_odds_ratio": -0.05274088308215141,
      "logits/chosen": 0.2680150866508484,
      "logits/rejected": 0.8705407381057739,
      "logps/chosen": -1.395832896232605,
      "logps/rejected": -14.312849998474121,
      "loss": 3.0833,
      "nll_loss": 3.078002452850342,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1395832896232605,
      "rewards/margins": 1.2917016744613647,
      "rewards/rejected": -1.43128502368927,
      "step": 1754
    },
    {
      "epoch": 1.0917573872472783,
      "grad_norm": 0.6560238003730774,
      "learning_rate": 6.125e-06,
      "log_odds_chosen": 12.961246490478516,
      "log_odds_ratio": -0.12485496699810028,
      "logits/chosen": 0.1595935821533203,
      "logits/rejected": 1.1436913013458252,
      "logps/chosen": -1.2793207168579102,
      "logps/rejected": -13.835441589355469,
      "loss": 2.8327,
      "nll_loss": 2.8201873302459717,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12793207168579102,
      "rewards/margins": 1.2556120157241821,
      "rewards/rejected": -1.3835442066192627,
      "step": 1755
    },
    {
      "epoch": 1.092379471228616,
      "grad_norm": 0.5627850294113159,
      "learning_rate": 6.1e-06,
      "log_odds_chosen": 10.657611846923828,
      "log_odds_ratio": -0.07279589027166367,
      "logits/chosen": 0.5128650665283203,
      "logits/rejected": 1.0891677141189575,
      "logps/chosen": -1.0589655637741089,
      "logps/rejected": -11.189449310302734,
      "loss": 3.5616,
      "nll_loss": 3.554288625717163,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10589656233787537,
      "rewards/margins": 1.0130484104156494,
      "rewards/rejected": -1.1189448833465576,
      "step": 1756
    },
    {
      "epoch": 1.0930015552099532,
      "grad_norm": 0.5608973503112793,
      "learning_rate": 6.075e-06,
      "log_odds_chosen": 14.850683212280273,
      "log_odds_ratio": -0.14289139211177826,
      "logits/chosen": 0.2707276940345764,
      "logits/rejected": 1.1643056869506836,
      "logps/chosen": -1.0261825323104858,
      "logps/rejected": -15.539467811584473,
      "loss": 2.7926,
      "nll_loss": 2.778284788131714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1026182621717453,
      "rewards/margins": 1.4513285160064697,
      "rewards/rejected": -1.553946852684021,
      "step": 1757
    },
    {
      "epoch": 1.0936236391912908,
      "grad_norm": 0.5250260829925537,
      "learning_rate": 6.0500000000000005e-06,
      "log_odds_chosen": 16.427053451538086,
      "log_odds_ratio": -0.006266839802265167,
      "logits/chosen": 0.35286745429039,
      "logits/rejected": 1.6788630485534668,
      "logps/chosen": -1.3709660768508911,
      "logps/rejected": -17.149799346923828,
      "loss": 3.1616,
      "nll_loss": 3.1609363555908203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1370965987443924,
      "rewards/margins": 1.5778833627700806,
      "rewards/rejected": -1.7149800062179565,
      "step": 1758
    },
    {
      "epoch": 1.0942457231726284,
      "grad_norm": 0.5247893929481506,
      "learning_rate": 6.025e-06,
      "log_odds_chosen": 12.925125122070312,
      "log_odds_ratio": -0.13547226786613464,
      "logits/chosen": 0.15424951910972595,
      "logits/rejected": 0.8768571019172668,
      "logps/chosen": -1.0417335033416748,
      "logps/rejected": -13.585920333862305,
      "loss": 2.6077,
      "nll_loss": 2.594149589538574,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10417336225509644,
      "rewards/margins": 1.2544187307357788,
      "rewards/rejected": -1.3585920333862305,
      "step": 1759
    },
    {
      "epoch": 1.0948678071539657,
      "grad_norm": 0.7948043942451477,
      "learning_rate": 6e-06,
      "log_odds_chosen": 10.840213775634766,
      "log_odds_ratio": -0.24047577381134033,
      "logits/chosen": 0.19382421672344208,
      "logits/rejected": 0.7827101349830627,
      "logps/chosen": -1.3276411294937134,
      "logps/rejected": -11.8134183883667,
      "loss": 3.0138,
      "nll_loss": 2.9897780418395996,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13276411592960358,
      "rewards/margins": 1.0485777854919434,
      "rewards/rejected": -1.1813418865203857,
      "step": 1760
    },
    {
      "epoch": 1.0954898911353033,
      "grad_norm": 0.4992409646511078,
      "learning_rate": 5.975e-06,
      "log_odds_chosen": 11.336246490478516,
      "log_odds_ratio": -0.10798382014036179,
      "logits/chosen": 0.30136653780937195,
      "logits/rejected": 0.8445361256599426,
      "logps/chosen": -1.0902445316314697,
      "logps/rejected": -12.009919166564941,
      "loss": 3.1134,
      "nll_loss": 3.1026368141174316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10902444273233414,
      "rewards/margins": 1.0919674634933472,
      "rewards/rejected": -1.2009918689727783,
      "step": 1761
    },
    {
      "epoch": 1.0961119751166408,
      "grad_norm": 0.5600183010101318,
      "learning_rate": 5.95e-06,
      "log_odds_chosen": 13.113090515136719,
      "log_odds_ratio": -0.07676493376493454,
      "logits/chosen": 0.345098614692688,
      "logits/rejected": 1.2766406536102295,
      "logps/chosen": -0.9305613040924072,
      "logps/rejected": -13.452409744262695,
      "loss": 3.1596,
      "nll_loss": 3.1518924236297607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09305613487958908,
      "rewards/margins": 1.2521848678588867,
      "rewards/rejected": -1.3452410697937012,
      "step": 1762
    },
    {
      "epoch": 1.0967340590979782,
      "grad_norm": 0.5616739988327026,
      "learning_rate": 5.925e-06,
      "log_odds_chosen": 11.648482322692871,
      "log_odds_ratio": -0.004391771741211414,
      "logits/chosen": 0.34886276721954346,
      "logits/rejected": 1.2682275772094727,
      "logps/chosen": -0.8755768537521362,
      "logps/rejected": -11.964359283447266,
      "loss": 2.9164,
      "nll_loss": 2.9159722328186035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08755768835544586,
      "rewards/margins": 1.1088783740997314,
      "rewards/rejected": -1.1964359283447266,
      "step": 1763
    },
    {
      "epoch": 1.0973561430793157,
      "grad_norm": 0.3752902150154114,
      "learning_rate": 5.9e-06,
      "log_odds_chosen": 15.04450798034668,
      "log_odds_ratio": -0.0849987119436264,
      "logits/chosen": 0.32096612453460693,
      "logits/rejected": 1.0471903085708618,
      "logps/chosen": -0.9109809398651123,
      "logps/rejected": -15.317863464355469,
      "loss": 3.5641,
      "nll_loss": 3.555647134780884,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09109809994697571,
      "rewards/margins": 1.4406882524490356,
      "rewards/rejected": -1.531786322593689,
      "step": 1764
    },
    {
      "epoch": 1.0979782270606533,
      "grad_norm": 0.5268061757087708,
      "learning_rate": 5.875e-06,
      "log_odds_chosen": 13.961103439331055,
      "log_odds_ratio": -0.0894414633512497,
      "logits/chosen": 0.34207406640052795,
      "logits/rejected": 1.2890832424163818,
      "logps/chosen": -1.0298205614089966,
      "logps/rejected": -14.520515441894531,
      "loss": 3.1615,
      "nll_loss": 3.1525745391845703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1029820516705513,
      "rewards/margins": 1.349069595336914,
      "rewards/rejected": -1.4520515203475952,
      "step": 1765
    },
    {
      "epoch": 1.0986003110419906,
      "grad_norm": 2.851046323776245,
      "learning_rate": 5.850000000000001e-06,
      "log_odds_chosen": 12.226277351379395,
      "log_odds_ratio": -0.09023725241422653,
      "logits/chosen": 0.23562204837799072,
      "logits/rejected": 0.862393856048584,
      "logps/chosen": -1.3986942768096924,
      "logps/rejected": -13.135051727294922,
      "loss": 3.2844,
      "nll_loss": 3.275351047515869,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13986943662166595,
      "rewards/margins": 1.173635721206665,
      "rewards/rejected": -1.3135051727294922,
      "step": 1766
    },
    {
      "epoch": 1.0992223950233282,
      "grad_norm": 0.31515470147132874,
      "learning_rate": 5.825000000000001e-06,
      "log_odds_chosen": 12.286757469177246,
      "log_odds_ratio": -0.14368560910224915,
      "logits/chosen": 0.30606141686439514,
      "logits/rejected": 0.9302334785461426,
      "logps/chosen": -1.1010560989379883,
      "logps/rejected": -13.028209686279297,
      "loss": 3.4221,
      "nll_loss": 3.4077136516571045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11010561138391495,
      "rewards/margins": 1.1927152872085571,
      "rewards/rejected": -1.3028209209442139,
      "step": 1767
    },
    {
      "epoch": 1.0998444790046655,
      "grad_norm": 0.6244063973426819,
      "learning_rate": 5.8e-06,
      "log_odds_chosen": 16.554611206054688,
      "log_odds_ratio": -0.005652461666613817,
      "logits/chosen": 0.27573028206825256,
      "logits/rejected": 1.5867812633514404,
      "logps/chosen": -0.9618663191795349,
      "logps/rejected": -16.94882583618164,
      "loss": 2.3865,
      "nll_loss": 2.3859264850616455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09618663787841797,
      "rewards/margins": 1.598695993423462,
      "rewards/rejected": -1.6948826313018799,
      "step": 1768
    },
    {
      "epoch": 1.100466562986003,
      "grad_norm": 0.5130845904350281,
      "learning_rate": 5.775000000000001e-06,
      "log_odds_chosen": 8.970913887023926,
      "log_odds_ratio": -0.13339346647262573,
      "logits/chosen": 0.18855807185173035,
      "logits/rejected": 0.5617761611938477,
      "logps/chosen": -1.3988968133926392,
      "logps/rejected": -10.13296127319336,
      "loss": 2.7901,
      "nll_loss": 2.7767951488494873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1398896872997284,
      "rewards/margins": 0.8734064102172852,
      "rewards/rejected": -1.0132960081100464,
      "step": 1769
    },
    {
      "epoch": 1.1010886469673407,
      "grad_norm": 1.5145896673202515,
      "learning_rate": 5.750000000000001e-06,
      "log_odds_chosen": 9.153180122375488,
      "log_odds_ratio": -0.08820240199565887,
      "logits/chosen": 0.39347460865974426,
      "logits/rejected": 0.7451803088188171,
      "logps/chosen": -1.138330340385437,
      "logps/rejected": -9.729453086853027,
      "loss": 3.2612,
      "nll_loss": 3.2524142265319824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11383302509784698,
      "rewards/margins": 0.8591122627258301,
      "rewards/rejected": -0.9729453325271606,
      "step": 1770
    },
    {
      "epoch": 1.101710730948678,
      "grad_norm": 0.5078622698783875,
      "learning_rate": 5.725e-06,
      "log_odds_chosen": 13.681772232055664,
      "log_odds_ratio": -0.006565184332430363,
      "logits/chosen": 0.354367196559906,
      "logits/rejected": 1.6420621871948242,
      "logps/chosen": -1.1012815237045288,
      "logps/rejected": -14.348349571228027,
      "loss": 2.9864,
      "nll_loss": 2.9857606887817383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11012815684080124,
      "rewards/margins": 1.3247069120407104,
      "rewards/rejected": -1.4348350763320923,
      "step": 1771
    },
    {
      "epoch": 1.1023328149300156,
      "grad_norm": 0.5735906362533569,
      "learning_rate": 5.7000000000000005e-06,
      "log_odds_chosen": 12.572833061218262,
      "log_odds_ratio": -0.002868115436285734,
      "logits/chosen": 0.202457457780838,
      "logits/rejected": 1.019594669342041,
      "logps/chosen": -0.9506032466888428,
      "logps/rejected": -12.859609603881836,
      "loss": 2.9578,
      "nll_loss": 2.9574880599975586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0950603187084198,
      "rewards/margins": 1.1909008026123047,
      "rewards/rejected": -1.2859611511230469,
      "step": 1772
    },
    {
      "epoch": 1.1029548989113531,
      "grad_norm": 0.38094738125801086,
      "learning_rate": 5.675000000000001e-06,
      "log_odds_chosen": 11.872749328613281,
      "log_odds_ratio": -0.00020267191575840116,
      "logits/chosen": 0.23029054701328278,
      "logits/rejected": 1.1186127662658691,
      "logps/chosen": -1.0671327114105225,
      "logps/rejected": -12.310469627380371,
      "loss": 3.1701,
      "nll_loss": 3.170095443725586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10671327263116837,
      "rewards/margins": 1.1243336200714111,
      "rewards/rejected": -1.2310469150543213,
      "step": 1773
    },
    {
      "epoch": 1.1035769828926905,
      "grad_norm": 0.5917351841926575,
      "learning_rate": 5.65e-06,
      "log_odds_chosen": 12.933120727539062,
      "log_odds_ratio": -0.18520431220531464,
      "logits/chosen": 0.4661065638065338,
      "logits/rejected": 1.3536592721939087,
      "logps/chosen": -0.9066671133041382,
      "logps/rejected": -13.326446533203125,
      "loss": 2.971,
      "nll_loss": 2.9524595737457275,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09066671133041382,
      "rewards/margins": 1.2419780492782593,
      "rewards/rejected": -1.3326445817947388,
      "step": 1774
    },
    {
      "epoch": 1.104199066874028,
      "grad_norm": 0.5249805450439453,
      "learning_rate": 5.625e-06,
      "log_odds_chosen": 14.55476188659668,
      "log_odds_ratio": -0.10264108330011368,
      "logits/chosen": 0.23866206407546997,
      "logits/rejected": 0.718443751335144,
      "logps/chosen": -0.9315866231918335,
      "logps/rejected": -14.850624084472656,
      "loss": 3.249,
      "nll_loss": 3.238722801208496,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09315866231918335,
      "rewards/margins": 1.3919038772583008,
      "rewards/rejected": -1.4850624799728394,
      "step": 1775
    },
    {
      "epoch": 1.1048211508553654,
      "grad_norm": 0.48771780729293823,
      "learning_rate": 5.600000000000001e-06,
      "log_odds_chosen": 10.107994079589844,
      "log_odds_ratio": -0.13262391090393066,
      "logits/chosen": 0.1523360162973404,
      "logits/rejected": 0.8286849856376648,
      "logps/chosen": -1.151200771331787,
      "logps/rejected": -10.873334884643555,
      "loss": 2.7703,
      "nll_loss": 2.757059335708618,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11512007564306259,
      "rewards/margins": 0.9722133874893188,
      "rewards/rejected": -1.0873334407806396,
      "step": 1776
    },
    {
      "epoch": 1.105443234836703,
      "grad_norm": 0.5238807797431946,
      "learning_rate": 5.575e-06,
      "log_odds_chosen": 12.62802505493164,
      "log_odds_ratio": -0.09053575247526169,
      "logits/chosen": 0.08372609317302704,
      "logits/rejected": 0.6036351919174194,
      "logps/chosen": -0.9051854014396667,
      "logps/rejected": -12.867120742797852,
      "loss": 2.6631,
      "nll_loss": 2.6540231704711914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0905185416340828,
      "rewards/margins": 1.1961935758590698,
      "rewards/rejected": -1.2867119312286377,
      "step": 1777
    },
    {
      "epoch": 1.1060653188180405,
      "grad_norm": 1.3776423931121826,
      "learning_rate": 5.55e-06,
      "log_odds_chosen": 12.941963195800781,
      "log_odds_ratio": -0.105911985039711,
      "logits/chosen": 0.24506743252277374,
      "logits/rejected": 1.3793317079544067,
      "logps/chosen": -1.4116719961166382,
      "logps/rejected": -14.101229667663574,
      "loss": 2.3892,
      "nll_loss": 2.378603458404541,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14116720855236053,
      "rewards/margins": 1.2689557075500488,
      "rewards/rejected": -1.4101228713989258,
      "step": 1778
    },
    {
      "epoch": 1.1066874027993778,
      "grad_norm": 0.5238211750984192,
      "learning_rate": 5.5250000000000005e-06,
      "log_odds_chosen": 13.194835662841797,
      "log_odds_ratio": -0.13603851199150085,
      "logits/chosen": 0.12899631261825562,
      "logits/rejected": 0.5138747692108154,
      "logps/chosen": -1.0636062622070312,
      "logps/rejected": -13.72635269165039,
      "loss": 2.9434,
      "nll_loss": 2.929769277572632,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10636062920093536,
      "rewards/margins": 1.2662746906280518,
      "rewards/rejected": -1.3726352453231812,
      "step": 1779
    },
    {
      "epoch": 1.1073094867807154,
      "grad_norm": 0.6795265078544617,
      "learning_rate": 5.500000000000001e-06,
      "log_odds_chosen": 9.647849082946777,
      "log_odds_ratio": -0.09886172413825989,
      "logits/chosen": 0.11817916482686996,
      "logits/rejected": 0.804192066192627,
      "logps/chosen": -1.1804863214492798,
      "logps/rejected": -10.412225723266602,
      "loss": 2.6481,
      "nll_loss": 2.6382088661193848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11804862320423126,
      "rewards/margins": 0.9231740236282349,
      "rewards/rejected": -1.0412226915359497,
      "step": 1780
    },
    {
      "epoch": 1.107931570762053,
      "grad_norm": 0.4870269298553467,
      "learning_rate": 5.475e-06,
      "log_odds_chosen": 13.686765670776367,
      "log_odds_ratio": -0.09282388538122177,
      "logits/chosen": 0.08411633968353271,
      "logits/rejected": 0.6649541854858398,
      "logps/chosen": -0.9135332703590393,
      "logps/rejected": -13.984970092773438,
      "loss": 2.7795,
      "nll_loss": 2.7702245712280273,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09135332703590393,
      "rewards/margins": 1.3071436882019043,
      "rewards/rejected": -1.3984969854354858,
      "step": 1781
    },
    {
      "epoch": 1.1085536547433903,
      "grad_norm": 0.47613298892974854,
      "learning_rate": 5.45e-06,
      "log_odds_chosen": 11.947006225585938,
      "log_odds_ratio": -0.004251046106219292,
      "logits/chosen": 0.21725809574127197,
      "logits/rejected": 0.9395244121551514,
      "logps/chosen": -1.0874109268188477,
      "logps/rejected": -12.611610412597656,
      "loss": 3.0012,
      "nll_loss": 3.0007901191711426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10874108970165253,
      "rewards/margins": 1.1524200439453125,
      "rewards/rejected": -1.2611610889434814,
      "step": 1782
    },
    {
      "epoch": 1.1091757387247279,
      "grad_norm": 0.5383672118186951,
      "learning_rate": 5.4250000000000006e-06,
      "log_odds_chosen": 10.831711769104004,
      "log_odds_ratio": -0.007172089070081711,
      "logits/chosen": 0.28789806365966797,
      "logits/rejected": 0.9247058629989624,
      "logps/chosen": -0.7739531397819519,
      "logps/rejected": -10.731185913085938,
      "loss": 3.1642,
      "nll_loss": 3.163468360900879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07739531993865967,
      "rewards/margins": 0.9957231879234314,
      "rewards/rejected": -1.0731184482574463,
      "step": 1783
    },
    {
      "epoch": 1.1097978227060654,
      "grad_norm": 0.58636474609375,
      "learning_rate": 5.4e-06,
      "log_odds_chosen": 2.5373220443725586,
      "log_odds_ratio": -0.48750007152557373,
      "logits/chosen": 0.18775655329227448,
      "logits/rejected": 0.3388059139251709,
      "logps/chosen": -0.9539185166358948,
      "logps/rejected": -3.1987171173095703,
      "loss": 2.3384,
      "nll_loss": 2.2896625995635986,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09539185464382172,
      "rewards/margins": 0.22447985410690308,
      "rewards/rejected": -0.319871723651886,
      "step": 1784
    },
    {
      "epoch": 1.1104199066874028,
      "grad_norm": 3.15838885307312,
      "learning_rate": 5.375e-06,
      "log_odds_chosen": 9.975196838378906,
      "log_odds_ratio": -0.0638727992773056,
      "logits/chosen": 0.09930723160505295,
      "logits/rejected": 1.253692865371704,
      "logps/chosen": -1.0593295097351074,
      "logps/rejected": -10.476996421813965,
      "loss": 2.2838,
      "nll_loss": 2.2774205207824707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10593295097351074,
      "rewards/margins": 0.9417667984962463,
      "rewards/rejected": -1.0476996898651123,
      "step": 1785
    },
    {
      "epoch": 1.1110419906687403,
      "grad_norm": 0.49239975214004517,
      "learning_rate": 5.3500000000000004e-06,
      "log_odds_chosen": 13.380136489868164,
      "log_odds_ratio": -0.00853438675403595,
      "logits/chosen": 0.004909025505185127,
      "logits/rejected": 0.968700647354126,
      "logps/chosen": -0.8748571872711182,
      "logps/rejected": -13.532394409179688,
      "loss": 2.3398,
      "nll_loss": 2.338928699493408,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08748571574687958,
      "rewards/margins": 1.2657537460327148,
      "rewards/rejected": -1.3532395362854004,
      "step": 1786
    },
    {
      "epoch": 1.1116640746500777,
      "grad_norm": 0.499944269657135,
      "learning_rate": 5.325e-06,
      "log_odds_chosen": 12.283882141113281,
      "log_odds_ratio": -0.026974359527230263,
      "logits/chosen": 0.1541229486465454,
      "logits/rejected": 1.2575428485870361,
      "logps/chosen": -1.1521315574645996,
      "logps/rejected": -13.037506103515625,
      "loss": 2.6301,
      "nll_loss": 2.62740421295166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11521315574645996,
      "rewards/margins": 1.188537359237671,
      "rewards/rejected": -1.3037505149841309,
      "step": 1787
    },
    {
      "epoch": 1.1122861586314152,
      "grad_norm": 0.5761716961860657,
      "learning_rate": 5.3e-06,
      "log_odds_chosen": 9.24363899230957,
      "log_odds_ratio": -0.13806326687335968,
      "logits/chosen": 0.26516130566596985,
      "logits/rejected": 0.7415081262588501,
      "logps/chosen": -1.1681675910949707,
      "logps/rejected": -10.058372497558594,
      "loss": 3.0679,
      "nll_loss": 3.0540828704833984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11681675910949707,
      "rewards/margins": 0.8890204429626465,
      "rewards/rejected": -1.0058372020721436,
      "step": 1788
    },
    {
      "epoch": 1.1129082426127528,
      "grad_norm": 0.40672749280929565,
      "learning_rate": 5.275e-06,
      "log_odds_chosen": 9.430431365966797,
      "log_odds_ratio": -0.06735905259847641,
      "logits/chosen": 0.17814627289772034,
      "logits/rejected": 0.5426368713378906,
      "logps/chosen": -1.0851701498031616,
      "logps/rejected": -10.129546165466309,
      "loss": 3.3547,
      "nll_loss": 3.3479928970336914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10851702094078064,
      "rewards/margins": 0.9044376611709595,
      "rewards/rejected": -1.0129547119140625,
      "step": 1789
    },
    {
      "epoch": 1.1135303265940901,
      "grad_norm": 1.1642420291900635,
      "learning_rate": 5.25e-06,
      "log_odds_chosen": 9.519857406616211,
      "log_odds_ratio": -0.08131834864616394,
      "logits/chosen": 0.15677277743816376,
      "logits/rejected": 0.6440751552581787,
      "logps/chosen": -1.1484295129776,
      "logps/rejected": -10.29475212097168,
      "loss": 3.0109,
      "nll_loss": 3.0027811527252197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11484295129776001,
      "rewards/margins": 0.9146323204040527,
      "rewards/rejected": -1.029475212097168,
      "step": 1790
    },
    {
      "epoch": 1.1141524105754277,
      "grad_norm": 0.6786203384399414,
      "learning_rate": 5.225e-06,
      "log_odds_chosen": 10.164011001586914,
      "log_odds_ratio": -0.024791117757558823,
      "logits/chosen": 0.16322621703147888,
      "logits/rejected": 0.7032598257064819,
      "logps/chosen": -1.053176760673523,
      "logps/rejected": -10.773063659667969,
      "loss": 2.464,
      "nll_loss": 2.4615001678466797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10531767457723618,
      "rewards/margins": 0.9719887375831604,
      "rewards/rejected": -1.0773065090179443,
      "step": 1791
    },
    {
      "epoch": 1.1147744945567652,
      "grad_norm": 0.9429541230201721,
      "learning_rate": 5.2e-06,
      "log_odds_chosen": 9.98597240447998,
      "log_odds_ratio": -0.12461309880018234,
      "logits/chosen": 0.26712629199028015,
      "logits/rejected": 1.0227980613708496,
      "logps/chosen": -1.159783959388733,
      "logps/rejected": -10.689939498901367,
      "loss": 3.3261,
      "nll_loss": 3.313620090484619,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11597839742898941,
      "rewards/margins": 0.9530156254768372,
      "rewards/rejected": -1.0689939260482788,
      "step": 1792
    },
    {
      "epoch": 1.1153965785381026,
      "grad_norm": 0.556646466255188,
      "learning_rate": 5.175e-06,
      "log_odds_chosen": 11.009801864624023,
      "log_odds_ratio": -0.1342574954032898,
      "logits/chosen": 0.4200906753540039,
      "logits/rejected": 1.036113977432251,
      "logps/chosen": -1.2086161375045776,
      "logps/rejected": -11.90412712097168,
      "loss": 2.9352,
      "nll_loss": 2.921736717224121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12086161226034164,
      "rewards/margins": 1.0695511102676392,
      "rewards/rejected": -1.1904127597808838,
      "step": 1793
    },
    {
      "epoch": 1.1160186625194402,
      "grad_norm": 0.5953820943832397,
      "learning_rate": 5.15e-06,
      "log_odds_chosen": 9.301898956298828,
      "log_odds_ratio": -0.12582379579544067,
      "logits/chosen": 0.18042080104351044,
      "logits/rejected": 0.9847798347473145,
      "logps/chosen": -1.0908702611923218,
      "logps/rejected": -10.03856372833252,
      "loss": 2.3381,
      "nll_loss": 2.3255486488342285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1090870276093483,
      "rewards/margins": 0.8947693109512329,
      "rewards/rejected": -1.0038564205169678,
      "step": 1794
    },
    {
      "epoch": 1.1166407465007775,
      "grad_norm": 0.5287635326385498,
      "learning_rate": 5.125e-06,
      "log_odds_chosen": 13.280316352844238,
      "log_odds_ratio": -0.0002730822598095983,
      "logits/chosen": 0.3489305078983307,
      "logits/rejected": 0.8643382787704468,
      "logps/chosen": -1.0686596632003784,
      "logps/rejected": -13.85269832611084,
      "loss": 3.1597,
      "nll_loss": 3.159668207168579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10686597228050232,
      "rewards/margins": 1.2784039974212646,
      "rewards/rejected": -1.3852697610855103,
      "step": 1795
    },
    {
      "epoch": 1.117262830482115,
      "grad_norm": 0.40285539627075195,
      "learning_rate": 5.1e-06,
      "log_odds_chosen": 13.911418914794922,
      "log_odds_ratio": -0.00035264677717350423,
      "logits/chosen": 0.3598060607910156,
      "logits/rejected": 1.39473295211792,
      "logps/chosen": -1.0195471048355103,
      "logps/rejected": -14.366622924804688,
      "loss": 3.6114,
      "nll_loss": 3.611367702484131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10195470601320267,
      "rewards/margins": 1.334707498550415,
      "rewards/rejected": -1.4366623163223267,
      "step": 1796
    },
    {
      "epoch": 1.1178849144634526,
      "grad_norm": 0.4381580054759979,
      "learning_rate": 5.0750000000000005e-06,
      "log_odds_chosen": 8.426492691040039,
      "log_odds_ratio": -0.1798814982175827,
      "logits/chosen": 0.21182674169540405,
      "logits/rejected": 0.8382251858711243,
      "logps/chosen": -0.8595851063728333,
      "logps/rejected": -8.753094673156738,
      "loss": 3.2126,
      "nll_loss": 3.1946401596069336,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08595851063728333,
      "rewards/margins": 0.7893509268760681,
      "rewards/rejected": -0.8753094673156738,
      "step": 1797
    },
    {
      "epoch": 1.11850699844479,
      "grad_norm": 0.5452844500541687,
      "learning_rate": 5.050000000000001e-06,
      "log_odds_chosen": 13.306865692138672,
      "log_odds_ratio": -0.0013203290291130543,
      "logits/chosen": 0.32285499572753906,
      "logits/rejected": 0.9466825127601624,
      "logps/chosen": -0.9000481963157654,
      "logps/rejected": -13.57664680480957,
      "loss": 3.4016,
      "nll_loss": 3.4015135765075684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0900048241019249,
      "rewards/margins": 1.2676599025726318,
      "rewards/rejected": -1.357664704322815,
      "step": 1798
    },
    {
      "epoch": 1.1191290824261275,
      "grad_norm": 0.4899674654006958,
      "learning_rate": 5.025e-06,
      "log_odds_chosen": 7.679771423339844,
      "log_odds_ratio": -0.2658516764640808,
      "logits/chosen": 0.24746796488761902,
      "logits/rejected": 0.7958407402038574,
      "logps/chosen": -1.0952465534210205,
      "logps/rejected": -8.41911792755127,
      "loss": 3.2483,
      "nll_loss": 3.221743106842041,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10952465236186981,
      "rewards/margins": 0.7323871850967407,
      "rewards/rejected": -0.8419118523597717,
      "step": 1799
    },
    {
      "epoch": 1.119751166407465,
      "grad_norm": 0.5063171982765198,
      "learning_rate": 5e-06,
      "log_odds_chosen": 12.445841789245605,
      "log_odds_ratio": -0.007602700963616371,
      "logits/chosen": 0.18936318159103394,
      "logits/rejected": 1.0643928050994873,
      "logps/chosen": -0.8866111636161804,
      "logps/rejected": -12.559014320373535,
      "loss": 3.2235,
      "nll_loss": 3.222745180130005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08866112679243088,
      "rewards/margins": 1.1672402620315552,
      "rewards/rejected": -1.2559013366699219,
      "step": 1800
    },
    {
      "epoch": 1.1203732503888024,
      "grad_norm": 11.608192443847656,
      "learning_rate": 4.975000000000001e-06,
      "log_odds_chosen": 6.945075511932373,
      "log_odds_ratio": -0.18060587346553802,
      "logits/chosen": 0.2911841869354248,
      "logits/rejected": 0.6231019496917725,
      "logps/chosen": -1.6782355308532715,
      "logps/rejected": -8.222484588623047,
      "loss": 3.2746,
      "nll_loss": 3.256551742553711,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16782355308532715,
      "rewards/margins": 0.6544249057769775,
      "rewards/rejected": -0.8222484588623047,
      "step": 1801
    },
    {
      "epoch": 1.12099533437014,
      "grad_norm": 0.6419535875320435,
      "learning_rate": 4.950000000000001e-06,
      "log_odds_chosen": 9.327446937561035,
      "log_odds_ratio": -0.36593329906463623,
      "logits/chosen": 0.21668633818626404,
      "logits/rejected": 0.6384451389312744,
      "logps/chosen": -1.1445276737213135,
      "logps/rejected": -10.077396392822266,
      "loss": 3.156,
      "nll_loss": 3.1194300651550293,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1144527718424797,
      "rewards/margins": 0.893286943435669,
      "rewards/rejected": -1.007739782333374,
      "step": 1802
    },
    {
      "epoch": 1.1216174183514775,
      "grad_norm": 0.5713719725608826,
      "learning_rate": 4.925e-06,
      "log_odds_chosen": 12.993343353271484,
      "log_odds_ratio": -0.18610899150371552,
      "logits/chosen": 0.13531741499900818,
      "logits/rejected": 1.0787577629089355,
      "logps/chosen": -0.9270740747451782,
      "logps/rejected": -13.453575134277344,
      "loss": 2.559,
      "nll_loss": 2.540419816970825,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09270741045475006,
      "rewards/margins": 1.252650260925293,
      "rewards/rejected": -1.3453575372695923,
      "step": 1803
    },
    {
      "epoch": 1.1222395023328149,
      "grad_norm": 0.5962578058242798,
      "learning_rate": 4.9000000000000005e-06,
      "log_odds_chosen": 9.101219177246094,
      "log_odds_ratio": -0.1596328318119049,
      "logits/chosen": 0.1289364993572235,
      "logits/rejected": 0.7330648899078369,
      "logps/chosen": -1.3524481058120728,
      "logps/rejected": -10.174253463745117,
      "loss": 2.687,
      "nll_loss": 2.6709885597229004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13524481654167175,
      "rewards/margins": 0.8821805715560913,
      "rewards/rejected": -1.017425298690796,
      "step": 1804
    },
    {
      "epoch": 1.1228615863141524,
      "grad_norm": 3.5306475162506104,
      "learning_rate": 4.875000000000001e-06,
      "log_odds_chosen": 10.527320861816406,
      "log_odds_ratio": -0.13010427355766296,
      "logits/chosen": 0.34094375371932983,
      "logits/rejected": 1.1596550941467285,
      "logps/chosen": -1.4645371437072754,
      "logps/rejected": -11.655326843261719,
      "loss": 3.0124,
      "nll_loss": 2.9993646144866943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14645370841026306,
      "rewards/margins": 1.0190789699554443,
      "rewards/rejected": -1.1655325889587402,
      "step": 1805
    },
    {
      "epoch": 1.1234836702954898,
      "grad_norm": 0.7061519026756287,
      "learning_rate": 4.85e-06,
      "log_odds_chosen": 9.48027515411377,
      "log_odds_ratio": -0.12131432443857193,
      "logits/chosen": 0.0875740572810173,
      "logits/rejected": 0.6032061576843262,
      "logps/chosen": -0.8424355983734131,
      "logps/rejected": -9.36489200592041,
      "loss": 2.832,
      "nll_loss": 2.819911479949951,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08424356579780579,
      "rewards/margins": 0.8522456884384155,
      "rewards/rejected": -0.9364892840385437,
      "step": 1806
    },
    {
      "epoch": 1.1241057542768274,
      "grad_norm": 0.5254723429679871,
      "learning_rate": 4.825e-06,
      "log_odds_chosen": 11.07059097290039,
      "log_odds_ratio": -0.009043725207448006,
      "logits/chosen": 0.22842171788215637,
      "logits/rejected": 0.9652547836303711,
      "logps/chosen": -1.137882947921753,
      "logps/rejected": -11.800371170043945,
      "loss": 3.1437,
      "nll_loss": 3.142822265625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11378829926252365,
      "rewards/margins": 1.066248893737793,
      "rewards/rejected": -1.180037021636963,
      "step": 1807
    },
    {
      "epoch": 1.124727838258165,
      "grad_norm": 5.473237037658691,
      "learning_rate": 4.800000000000001e-06,
      "log_odds_chosen": 2.729802370071411,
      "log_odds_ratio": -0.5633368492126465,
      "logits/chosen": 0.1724277287721634,
      "logits/rejected": 0.2686040699481964,
      "logps/chosen": -1.4853651523590088,
      "logps/rejected": -4.039682388305664,
      "loss": 3.0733,
      "nll_loss": 3.0169594287872314,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14853651821613312,
      "rewards/margins": 0.2554317116737366,
      "rewards/rejected": -0.4039682447910309,
      "step": 1808
    },
    {
      "epoch": 1.1253499222395023,
      "grad_norm": 0.5656214952468872,
      "learning_rate": 4.775e-06,
      "log_odds_chosen": 8.675018310546875,
      "log_odds_ratio": -0.022862181067466736,
      "logits/chosen": 0.10422070324420929,
      "logits/rejected": 0.7279854416847229,
      "logps/chosen": -1.305293083190918,
      "logps/rejected": -9.647176742553711,
      "loss": 2.836,
      "nll_loss": 2.8336758613586426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13052931427955627,
      "rewards/margins": 0.8341884016990662,
      "rewards/rejected": -0.9647176265716553,
      "step": 1809
    },
    {
      "epoch": 1.1259720062208398,
      "grad_norm": 0.6115718483924866,
      "learning_rate": 4.75e-06,
      "log_odds_chosen": 10.84763240814209,
      "log_odds_ratio": -0.08447157591581345,
      "logits/chosen": 0.288266122341156,
      "logits/rejected": 1.0908019542694092,
      "logps/chosen": -1.0053138732910156,
      "logps/rejected": -11.463065147399902,
      "loss": 2.7034,
      "nll_loss": 2.6949832439422607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10053138434886932,
      "rewards/margins": 1.0457751750946045,
      "rewards/rejected": -1.1463065147399902,
      "step": 1810
    },
    {
      "epoch": 1.1265940902021774,
      "grad_norm": 0.45324620604515076,
      "learning_rate": 4.7250000000000005e-06,
      "log_odds_chosen": 6.809017181396484,
      "log_odds_ratio": -0.136130690574646,
      "logits/chosen": 0.09738215804100037,
      "logits/rejected": 0.2309304028749466,
      "logps/chosen": -0.8674330711364746,
      "logps/rejected": -7.067953586578369,
      "loss": 3.1252,
      "nll_loss": 3.111586570739746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0867433100938797,
      "rewards/margins": 0.6200520992279053,
      "rewards/rejected": -0.706795334815979,
      "step": 1811
    },
    {
      "epoch": 1.1272161741835147,
      "grad_norm": 0.41881006956100464,
      "learning_rate": 4.7e-06,
      "log_odds_chosen": 9.960844993591309,
      "log_odds_ratio": -0.2785137891769409,
      "logits/chosen": 0.21459250152111053,
      "logits/rejected": 0.8106105327606201,
      "logps/chosen": -1.2352036237716675,
      "logps/rejected": -10.74962329864502,
      "loss": 3.3827,
      "nll_loss": 3.3548502922058105,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12352035939693451,
      "rewards/margins": 0.9514420628547668,
      "rewards/rejected": -1.0749623775482178,
      "step": 1812
    },
    {
      "epoch": 1.1278382581648523,
      "grad_norm": 0.34027522802352905,
      "learning_rate": 4.675e-06,
      "log_odds_chosen": 7.443631649017334,
      "log_odds_ratio": -0.09579671174287796,
      "logits/chosen": 0.35332971811294556,
      "logits/rejected": 0.8371080160140991,
      "logps/chosen": -1.1036956310272217,
      "logps/rejected": -8.065750122070312,
      "loss": 3.8843,
      "nll_loss": 3.874685287475586,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11036956310272217,
      "rewards/margins": 0.6962054371833801,
      "rewards/rejected": -0.8065750002861023,
      "step": 1813
    },
    {
      "epoch": 1.1284603421461896,
      "grad_norm": 6.815546035766602,
      "learning_rate": 4.65e-06,
      "log_odds_chosen": 8.074342727661133,
      "log_odds_ratio": -0.07201213389635086,
      "logits/chosen": 0.18229568004608154,
      "logits/rejected": 0.6928460001945496,
      "logps/chosen": -0.9151440858840942,
      "logps/rejected": -8.362491607666016,
      "loss": 2.7987,
      "nll_loss": 2.791532516479492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09151440858840942,
      "rewards/margins": 0.7447347044944763,
      "rewards/rejected": -0.8362491130828857,
      "step": 1814
    },
    {
      "epoch": 1.1290824261275272,
      "grad_norm": 0.6330059766769409,
      "learning_rate": 4.625e-06,
      "log_odds_chosen": 8.535118103027344,
      "log_odds_ratio": -0.13910330832004547,
      "logits/chosen": 0.2871561646461487,
      "logits/rejected": 1.1333131790161133,
      "logps/chosen": -1.05423903465271,
      "logps/rejected": -9.111215591430664,
      "loss": 2.8813,
      "nll_loss": 2.867426633834839,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10542389750480652,
      "rewards/margins": 0.8056977391242981,
      "rewards/rejected": -0.911121666431427,
      "step": 1815
    },
    {
      "epoch": 1.1297045101088647,
      "grad_norm": 0.5443134903907776,
      "learning_rate": 4.6e-06,
      "log_odds_chosen": 8.793806076049805,
      "log_odds_ratio": -0.11311511695384979,
      "logits/chosen": 0.03522084280848503,
      "logits/rejected": 0.4860665202140808,
      "logps/chosen": -1.1860641241073608,
      "logps/rejected": -9.650390625,
      "loss": 2.5238,
      "nll_loss": 2.5124518871307373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11860641837120056,
      "rewards/margins": 0.8464326858520508,
      "rewards/rejected": -0.965039074420929,
      "step": 1816
    },
    {
      "epoch": 1.130326594090202,
      "grad_norm": 2.309431314468384,
      "learning_rate": 4.575e-06,
      "log_odds_chosen": 9.45278549194336,
      "log_odds_ratio": -0.10079053044319153,
      "logits/chosen": 0.2651061713695526,
      "logits/rejected": 1.1936180591583252,
      "logps/chosen": -1.343247652053833,
      "logps/rejected": -10.476812362670898,
      "loss": 3.1809,
      "nll_loss": 3.1708319187164307,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13432475924491882,
      "rewards/margins": 0.9133565425872803,
      "rewards/rejected": -1.0476813316345215,
      "step": 1817
    },
    {
      "epoch": 1.1309486780715396,
      "grad_norm": 0.5004618763923645,
      "learning_rate": 4.5500000000000005e-06,
      "log_odds_chosen": 9.56799030303955,
      "log_odds_ratio": -0.003998002503067255,
      "logits/chosen": 0.12169669568538666,
      "logits/rejected": 0.5667641162872314,
      "logps/chosen": -1.166806697845459,
      "logps/rejected": -10.264081954956055,
      "loss": 2.6758,
      "nll_loss": 2.675359010696411,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11668066680431366,
      "rewards/margins": 0.9097274541854858,
      "rewards/rejected": -1.0264081954956055,
      "step": 1818
    },
    {
      "epoch": 1.1315707620528772,
      "grad_norm": 0.616479754447937,
      "learning_rate": 4.525e-06,
      "log_odds_chosen": 11.951116561889648,
      "log_odds_ratio": -0.013583849184215069,
      "logits/chosen": 0.08394649624824524,
      "logits/rejected": 0.9181208610534668,
      "logps/chosen": -1.046438217163086,
      "logps/rejected": -12.497675895690918,
      "loss": 2.5327,
      "nll_loss": 2.5313260555267334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1046438217163086,
      "rewards/margins": 1.1451237201690674,
      "rewards/rejected": -1.249767541885376,
      "step": 1819
    },
    {
      "epoch": 1.1321928460342146,
      "grad_norm": 0.6492127776145935,
      "learning_rate": 4.5e-06,
      "log_odds_chosen": 14.50720500946045,
      "log_odds_ratio": -0.0004224668664392084,
      "logits/chosen": 0.2038557529449463,
      "logits/rejected": 1.0356361865997314,
      "logps/chosen": -0.7524871826171875,
      "logps/rejected": -14.405542373657227,
      "loss": 2.7266,
      "nll_loss": 2.726580858230591,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07524871826171875,
      "rewards/margins": 1.3653055429458618,
      "rewards/rejected": -1.4405542612075806,
      "step": 1820
    },
    {
      "epoch": 1.1328149300155521,
      "grad_norm": 0.3873848021030426,
      "learning_rate": 4.475e-06,
      "log_odds_chosen": 9.258432388305664,
      "log_odds_ratio": -0.1324738711118698,
      "logits/chosen": 0.24150168895721436,
      "logits/rejected": 0.7265496253967285,
      "logps/chosen": -0.9946005344390869,
      "logps/rejected": -9.797269821166992,
      "loss": 3.4301,
      "nll_loss": 3.4168717861175537,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09946005046367645,
      "rewards/margins": 0.8802669048309326,
      "rewards/rejected": -0.9797269105911255,
      "step": 1821
    },
    {
      "epoch": 1.1334370139968897,
      "grad_norm": 0.651413083076477,
      "learning_rate": 4.45e-06,
      "log_odds_chosen": 6.027961730957031,
      "log_odds_ratio": -0.16579578816890717,
      "logits/chosen": 0.2767557203769684,
      "logits/rejected": 0.3758124113082886,
      "logps/chosen": -1.2263211011886597,
      "logps/rejected": -6.595951080322266,
      "loss": 3.507,
      "nll_loss": 3.4903945922851562,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12263211607933044,
      "rewards/margins": 0.5369629859924316,
      "rewards/rejected": -0.6595950722694397,
      "step": 1822
    },
    {
      "epoch": 1.134059097978227,
      "grad_norm": 0.4178617596626282,
      "learning_rate": 4.425e-06,
      "log_odds_chosen": 9.204516410827637,
      "log_odds_ratio": -0.08950802683830261,
      "logits/chosen": 0.25004613399505615,
      "logits/rejected": 0.8217270374298096,
      "logps/chosen": -0.8077847361564636,
      "logps/rejected": -9.325156211853027,
      "loss": 3.2034,
      "nll_loss": 3.1944754123687744,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08077847212553024,
      "rewards/margins": 0.8517371416091919,
      "rewards/rejected": -0.9325156807899475,
      "step": 1823
    },
    {
      "epoch": 1.1346811819595646,
      "grad_norm": 0.8599444031715393,
      "learning_rate": 4.4e-06,
      "log_odds_chosen": 7.78436279296875,
      "log_odds_ratio": -0.3169845640659332,
      "logits/chosen": 0.21967101097106934,
      "logits/rejected": 0.54742431640625,
      "logps/chosen": -1.0425914525985718,
      "logps/rejected": -8.581308364868164,
      "loss": 2.8782,
      "nll_loss": 2.8464527130126953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10425914824008942,
      "rewards/margins": 0.7538717985153198,
      "rewards/rejected": -0.858130931854248,
      "step": 1824
    },
    {
      "epoch": 1.1353032659409021,
      "grad_norm": 0.42188894748687744,
      "learning_rate": 4.375e-06,
      "log_odds_chosen": 6.164718151092529,
      "log_odds_ratio": -0.1897582858800888,
      "logits/chosen": 0.31425851583480835,
      "logits/rejected": 0.5133528709411621,
      "logps/chosen": -1.1853477954864502,
      "logps/rejected": -7.080450057983398,
      "loss": 3.604,
      "nll_loss": 3.5849769115448,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11853478848934174,
      "rewards/margins": 0.5895102024078369,
      "rewards/rejected": -0.7080450057983398,
      "step": 1825
    },
    {
      "epoch": 1.1359253499222395,
      "grad_norm": 0.6564217805862427,
      "learning_rate": 4.35e-06,
      "log_odds_chosen": 8.594808578491211,
      "log_odds_ratio": -0.06983879208564758,
      "logits/chosen": 0.27198323607444763,
      "logits/rejected": 1.182363748550415,
      "logps/chosen": -1.3396228551864624,
      "logps/rejected": -9.5281400680542,
      "loss": 3.3834,
      "nll_loss": 3.3763747215270996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13396227359771729,
      "rewards/margins": 0.8188517093658447,
      "rewards/rejected": -0.9528140425682068,
      "step": 1826
    },
    {
      "epoch": 1.136547433903577,
      "grad_norm": 0.899903416633606,
      "learning_rate": 4.325e-06,
      "log_odds_chosen": 9.354620933532715,
      "log_odds_ratio": -0.019159851595759392,
      "logits/chosen": 0.1395963877439499,
      "logits/rejected": 0.6038464307785034,
      "logps/chosen": -1.1089383363723755,
      "logps/rejected": -9.972417831420898,
      "loss": 2.7222,
      "nll_loss": 2.7203056812286377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11089383065700531,
      "rewards/margins": 0.8863480091094971,
      "rewards/rejected": -0.9972418546676636,
      "step": 1827
    },
    {
      "epoch": 1.1371695178849144,
      "grad_norm": 0.5476040244102478,
      "learning_rate": 4.2999999999999995e-06,
      "log_odds_chosen": 12.213828086853027,
      "log_odds_ratio": -0.005907875020056963,
      "logits/chosen": 0.12920621037483215,
      "logits/rejected": 1.0453824996948242,
      "logps/chosen": -0.9840941429138184,
      "logps/rejected": -12.59911060333252,
      "loss": 2.2828,
      "nll_loss": 2.2822105884552,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09840941429138184,
      "rewards/margins": 1.1615017652511597,
      "rewards/rejected": -1.2599111795425415,
      "step": 1828
    },
    {
      "epoch": 1.137791601866252,
      "grad_norm": 0.5353556275367737,
      "learning_rate": 4.2750000000000006e-06,
      "log_odds_chosen": 8.780527114868164,
      "log_odds_ratio": -0.19837351143360138,
      "logits/chosen": 0.1902414858341217,
      "logits/rejected": 1.0131205320358276,
      "logps/chosen": -1.11326265335083,
      "logps/rejected": -9.524097442626953,
      "loss": 2.8569,
      "nll_loss": 2.837033271789551,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11132626235485077,
      "rewards/margins": 0.8410835266113281,
      "rewards/rejected": -0.9524097442626953,
      "step": 1829
    },
    {
      "epoch": 1.1384136858475895,
      "grad_norm": 0.4632808268070221,
      "learning_rate": 4.250000000000001e-06,
      "log_odds_chosen": 12.49989128112793,
      "log_odds_ratio": -0.01967625506222248,
      "logits/chosen": 0.11193087697029114,
      "logits/rejected": 0.8796758651733398,
      "logps/chosen": -0.9191639423370361,
      "logps/rejected": -12.864387512207031,
      "loss": 2.8738,
      "nll_loss": 2.871879816055298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09191639721393585,
      "rewards/margins": 1.1945223808288574,
      "rewards/rejected": -1.2864387035369873,
      "step": 1830
    },
    {
      "epoch": 1.1390357698289268,
      "grad_norm": 0.562156617641449,
      "learning_rate": 4.225e-06,
      "log_odds_chosen": 9.081823348999023,
      "log_odds_ratio": -0.08209239691495895,
      "logits/chosen": -0.011233558878302574,
      "logits/rejected": 0.5702467560768127,
      "logps/chosen": -0.9023143649101257,
      "logps/rejected": -9.305441856384277,
      "loss": 2.3841,
      "nll_loss": 2.3758649826049805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09023144096136093,
      "rewards/margins": 0.8403127789497375,
      "rewards/rejected": -0.9305441975593567,
      "step": 1831
    },
    {
      "epoch": 1.1396578538102644,
      "grad_norm": 0.5008666515350342,
      "learning_rate": 4.2000000000000004e-06,
      "log_odds_chosen": 8.269814491271973,
      "log_odds_ratio": -0.020565472543239594,
      "logits/chosen": 0.047942303121089935,
      "logits/rejected": 0.6168398857116699,
      "logps/chosen": -0.8375715017318726,
      "logps/rejected": -8.36706829071045,
      "loss": 2.8247,
      "nll_loss": 2.8226683139801025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08375715464353561,
      "rewards/margins": 0.7529496550559998,
      "rewards/rejected": -0.836706817150116,
      "step": 1832
    },
    {
      "epoch": 1.1402799377916017,
      "grad_norm": 0.6325581669807434,
      "learning_rate": 4.175000000000001e-06,
      "log_odds_chosen": 5.459460735321045,
      "log_odds_ratio": -0.12678371369838715,
      "logits/chosen": 0.324423611164093,
      "logits/rejected": 0.6908161044120789,
      "logps/chosen": -1.1747678518295288,
      "logps/rejected": -6.291014671325684,
      "loss": 3.437,
      "nll_loss": 3.424285411834717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11747679114341736,
      "rewards/margins": 0.5116246938705444,
      "rewards/rejected": -0.6291014552116394,
      "step": 1833
    },
    {
      "epoch": 1.1409020217729393,
      "grad_norm": 1.8699506521224976,
      "learning_rate": 4.15e-06,
      "log_odds_chosen": 5.80875825881958,
      "log_odds_ratio": -0.19153320789337158,
      "logits/chosen": 0.1311289221048355,
      "logits/rejected": 0.5716495513916016,
      "logps/chosen": -1.1367210149765015,
      "logps/rejected": -6.573773384094238,
      "loss": 2.6326,
      "nll_loss": 2.6134707927703857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11367210745811462,
      "rewards/margins": 0.5437052249908447,
      "rewards/rejected": -0.6573773622512817,
      "step": 1834
    },
    {
      "epoch": 1.1415241057542769,
      "grad_norm": 0.5624792575836182,
      "learning_rate": 4.125e-06,
      "log_odds_chosen": 10.041004180908203,
      "log_odds_ratio": -0.10364656895399094,
      "logits/chosen": 0.3881472647190094,
      "logits/rejected": 1.2136845588684082,
      "logps/chosen": -1.314047932624817,
      "logps/rejected": -11.067365646362305,
      "loss": 3.0385,
      "nll_loss": 3.0281782150268555,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1314047873020172,
      "rewards/margins": 0.9753317832946777,
      "rewards/rejected": -1.106736660003662,
      "step": 1835
    },
    {
      "epoch": 1.1421461897356142,
      "grad_norm": 0.6248362064361572,
      "learning_rate": 4.1000000000000006e-06,
      "log_odds_chosen": 7.472942352294922,
      "log_odds_ratio": -0.2258700728416443,
      "logits/chosen": 0.26532885432243347,
      "logits/rejected": 1.0628668069839478,
      "logps/chosen": -1.1556396484375,
      "logps/rejected": -8.379240989685059,
      "loss": 2.5733,
      "nll_loss": 2.5506765842437744,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11556396633386612,
      "rewards/margins": 0.7223601341247559,
      "rewards/rejected": -0.8379241228103638,
      "step": 1836
    },
    {
      "epoch": 1.1427682737169518,
      "grad_norm": 0.640946626663208,
      "learning_rate": 4.075e-06,
      "log_odds_chosen": 11.358403205871582,
      "log_odds_ratio": -0.011467165313661098,
      "logits/chosen": 0.1835147589445114,
      "logits/rejected": 0.8875374794006348,
      "logps/chosen": -1.098917007446289,
      "logps/rejected": -12.045377731323242,
      "loss": 2.4127,
      "nll_loss": 2.4115188121795654,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10989170521497726,
      "rewards/margins": 1.0946459770202637,
      "rewards/rejected": -1.2045376300811768,
      "step": 1837
    },
    {
      "epoch": 1.1433903576982893,
      "grad_norm": 0.6438122987747192,
      "learning_rate": 4.05e-06,
      "log_odds_chosen": 9.392248153686523,
      "log_odds_ratio": -0.04183017462491989,
      "logits/chosen": 0.30040043592453003,
      "logits/rejected": 1.151014804840088,
      "logps/chosen": -1.1815377473831177,
      "logps/rejected": -10.117670059204102,
      "loss": 3.2376,
      "nll_loss": 3.2334647178649902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11815378069877625,
      "rewards/margins": 0.8936132192611694,
      "rewards/rejected": -1.011767029762268,
      "step": 1838
    },
    {
      "epoch": 1.1440124416796267,
      "grad_norm": 0.5689122676849365,
      "learning_rate": 4.0250000000000004e-06,
      "log_odds_chosen": 11.757156372070312,
      "log_odds_ratio": -0.15141499042510986,
      "logits/chosen": 0.38766422867774963,
      "logits/rejected": 1.2692524194717407,
      "logps/chosen": -1.2827192544937134,
      "logps/rejected": -12.719758033752441,
      "loss": 3.2818,
      "nll_loss": 3.266641139984131,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1282719373703003,
      "rewards/margins": 1.143703818321228,
      "rewards/rejected": -1.2719756364822388,
      "step": 1839
    },
    {
      "epoch": 1.1446345256609642,
      "grad_norm": 0.5184850692749023,
      "learning_rate": 4.000000000000001e-06,
      "log_odds_chosen": 9.902469635009766,
      "log_odds_ratio": -0.006094024516642094,
      "logits/chosen": 0.17294877767562866,
      "logits/rejected": 0.9480814933776855,
      "logps/chosen": -1.2275193929672241,
      "logps/rejected": -10.757221221923828,
      "loss": 3.0007,
      "nll_loss": 3.000067949295044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12275194376707077,
      "rewards/margins": 0.9529702663421631,
      "rewards/rejected": -1.0757222175598145,
      "step": 1840
    },
    {
      "epoch": 1.1452566096423018,
      "grad_norm": 0.7144693732261658,
      "learning_rate": 3.975e-06,
      "log_odds_chosen": 3.845844030380249,
      "log_odds_ratio": -0.27779051661491394,
      "logits/chosen": 0.24042358994483948,
      "logits/rejected": 0.4206497073173523,
      "logps/chosen": -1.3415154218673706,
      "logps/rejected": -4.978623390197754,
      "loss": 3.2546,
      "nll_loss": 3.2268238067626953,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13415154814720154,
      "rewards/margins": 0.36371082067489624,
      "rewards/rejected": -0.4978623688220978,
      "step": 1841
    },
    {
      "epoch": 1.1458786936236391,
      "grad_norm": 0.6551204919815063,
      "learning_rate": 3.95e-06,
      "log_odds_chosen": 8.937030792236328,
      "log_odds_ratio": -0.2580510675907135,
      "logits/chosen": 0.36829322576522827,
      "logits/rejected": 0.6801607012748718,
      "logps/chosen": -0.9728015065193176,
      "logps/rejected": -9.46196174621582,
      "loss": 3.3067,
      "nll_loss": 3.2808713912963867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09728015959262848,
      "rewards/margins": 0.8489160537719727,
      "rewards/rejected": -0.9461961984634399,
      "step": 1842
    },
    {
      "epoch": 1.1465007776049767,
      "grad_norm": 0.5541713237762451,
      "learning_rate": 3.9250000000000005e-06,
      "log_odds_chosen": 11.716171264648438,
      "log_odds_ratio": -0.11533959209918976,
      "logits/chosen": 0.31325313448905945,
      "logits/rejected": 0.6892248392105103,
      "logps/chosen": -0.795279324054718,
      "logps/rejected": -11.923837661743164,
      "loss": 3.444,
      "nll_loss": 3.4324710369110107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07952792942523956,
      "rewards/margins": 1.1128557920455933,
      "rewards/rejected": -1.1923837661743164,
      "step": 1843
    },
    {
      "epoch": 1.1471228615863143,
      "grad_norm": 0.7045103907585144,
      "learning_rate": 3.9e-06,
      "log_odds_chosen": 9.548257827758789,
      "log_odds_ratio": -0.12770326435565948,
      "logits/chosen": 0.19907158613204956,
      "logits/rejected": 0.7976346611976624,
      "logps/chosen": -1.2352797985076904,
      "logps/rejected": -10.314152717590332,
      "loss": 2.605,
      "nll_loss": 2.5922439098358154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12352797389030457,
      "rewards/margins": 0.90788733959198,
      "rewards/rejected": -1.031415343284607,
      "step": 1844
    },
    {
      "epoch": 1.1477449455676516,
      "grad_norm": 0.5564942955970764,
      "learning_rate": 3.875e-06,
      "log_odds_chosen": 13.43726634979248,
      "log_odds_ratio": -0.015584073029458523,
      "logits/chosen": 0.24572142958641052,
      "logits/rejected": 1.1272077560424805,
      "logps/chosen": -1.1543810367584229,
      "logps/rejected": -14.049823760986328,
      "loss": 2.7776,
      "nll_loss": 2.7759971618652344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11543810367584229,
      "rewards/margins": 1.2895443439483643,
      "rewards/rejected": -1.4049824476242065,
      "step": 1845
    },
    {
      "epoch": 1.1483670295489892,
      "grad_norm": 0.539889931678772,
      "learning_rate": 3.85e-06,
      "log_odds_chosen": 7.889827728271484,
      "log_odds_ratio": -0.23806874454021454,
      "logits/chosen": 0.3377843201160431,
      "logits/rejected": 0.6474952697753906,
      "logps/chosen": -1.1127874851226807,
      "logps/rejected": -8.637657165527344,
      "loss": 3.512,
      "nll_loss": 3.4881978034973145,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11127874255180359,
      "rewards/margins": 0.7524869441986084,
      "rewards/rejected": -0.8637657165527344,
      "step": 1846
    },
    {
      "epoch": 1.1489891135303265,
      "grad_norm": 0.5164604187011719,
      "learning_rate": 3.825e-06,
      "log_odds_chosen": 7.681192398071289,
      "log_odds_ratio": -0.15600374341011047,
      "logits/chosen": 0.2687627375125885,
      "logits/rejected": 0.6659653186798096,
      "logps/chosen": -1.1743711233139038,
      "logps/rejected": -8.557535171508789,
      "loss": 3.3275,
      "nll_loss": 3.3118975162506104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11743712425231934,
      "rewards/margins": 0.7383164167404175,
      "rewards/rejected": -0.8557536005973816,
      "step": 1847
    },
    {
      "epoch": 1.149611197511664,
      "grad_norm": 0.5505557656288147,
      "learning_rate": 3.8e-06,
      "log_odds_chosen": 8.284540176391602,
      "log_odds_ratio": -0.04684235155582428,
      "logits/chosen": 0.1542847603559494,
      "logits/rejected": 0.35668423771858215,
      "logps/chosen": -0.7916722893714905,
      "logps/rejected": -8.380890846252441,
      "loss": 3.157,
      "nll_loss": 3.152329921722412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07916723191738129,
      "rewards/margins": 0.7589218616485596,
      "rewards/rejected": -0.8380890488624573,
      "step": 1848
    },
    {
      "epoch": 1.1502332814930016,
      "grad_norm": 0.6552994847297668,
      "learning_rate": 3.775e-06,
      "log_odds_chosen": 15.202106475830078,
      "log_odds_ratio": -0.01975049264729023,
      "logits/chosen": 0.44040918350219727,
      "logits/rejected": 1.5805052518844604,
      "logps/chosen": -1.228332757949829,
      "logps/rejected": -16.05695152282715,
      "loss": 2.9889,
      "nll_loss": 2.9869132041931152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12283328920602798,
      "rewards/margins": 1.4828619956970215,
      "rewards/rejected": -1.6056952476501465,
      "step": 1849
    },
    {
      "epoch": 1.150855365474339,
      "grad_norm": 7.42947244644165,
      "learning_rate": 3.75e-06,
      "log_odds_chosen": 5.40254545211792,
      "log_odds_ratio": -0.21582013368606567,
      "logits/chosen": 0.16621777415275574,
      "logits/rejected": 0.7174571752548218,
      "logps/chosen": -1.1043572425842285,
      "logps/rejected": -6.091453552246094,
      "loss": 2.4599,
      "nll_loss": 2.4383544921875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11043573170900345,
      "rewards/margins": 0.49870961904525757,
      "rewards/rejected": -0.6091454029083252,
      "step": 1850
    },
    {
      "epoch": 1.1514774494556765,
      "grad_norm": 0.5765233635902405,
      "learning_rate": 3.725e-06,
      "log_odds_chosen": 11.660882949829102,
      "log_odds_ratio": -0.11469542235136032,
      "logits/chosen": 0.15178298950195312,
      "logits/rejected": 0.9491125345230103,
      "logps/chosen": -1.121006965637207,
      "logps/rejected": -12.402009963989258,
      "loss": 2.7552,
      "nll_loss": 2.7437798976898193,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11210069060325623,
      "rewards/margins": 1.1281002759933472,
      "rewards/rejected": -1.2402009963989258,
      "step": 1851
    },
    {
      "epoch": 1.1520995334370139,
      "grad_norm": 0.792013943195343,
      "learning_rate": 3.7e-06,
      "log_odds_chosen": 4.792044639587402,
      "log_odds_ratio": -0.3845703601837158,
      "logits/chosen": 0.11802121996879578,
      "logits/rejected": 0.3419199585914612,
      "logps/chosen": -1.216029405593872,
      "logps/rejected": -5.794693946838379,
      "loss": 2.6536,
      "nll_loss": 2.6151530742645264,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12160294502973557,
      "rewards/margins": 0.4578664302825928,
      "rewards/rejected": -0.5794694423675537,
      "step": 1852
    },
    {
      "epoch": 1.1527216174183514,
      "grad_norm": 0.5236743688583374,
      "learning_rate": 3.675e-06,
      "log_odds_chosen": 9.352622985839844,
      "log_odds_ratio": -0.05456789955496788,
      "logits/chosen": 0.17611032724380493,
      "logits/rejected": 0.6998663544654846,
      "logps/chosen": -1.2798631191253662,
      "logps/rejected": -9.97517204284668,
      "loss": 3.0422,
      "nll_loss": 3.0367519855499268,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12798631191253662,
      "rewards/margins": 0.8695309162139893,
      "rewards/rejected": -0.9975172281265259,
      "step": 1853
    },
    {
      "epoch": 1.153343701399689,
      "grad_norm": 0.5596763491630554,
      "learning_rate": 3.6499999999999998e-06,
      "log_odds_chosen": 11.331252098083496,
      "log_odds_ratio": -0.17287488281726837,
      "logits/chosen": 0.16702744364738464,
      "logits/rejected": 0.8460006713867188,
      "logps/chosen": -0.9080240726470947,
      "logps/rejected": -11.69823169708252,
      "loss": 2.494,
      "nll_loss": 2.4767417907714844,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09080240875482559,
      "rewards/margins": 1.0790207386016846,
      "rewards/rejected": -1.169823169708252,
      "step": 1854
    },
    {
      "epoch": 1.1539657853810263,
      "grad_norm": 0.5587321519851685,
      "learning_rate": 3.625e-06,
      "log_odds_chosen": 12.046408653259277,
      "log_odds_ratio": -0.09323586523532867,
      "logits/chosen": 0.31840091943740845,
      "logits/rejected": 1.0333279371261597,
      "logps/chosen": -1.2253146171569824,
      "logps/rejected": -12.962678909301758,
      "loss": 3.1042,
      "nll_loss": 3.094883441925049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.122531458735466,
      "rewards/margins": 1.1737364530563354,
      "rewards/rejected": -1.2962678670883179,
      "step": 1855
    },
    {
      "epoch": 1.154587869362364,
      "grad_norm": 5.766540050506592,
      "learning_rate": 3.6e-06,
      "log_odds_chosen": 7.31904935836792,
      "log_odds_ratio": -0.12297351658344269,
      "logits/chosen": 0.2189761996269226,
      "logits/rejected": 0.9063780903816223,
      "logps/chosen": -1.0311709642410278,
      "logps/rejected": -7.974186897277832,
      "loss": 2.1915,
      "nll_loss": 2.1791834831237793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10311709344387054,
      "rewards/margins": 0.6943015456199646,
      "rewards/rejected": -0.7974186539649963,
      "step": 1856
    },
    {
      "epoch": 1.1552099533437015,
      "grad_norm": 0.5681399703025818,
      "learning_rate": 3.575e-06,
      "log_odds_chosen": 12.468045234680176,
      "log_odds_ratio": -0.008419721387326717,
      "logits/chosen": 0.30339351296424866,
      "logits/rejected": 1.4633084535598755,
      "logps/chosen": -1.0308117866516113,
      "logps/rejected": -13.030942916870117,
      "loss": 2.5128,
      "nll_loss": 2.5119991302490234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10308118164539337,
      "rewards/margins": 1.2000131607055664,
      "rewards/rejected": -1.3030942678451538,
      "step": 1857
    },
    {
      "epoch": 1.1558320373250388,
      "grad_norm": 0.616908848285675,
      "learning_rate": 3.55e-06,
      "log_odds_chosen": 6.324096202850342,
      "log_odds_ratio": -0.22164684534072876,
      "logits/chosen": 0.2558293640613556,
      "logits/rejected": 0.6237554550170898,
      "logps/chosen": -0.7948259115219116,
      "logps/rejected": -6.515898704528809,
      "loss": 2.936,
      "nll_loss": 2.9138057231903076,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07948260009288788,
      "rewards/margins": 0.5721072554588318,
      "rewards/rejected": -0.6515898704528809,
      "step": 1858
    },
    {
      "epoch": 1.1564541213063764,
      "grad_norm": 0.6301389932632446,
      "learning_rate": 3.5249999999999997e-06,
      "log_odds_chosen": 7.7862467765808105,
      "log_odds_ratio": -0.2558721601963043,
      "logits/chosen": 0.2493916153907776,
      "logits/rejected": 0.9461332559585571,
      "logps/chosen": -1.1818265914916992,
      "logps/rejected": -8.736333847045898,
      "loss": 3.0082,
      "nll_loss": 2.98262619972229,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11818264424800873,
      "rewards/margins": 0.7554507255554199,
      "rewards/rejected": -0.8736333847045898,
      "step": 1859
    },
    {
      "epoch": 1.157076205287714,
      "grad_norm": 0.6364076137542725,
      "learning_rate": 3.5000000000000004e-06,
      "log_odds_chosen": 5.857247352600098,
      "log_odds_ratio": -0.38049325346946716,
      "logits/chosen": 0.0820159912109375,
      "logits/rejected": 0.3089779317378998,
      "logps/chosen": -1.0933566093444824,
      "logps/rejected": -6.80075216293335,
      "loss": 2.4313,
      "nll_loss": 2.393275022506714,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10933566093444824,
      "rewards/margins": 0.5707396268844604,
      "rewards/rejected": -0.6800752282142639,
      "step": 1860
    },
    {
      "epoch": 1.1576982892690513,
      "grad_norm": 0.5961266160011292,
      "learning_rate": 3.4750000000000006e-06,
      "log_odds_chosen": 7.8994646072387695,
      "log_odds_ratio": -0.1790763884782791,
      "logits/chosen": 0.2821861505508423,
      "logits/rejected": 0.820894718170166,
      "logps/chosen": -1.0732139348983765,
      "logps/rejected": -8.610696792602539,
      "loss": 3.1621,
      "nll_loss": 3.1442208290100098,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10732139647006989,
      "rewards/margins": 0.7537482976913452,
      "rewards/rejected": -0.8610696196556091,
      "step": 1861
    },
    {
      "epoch": 1.1583203732503888,
      "grad_norm": 0.5593898892402649,
      "learning_rate": 3.4500000000000004e-06,
      "log_odds_chosen": 15.28304386138916,
      "log_odds_ratio": -0.00035143038257956505,
      "logits/chosen": 0.2785932719707489,
      "logits/rejected": 1.468186855316162,
      "logps/chosen": -1.0075515508651733,
      "logps/rejected": -15.71903133392334,
      "loss": 3.0106,
      "nll_loss": 3.0106000900268555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10075514763593674,
      "rewards/margins": 1.4711480140686035,
      "rewards/rejected": -1.571903109550476,
      "step": 1862
    },
    {
      "epoch": 1.1589424572317264,
      "grad_norm": 0.6021642684936523,
      "learning_rate": 3.4250000000000002e-06,
      "log_odds_chosen": 9.942659378051758,
      "log_odds_ratio": -0.0021312516182661057,
      "logits/chosen": 0.46570685505867004,
      "logits/rejected": 1.2161035537719727,
      "logps/chosen": -0.8891934156417847,
      "logps/rejected": -10.150493621826172,
      "loss": 3.5963,
      "nll_loss": 3.596067428588867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08891934156417847,
      "rewards/margins": 0.9261300563812256,
      "rewards/rejected": -1.0150494575500488,
      "step": 1863
    },
    {
      "epoch": 1.1595645412130637,
      "grad_norm": 0.5865781903266907,
      "learning_rate": 3.4000000000000005e-06,
      "log_odds_chosen": 11.61959171295166,
      "log_odds_ratio": -0.0861300453543663,
      "logits/chosen": 0.24954769015312195,
      "logits/rejected": 0.8293600082397461,
      "logps/chosen": -0.9044547080993652,
      "logps/rejected": -11.792139053344727,
      "loss": 3.3048,
      "nll_loss": 3.296236515045166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09044547379016876,
      "rewards/margins": 1.088768482208252,
      "rewards/rejected": -1.1792140007019043,
      "step": 1864
    },
    {
      "epoch": 1.1601866251944013,
      "grad_norm": 0.6469467282295227,
      "learning_rate": 3.3750000000000003e-06,
      "log_odds_chosen": 14.448548316955566,
      "log_odds_ratio": -0.00036828042357228696,
      "logits/chosen": 0.365757554769516,
      "logits/rejected": 1.18428635597229,
      "logps/chosen": -0.9528321027755737,
      "logps/rejected": -14.689295768737793,
      "loss": 3.5187,
      "nll_loss": 3.518705129623413,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09528321772813797,
      "rewards/margins": 1.3736464977264404,
      "rewards/rejected": -1.4689295291900635,
      "step": 1865
    },
    {
      "epoch": 1.1608087091757386,
      "grad_norm": 0.9285478591918945,
      "learning_rate": 3.3500000000000005e-06,
      "log_odds_chosen": 14.083860397338867,
      "log_odds_ratio": -0.07330363243818283,
      "logits/chosen": 0.35714104771614075,
      "logits/rejected": 1.0330679416656494,
      "logps/chosen": -1.1653668880462646,
      "logps/rejected": -14.7406005859375,
      "loss": 2.6367,
      "nll_loss": 2.6293601989746094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1165367066860199,
      "rewards/margins": 1.3575233221054077,
      "rewards/rejected": -1.4740599393844604,
      "step": 1866
    },
    {
      "epoch": 1.1614307931570762,
      "grad_norm": 0.9891173243522644,
      "learning_rate": 3.3250000000000004e-06,
      "log_odds_chosen": 9.35483169555664,
      "log_odds_ratio": -0.08359131217002869,
      "logits/chosen": -0.014555513858795166,
      "logits/rejected": 0.7392335534095764,
      "logps/chosen": -1.41230046749115,
      "logps/rejected": -10.510553359985352,
      "loss": 2.5851,
      "nll_loss": 2.576704978942871,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.141230046749115,
      "rewards/margins": 0.909825325012207,
      "rewards/rejected": -1.0510554313659668,
      "step": 1867
    },
    {
      "epoch": 1.1620528771384138,
      "grad_norm": 0.5011563897132874,
      "learning_rate": 3.3e-06,
      "log_odds_chosen": 13.634634017944336,
      "log_odds_ratio": -0.0271232258528471,
      "logits/chosen": 0.1796010583639145,
      "logits/rejected": 0.7554137110710144,
      "logps/chosen": -1.1477197408676147,
      "logps/rejected": -14.320469856262207,
      "loss": 3.1094,
      "nll_loss": 3.106682300567627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11477198451757431,
      "rewards/margins": 1.317275047302246,
      "rewards/rejected": -1.4320470094680786,
      "step": 1868
    },
    {
      "epoch": 1.162674961119751,
      "grad_norm": 0.9561259746551514,
      "learning_rate": 3.2750000000000004e-06,
      "log_odds_chosen": 6.736767768859863,
      "log_odds_ratio": -0.26292210817337036,
      "logits/chosen": 0.23208418488502502,
      "logits/rejected": 0.7597588300704956,
      "logps/chosen": -1.131181001663208,
      "logps/rejected": -7.403827667236328,
      "loss": 3.0907,
      "nll_loss": 3.06437349319458,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11311808228492737,
      "rewards/margins": 0.6272647380828857,
      "rewards/rejected": -0.7403827905654907,
      "step": 1869
    },
    {
      "epoch": 1.1632970451010887,
      "grad_norm": 0.5985185503959656,
      "learning_rate": 3.2500000000000002e-06,
      "log_odds_chosen": 8.262455940246582,
      "log_odds_ratio": -0.18344323337078094,
      "logits/chosen": 0.3155595064163208,
      "logits/rejected": 1.0940701961517334,
      "logps/chosen": -1.1838172674179077,
      "logps/rejected": -9.031631469726562,
      "loss": 2.8803,
      "nll_loss": 2.8619325160980225,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11838172376155853,
      "rewards/margins": 0.7847813963890076,
      "rewards/rejected": -0.9031630754470825,
      "step": 1870
    },
    {
      "epoch": 1.163919129082426,
      "grad_norm": 0.4131832420825958,
      "learning_rate": 3.225e-06,
      "log_odds_chosen": 12.02147102355957,
      "log_odds_ratio": -2.4021293938858435e-05,
      "logits/chosen": 0.40716037154197693,
      "logits/rejected": 1.4584306478500366,
      "logps/chosen": -1.1131404638290405,
      "logps/rejected": -12.671215057373047,
      "loss": 3.6546,
      "nll_loss": 3.6546096801757812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11131405085325241,
      "rewards/margins": 1.1558074951171875,
      "rewards/rejected": -1.2671215534210205,
      "step": 1871
    },
    {
      "epoch": 1.1645412130637636,
      "grad_norm": 0.4233200252056122,
      "learning_rate": 3.2000000000000003e-06,
      "log_odds_chosen": 4.853551864624023,
      "log_odds_ratio": -0.28688034415245056,
      "logits/chosen": 0.36333611607551575,
      "logits/rejected": 0.8579514026641846,
      "logps/chosen": -0.8897204399108887,
      "logps/rejected": -5.366914749145508,
      "loss": 3.5207,
      "nll_loss": 3.4920222759246826,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08897203207015991,
      "rewards/margins": 0.4477194547653198,
      "rewards/rejected": -0.5366915464401245,
      "step": 1872
    },
    {
      "epoch": 1.1651632970451011,
      "grad_norm": 0.765011191368103,
      "learning_rate": 3.175e-06,
      "log_odds_chosen": 12.80850887298584,
      "log_odds_ratio": -0.07324609905481339,
      "logits/chosen": 0.21928678452968597,
      "logits/rejected": 0.8805117607116699,
      "logps/chosen": -1.0177444219589233,
      "logps/rejected": -13.257856369018555,
      "loss": 2.8547,
      "nll_loss": 2.847330331802368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1017744317650795,
      "rewards/margins": 1.2240111827850342,
      "rewards/rejected": -1.325785517692566,
      "step": 1873
    },
    {
      "epoch": 1.1657853810264385,
      "grad_norm": 0.777523934841156,
      "learning_rate": 3.1500000000000003e-06,
      "log_odds_chosen": 10.86896800994873,
      "log_odds_ratio": -0.01855648122727871,
      "logits/chosen": 0.12363356351852417,
      "logits/rejected": 0.8761640787124634,
      "logps/chosen": -0.9145961999893188,
      "logps/rejected": -11.168540954589844,
      "loss": 2.5963,
      "nll_loss": 2.594484806060791,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09145961701869965,
      "rewards/margins": 1.0253944396972656,
      "rewards/rejected": -1.116853952407837,
      "step": 1874
    },
    {
      "epoch": 1.166407465007776,
      "grad_norm": 0.7729889154434204,
      "learning_rate": 3.125e-06,
      "log_odds_chosen": 11.862380981445312,
      "log_odds_ratio": -0.10916010290384293,
      "logits/chosen": 0.27635759115219116,
      "logits/rejected": 1.2793161869049072,
      "logps/chosen": -1.297967791557312,
      "logps/rejected": -12.863824844360352,
      "loss": 2.5581,
      "nll_loss": 2.5471975803375244,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12979678809642792,
      "rewards/margins": 1.156585693359375,
      "rewards/rejected": -1.2863824367523193,
      "step": 1875
    },
    {
      "epoch": 1.1670295489891136,
      "grad_norm": 0.5068461894989014,
      "learning_rate": 3.1e-06,
      "log_odds_chosen": 10.792291641235352,
      "log_odds_ratio": -0.23234255611896515,
      "logits/chosen": 0.22562365233898163,
      "logits/rejected": 0.9575470685958862,
      "logps/chosen": -1.1553939580917358,
      "logps/rejected": -11.633508682250977,
      "loss": 3.1962,
      "nll_loss": 3.173001289367676,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11553940176963806,
      "rewards/margins": 1.0478113889694214,
      "rewards/rejected": -1.1633508205413818,
      "step": 1876
    },
    {
      "epoch": 1.167651632970451,
      "grad_norm": 0.5159857273101807,
      "learning_rate": 3.075e-06,
      "log_odds_chosen": 11.215340614318848,
      "log_odds_ratio": -0.0021687590051442385,
      "logits/chosen": 0.31632381677627563,
      "logits/rejected": 0.8091593384742737,
      "logps/chosen": -1.214293122291565,
      "logps/rejected": -11.843792915344238,
      "loss": 3.2419,
      "nll_loss": 3.2416889667510986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12142931669950485,
      "rewards/margins": 1.0629500150680542,
      "rewards/rejected": -1.1843793392181396,
      "step": 1877
    },
    {
      "epoch": 1.1682737169517885,
      "grad_norm": 0.7218311429023743,
      "learning_rate": 3.05e-06,
      "log_odds_chosen": 10.48986530303955,
      "log_odds_ratio": -0.13859295845031738,
      "logits/chosen": 0.20560768246650696,
      "logits/rejected": 0.9777731895446777,
      "logps/chosen": -1.1056513786315918,
      "logps/rejected": -11.218057632446289,
      "loss": 2.6993,
      "nll_loss": 2.685471534729004,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11056514084339142,
      "rewards/margins": 1.0112406015396118,
      "rewards/rejected": -1.1218056678771973,
      "step": 1878
    },
    {
      "epoch": 1.168895800933126,
      "grad_norm": 0.5191105604171753,
      "learning_rate": 3.0250000000000003e-06,
      "log_odds_chosen": 10.223533630371094,
      "log_odds_ratio": -0.09211105108261108,
      "logits/chosen": 0.2968738079071045,
      "logits/rejected": 1.1603482961654663,
      "logps/chosen": -1.0198242664337158,
      "logps/rejected": -10.802665710449219,
      "loss": 2.9144,
      "nll_loss": 2.9052276611328125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10198242217302322,
      "rewards/margins": 0.9782842993736267,
      "rewards/rejected": -1.0802667140960693,
      "step": 1879
    },
    {
      "epoch": 1.1695178849144634,
      "grad_norm": 0.8860030770301819,
      "learning_rate": 3e-06,
      "log_odds_chosen": 12.134418487548828,
      "log_odds_ratio": -0.02829170599579811,
      "logits/chosen": 0.06693208962678909,
      "logits/rejected": 1.0038888454437256,
      "logps/chosen": -0.9355678558349609,
      "logps/rejected": -12.557945251464844,
      "loss": 2.4539,
      "nll_loss": 2.45109486579895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09355678409337997,
      "rewards/margins": 1.1622377634048462,
      "rewards/rejected": -1.2557945251464844,
      "step": 1880
    },
    {
      "epoch": 1.170139968895801,
      "grad_norm": 0.536885678768158,
      "learning_rate": 2.975e-06,
      "log_odds_chosen": 11.629366874694824,
      "log_odds_ratio": -0.14055614173412323,
      "logits/chosen": 0.39352768659591675,
      "logits/rejected": 0.9422281980514526,
      "logps/chosen": -1.0389788150787354,
      "logps/rejected": -12.211631774902344,
      "loss": 3.2538,
      "nll_loss": 3.2397899627685547,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10389788448810577,
      "rewards/margins": 1.1172653436660767,
      "rewards/rejected": -1.2211631536483765,
      "step": 1881
    },
    {
      "epoch": 1.1707620528771385,
      "grad_norm": 0.5991978645324707,
      "learning_rate": 2.95e-06,
      "log_odds_chosen": 11.063156127929688,
      "log_odds_ratio": -0.03398752585053444,
      "logits/chosen": 0.30659687519073486,
      "logits/rejected": 1.1551581621170044,
      "logps/chosen": -1.0710886716842651,
      "logps/rejected": -11.712461471557617,
      "loss": 3.1061,
      "nll_loss": 3.1026854515075684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10710887610912323,
      "rewards/margins": 1.06413733959198,
      "rewards/rejected": -1.1712462902069092,
      "step": 1882
    },
    {
      "epoch": 1.1713841368584759,
      "grad_norm": 0.41771650314331055,
      "learning_rate": 2.9250000000000004e-06,
      "log_odds_chosen": 11.414490699768066,
      "log_odds_ratio": -0.0612252876162529,
      "logits/chosen": 0.31374531984329224,
      "logits/rejected": 1.0078083276748657,
      "logps/chosen": -0.7967260479927063,
      "logps/rejected": -11.597383499145508,
      "loss": 3.441,
      "nll_loss": 3.4348974227905273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07967260479927063,
      "rewards/margins": 1.0800657272338867,
      "rewards/rejected": -1.159738302230835,
      "step": 1883
    },
    {
      "epoch": 1.1720062208398134,
      "grad_norm": 0.5465978384017944,
      "learning_rate": 2.9e-06,
      "log_odds_chosen": 12.942608833312988,
      "log_odds_ratio": -0.027714960277080536,
      "logits/chosen": 0.2509208023548126,
      "logits/rejected": 1.0975139141082764,
      "logps/chosen": -0.9628081321716309,
      "logps/rejected": -13.391410827636719,
      "loss": 2.9462,
      "nll_loss": 2.943429470062256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09628081321716309,
      "rewards/margins": 1.2428603172302246,
      "rewards/rejected": -1.3391410112380981,
      "step": 1884
    },
    {
      "epoch": 1.1726283048211508,
      "grad_norm": 0.5699530839920044,
      "learning_rate": 2.8750000000000004e-06,
      "log_odds_chosen": 9.245712280273438,
      "log_odds_ratio": -0.09234052896499634,
      "logits/chosen": 0.12563082575798035,
      "logits/rejected": 0.7737298011779785,
      "logps/chosen": -0.9688528776168823,
      "logps/rejected": -9.617731094360352,
      "loss": 2.6494,
      "nll_loss": 2.640190362930298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09688529372215271,
      "rewards/margins": 0.8648878931999207,
      "rewards/rejected": -0.961773157119751,
      "step": 1885
    },
    {
      "epoch": 1.1732503888024883,
      "grad_norm": 0.5037267208099365,
      "learning_rate": 2.8500000000000002e-06,
      "log_odds_chosen": 11.182029724121094,
      "log_odds_ratio": -0.0029421194922178984,
      "logits/chosen": 0.27966320514678955,
      "logits/rejected": 1.0112600326538086,
      "logps/chosen": -0.8653491735458374,
      "logps/rejected": -11.44352912902832,
      "loss": 3.3134,
      "nll_loss": 3.313063144683838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08653491735458374,
      "rewards/margins": 1.0578179359436035,
      "rewards/rejected": -1.144352912902832,
      "step": 1886
    },
    {
      "epoch": 1.173872472783826,
      "grad_norm": 0.6436689496040344,
      "learning_rate": 2.825e-06,
      "log_odds_chosen": 8.10422420501709,
      "log_odds_ratio": -0.1899256408214569,
      "logits/chosen": 0.2365320920944214,
      "logits/rejected": 0.6297204494476318,
      "logps/chosen": -0.9122384190559387,
      "logps/rejected": -8.375236511230469,
      "loss": 3.0991,
      "nll_loss": 3.0801029205322266,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0912238359451294,
      "rewards/margins": 0.7462998628616333,
      "rewards/rejected": -0.8375236988067627,
      "step": 1887
    },
    {
      "epoch": 1.1744945567651632,
      "grad_norm": 0.7984195947647095,
      "learning_rate": 2.8000000000000003e-06,
      "log_odds_chosen": 6.958744049072266,
      "log_odds_ratio": -0.2660243809223175,
      "logits/chosen": 0.23720431327819824,
      "logits/rejected": 0.4664103388786316,
      "logps/chosen": -1.3364598751068115,
      "logps/rejected": -8.00857925415039,
      "loss": 3.1654,
      "nll_loss": 3.138822317123413,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13364599645137787,
      "rewards/margins": 0.6672119498252869,
      "rewards/rejected": -0.8008580207824707,
      "step": 1888
    },
    {
      "epoch": 1.1751166407465008,
      "grad_norm": 0.3695003092288971,
      "learning_rate": 2.775e-06,
      "log_odds_chosen": 10.706849098205566,
      "log_odds_ratio": -0.013503124937415123,
      "logits/chosen": 0.39301347732543945,
      "logits/rejected": 1.17519211769104,
      "logps/chosen": -1.2024290561676025,
      "logps/rejected": -11.415552139282227,
      "loss": 3.8668,
      "nll_loss": 3.865440607070923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12024290859699249,
      "rewards/margins": 1.0213122367858887,
      "rewards/rejected": -1.1415551900863647,
      "step": 1889
    },
    {
      "epoch": 1.1757387247278381,
      "grad_norm": 0.4984581768512726,
      "learning_rate": 2.7500000000000004e-06,
      "log_odds_chosen": 9.745952606201172,
      "log_odds_ratio": -0.015363501384854317,
      "logits/chosen": 0.22919657826423645,
      "logits/rejected": 0.6598438024520874,
      "logps/chosen": -0.9327662587165833,
      "logps/rejected": -9.936866760253906,
      "loss": 3.3242,
      "nll_loss": 3.322669506072998,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09327662736177444,
      "rewards/margins": 0.9004101157188416,
      "rewards/rejected": -0.9936867356300354,
      "step": 1890
    },
    {
      "epoch": 1.1763608087091757,
      "grad_norm": 0.47122901678085327,
      "learning_rate": 2.725e-06,
      "log_odds_chosen": 5.912145614624023,
      "log_odds_ratio": -0.23905989527702332,
      "logits/chosen": 0.11490034312009811,
      "logits/rejected": 0.5542351007461548,
      "logps/chosen": -1.2273865938186646,
      "logps/rejected": -6.7846503257751465,
      "loss": 3.0799,
      "nll_loss": 3.0560312271118164,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12273865938186646,
      "rewards/margins": 0.5557263493537903,
      "rewards/rejected": -0.6784650087356567,
      "step": 1891
    },
    {
      "epoch": 1.1769828926905133,
      "grad_norm": 0.5710389018058777,
      "learning_rate": 2.7e-06,
      "log_odds_chosen": 8.211050987243652,
      "log_odds_ratio": -0.14245814085006714,
      "logits/chosen": 0.22267159819602966,
      "logits/rejected": 0.8585968613624573,
      "logps/chosen": -0.9339166879653931,
      "logps/rejected": -8.653953552246094,
      "loss": 2.8897,
      "nll_loss": 2.875443458557129,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09339166432619095,
      "rewards/margins": 0.7720035910606384,
      "rewards/rejected": -0.8653952479362488,
      "step": 1892
    },
    {
      "epoch": 1.1776049766718506,
      "grad_norm": 0.8877511024475098,
      "learning_rate": 2.6750000000000002e-06,
      "log_odds_chosen": 11.314347267150879,
      "log_odds_ratio": -0.1028999611735344,
      "logits/chosen": 0.08349957317113876,
      "logits/rejected": 0.7912660837173462,
      "logps/chosen": -0.825186550617218,
      "logps/rejected": -11.350780487060547,
      "loss": 2.194,
      "nll_loss": 2.183689832687378,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08251865208148956,
      "rewards/margins": 1.0525593757629395,
      "rewards/rejected": -1.135077953338623,
      "step": 1893
    },
    {
      "epoch": 1.1782270606531882,
      "grad_norm": 0.6872034072875977,
      "learning_rate": 2.65e-06,
      "log_odds_chosen": 11.63272762298584,
      "log_odds_ratio": -0.024180106818675995,
      "logits/chosen": 0.2922358214855194,
      "logits/rejected": 1.0772125720977783,
      "logps/chosen": -1.1416473388671875,
      "logps/rejected": -12.256592750549316,
      "loss": 2.7443,
      "nll_loss": 2.7419087886810303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11416472494602203,
      "rewards/margins": 1.111494541168213,
      "rewards/rejected": -1.2256593704223633,
      "step": 1894
    },
    {
      "epoch": 1.1788491446345257,
      "grad_norm": 0.5088832974433899,
      "learning_rate": 2.625e-06,
      "log_odds_chosen": 7.584236145019531,
      "log_odds_ratio": -0.17331501841545105,
      "logits/chosen": 0.10206744074821472,
      "logits/rejected": 0.6197749376296997,
      "logps/chosen": -1.1107993125915527,
      "logps/rejected": -8.259223937988281,
      "loss": 2.9389,
      "nll_loss": 2.92161226272583,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11107993870973587,
      "rewards/margins": 0.7148424983024597,
      "rewards/rejected": -0.8259223699569702,
      "step": 1895
    },
    {
      "epoch": 1.179471228615863,
      "grad_norm": 2.0029592514038086,
      "learning_rate": 2.6e-06,
      "log_odds_chosen": 8.409797668457031,
      "log_odds_ratio": -0.11552348732948303,
      "logits/chosen": 0.2863270342350006,
      "logits/rejected": 0.6867630481719971,
      "logps/chosen": -1.2449003458023071,
      "logps/rejected": -9.326054573059082,
      "loss": 3.5466,
      "nll_loss": 3.53503680229187,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12449003756046295,
      "rewards/margins": 0.8081154227256775,
      "rewards/rejected": -0.932605504989624,
      "step": 1896
    },
    {
      "epoch": 1.1800933125972006,
      "grad_norm": 3.2555623054504395,
      "learning_rate": 2.575e-06,
      "log_odds_chosen": 9.229421615600586,
      "log_odds_ratio": -0.47473472356796265,
      "logits/chosen": 0.11302335560321808,
      "logits/rejected": 0.9297094345092773,
      "logps/chosen": -1.431499719619751,
      "logps/rejected": -9.87209701538086,
      "loss": 2.771,
      "nll_loss": 2.7234785556793213,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1431499719619751,
      "rewards/margins": 0.8440598249435425,
      "rewards/rejected": -0.9872097373008728,
      "step": 1897
    },
    {
      "epoch": 1.1807153965785382,
      "grad_norm": 0.5698276162147522,
      "learning_rate": 2.55e-06,
      "log_odds_chosen": 7.369045257568359,
      "log_odds_ratio": -0.22586429119110107,
      "logits/chosen": 0.23034963011741638,
      "logits/rejected": 0.6743384003639221,
      "logps/chosen": -1.0172677040100098,
      "logps/rejected": -7.886266708374023,
      "loss": 3.0338,
      "nll_loss": 3.0111746788024902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10172677040100098,
      "rewards/margins": 0.6868999004364014,
      "rewards/rejected": -0.7886266708374023,
      "step": 1898
    },
    {
      "epoch": 1.1813374805598755,
      "grad_norm": 0.622840404510498,
      "learning_rate": 2.5250000000000004e-06,
      "log_odds_chosen": 11.24142837524414,
      "log_odds_ratio": -0.08297882229089737,
      "logits/chosen": 0.16179004311561584,
      "logits/rejected": 1.013938069343567,
      "logps/chosen": -1.1062037944793701,
      "logps/rejected": -11.925838470458984,
      "loss": 2.5435,
      "nll_loss": 2.5351955890655518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11062037199735641,
      "rewards/margins": 1.0819635391235352,
      "rewards/rejected": -1.1925837993621826,
      "step": 1899
    },
    {
      "epoch": 1.181959564541213,
      "grad_norm": 0.5149745345115662,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 7.838262557983398,
      "log_odds_ratio": -0.16004499793052673,
      "logits/chosen": 0.2962034046649933,
      "logits/rejected": 0.741732120513916,
      "logps/chosen": -0.8723784685134888,
      "logps/rejected": -8.200945854187012,
      "loss": 3.3196,
      "nll_loss": 3.3036234378814697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08723784238100052,
      "rewards/margins": 0.7328566908836365,
      "rewards/rejected": -0.8200945854187012,
      "step": 1900
    },
    {
      "epoch": 1.1825816485225507,
      "grad_norm": 0.4984101951122284,
      "learning_rate": 2.4750000000000004e-06,
      "log_odds_chosen": 14.134041786193848,
      "log_odds_ratio": -6.005212526360992e-06,
      "logits/chosen": 0.24021825194358826,
      "logits/rejected": 1.185408592224121,
      "logps/chosen": -1.0472640991210938,
      "logps/rejected": -14.570968627929688,
      "loss": 3.3729,
      "nll_loss": 3.372877359390259,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10472641885280609,
      "rewards/margins": 1.3523705005645752,
      "rewards/rejected": -1.457096815109253,
      "step": 1901
    },
    {
      "epoch": 1.183203732503888,
      "grad_norm": 0.8151748180389404,
      "learning_rate": 2.4500000000000003e-06,
      "log_odds_chosen": 12.901820182800293,
      "log_odds_ratio": -0.18067723512649536,
      "logits/chosen": 0.2967347502708435,
      "logits/rejected": 0.981220006942749,
      "logps/chosen": -1.1056855916976929,
      "logps/rejected": -13.67314624786377,
      "loss": 2.6885,
      "nll_loss": 2.67038631439209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1105685606598854,
      "rewards/margins": 1.2567460536956787,
      "rewards/rejected": -1.3673145771026611,
      "step": 1902
    },
    {
      "epoch": 1.1838258164852256,
      "grad_norm": 0.5125972628593445,
      "learning_rate": 2.425e-06,
      "log_odds_chosen": 8.416955947875977,
      "log_odds_ratio": -0.2502775192260742,
      "logits/chosen": 0.32387620210647583,
      "logits/rejected": 0.6450578570365906,
      "logps/chosen": -1.0750399827957153,
      "logps/rejected": -9.137263298034668,
      "loss": 3.3234,
      "nll_loss": 3.2983758449554443,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1075039878487587,
      "rewards/margins": 0.8062223792076111,
      "rewards/rejected": -0.9137263298034668,
      "step": 1903
    },
    {
      "epoch": 1.184447900466563,
      "grad_norm": 8.312177658081055,
      "learning_rate": 2.4000000000000003e-06,
      "log_odds_chosen": 5.083847522735596,
      "log_odds_ratio": -0.8792902231216431,
      "logits/chosen": 0.3725608289241791,
      "logits/rejected": 0.6591469049453735,
      "logps/chosen": -1.7544732093811035,
      "logps/rejected": -6.63493013381958,
      "loss": 2.8066,
      "nll_loss": 2.718681812286377,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17544731497764587,
      "rewards/margins": 0.48804575204849243,
      "rewards/rejected": -0.6634930372238159,
      "step": 1904
    },
    {
      "epoch": 1.1850699844479005,
      "grad_norm": 0.5036944150924683,
      "learning_rate": 2.375e-06,
      "log_odds_chosen": 11.233821868896484,
      "log_odds_ratio": -0.07140740752220154,
      "logits/chosen": 0.28917139768600464,
      "logits/rejected": 1.0645439624786377,
      "logps/chosen": -0.9894380569458008,
      "logps/rejected": -11.69849967956543,
      "loss": 3.0867,
      "nll_loss": 3.0795483589172363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0989438146352768,
      "rewards/margins": 1.0709060430526733,
      "rewards/rejected": -1.1698498725891113,
      "step": 1905
    },
    {
      "epoch": 1.185692068429238,
      "grad_norm": 0.6781418323516846,
      "learning_rate": 2.35e-06,
      "log_odds_chosen": 11.125399589538574,
      "log_odds_ratio": -0.11678078025579453,
      "logits/chosen": 0.1555495262145996,
      "logits/rejected": 0.8147391080856323,
      "logps/chosen": -0.9576234221458435,
      "logps/rejected": -11.602968215942383,
      "loss": 2.5192,
      "nll_loss": 2.507535219192505,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09576234221458435,
      "rewards/margins": 1.0645344257354736,
      "rewards/rejected": -1.1602967977523804,
      "step": 1906
    },
    {
      "epoch": 1.1863141524105754,
      "grad_norm": 0.44551971554756165,
      "learning_rate": 2.325e-06,
      "log_odds_chosen": 12.629438400268555,
      "log_odds_ratio": -0.014774455688893795,
      "logits/chosen": 0.3876957595348358,
      "logits/rejected": 1.470685362815857,
      "logps/chosen": -1.0328607559204102,
      "logps/rejected": -13.184697151184082,
      "loss": 3.1014,
      "nll_loss": 3.099900960922241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10328607261180878,
      "rewards/margins": 1.2151837348937988,
      "rewards/rejected": -1.318469762802124,
      "step": 1907
    },
    {
      "epoch": 1.186936236391913,
      "grad_norm": 0.7646956443786621,
      "learning_rate": 2.3e-06,
      "log_odds_chosen": 12.166587829589844,
      "log_odds_ratio": -0.05765797197818756,
      "logits/chosen": 0.31056147813796997,
      "logits/rejected": 1.0030930042266846,
      "logps/chosen": -0.8595593571662903,
      "logps/rejected": -12.483160018920898,
      "loss": 2.7176,
      "nll_loss": 2.7118818759918213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08595594763755798,
      "rewards/margins": 1.1623600721359253,
      "rewards/rejected": -1.2483160495758057,
      "step": 1908
    },
    {
      "epoch": 1.1875583203732503,
      "grad_norm": 0.5104154348373413,
      "learning_rate": 2.2750000000000002e-06,
      "log_odds_chosen": 10.230204582214355,
      "log_odds_ratio": -0.021832279860973358,
      "logits/chosen": 0.19930848479270935,
      "logits/rejected": 0.8651219606399536,
      "logps/chosen": -1.2513785362243652,
      "logps/rejected": -11.117950439453125,
      "loss": 3.0008,
      "nll_loss": 2.9986038208007812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12513785064220428,
      "rewards/margins": 0.9866572618484497,
      "rewards/rejected": -1.11179518699646,
      "step": 1909
    },
    {
      "epoch": 1.1881804043545878,
      "grad_norm": 0.45462679862976074,
      "learning_rate": 2.25e-06,
      "log_odds_chosen": 10.267870903015137,
      "log_odds_ratio": -0.04462050646543503,
      "logits/chosen": 0.2697553336620331,
      "logits/rejected": 1.0339744091033936,
      "logps/chosen": -0.8061079382896423,
      "logps/rejected": -10.42862319946289,
      "loss": 2.9761,
      "nll_loss": 2.9715969562530518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08061079680919647,
      "rewards/margins": 0.9622515439987183,
      "rewards/rejected": -1.0428624153137207,
      "step": 1910
    },
    {
      "epoch": 1.1888024883359254,
      "grad_norm": 0.6434316635131836,
      "learning_rate": 2.225e-06,
      "log_odds_chosen": 15.738075256347656,
      "log_odds_ratio": -1.4901207805451122e-06,
      "logits/chosen": 0.23433563113212585,
      "logits/rejected": 1.4968116283416748,
      "logps/chosen": -1.2706167697906494,
      "logps/rejected": -16.622678756713867,
      "loss": 2.814,
      "nll_loss": 2.8140363693237305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12706167995929718,
      "rewards/margins": 1.5352060794830322,
      "rewards/rejected": -1.662267804145813,
      "step": 1911
    },
    {
      "epoch": 1.1894245723172627,
      "grad_norm": 0.5044808387756348,
      "learning_rate": 2.2e-06,
      "log_odds_chosen": 10.514906883239746,
      "log_odds_ratio": -0.1468035727739334,
      "logits/chosen": 0.1761753112077713,
      "logits/rejected": 0.8280108571052551,
      "logps/chosen": -1.1078742742538452,
      "logps/rejected": -11.255012512207031,
      "loss": 2.6505,
      "nll_loss": 2.6358470916748047,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11078742891550064,
      "rewards/margins": 1.0147137641906738,
      "rewards/rejected": -1.125501275062561,
      "step": 1912
    },
    {
      "epoch": 1.1900466562986003,
      "grad_norm": 0.5661005973815918,
      "learning_rate": 2.175e-06,
      "log_odds_chosen": 10.606189727783203,
      "log_odds_ratio": -0.3332817554473877,
      "logits/chosen": 0.35058706998825073,
      "logits/rejected": 0.8177105188369751,
      "logps/chosen": -0.9934715032577515,
      "logps/rejected": -11.091846466064453,
      "loss": 3.4516,
      "nll_loss": 3.4183170795440674,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09934714436531067,
      "rewards/margins": 1.0098373889923096,
      "rewards/rejected": -1.1091846227645874,
      "step": 1913
    },
    {
      "epoch": 1.1906687402799379,
      "grad_norm": 4.56205940246582,
      "learning_rate": 2.1499999999999997e-06,
      "log_odds_chosen": 10.877635955810547,
      "log_odds_ratio": -0.10927483439445496,
      "logits/chosen": 0.3463170528411865,
      "logits/rejected": 0.9182144403457642,
      "logps/chosen": -1.5428080558776855,
      "logps/rejected": -12.063437461853027,
      "loss": 3.6504,
      "nll_loss": 3.639470100402832,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15428081154823303,
      "rewards/margins": 1.05206298828125,
      "rewards/rejected": -1.2063437700271606,
      "step": 1914
    },
    {
      "epoch": 1.1912908242612752,
      "grad_norm": 0.582009494304657,
      "learning_rate": 2.1250000000000004e-06,
      "log_odds_chosen": 11.911093711853027,
      "log_odds_ratio": -0.023971393704414368,
      "logits/chosen": 0.4232202172279358,
      "logits/rejected": 1.2884031534194946,
      "logps/chosen": -1.0782442092895508,
      "logps/rejected": -12.535236358642578,
      "loss": 3.2242,
      "nll_loss": 3.2217798233032227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1078244298696518,
      "rewards/margins": 1.1456992626190186,
      "rewards/rejected": -1.2535237073898315,
      "step": 1915
    },
    {
      "epoch": 1.1919129082426128,
      "grad_norm": 0.4537990987300873,
      "learning_rate": 2.1000000000000002e-06,
      "log_odds_chosen": 12.903481483459473,
      "log_odds_ratio": -0.002569683827459812,
      "logits/chosen": 0.23533426225185394,
      "logits/rejected": 1.1300369501113892,
      "logps/chosen": -1.0497938394546509,
      "logps/rejected": -13.376167297363281,
      "loss": 2.9572,
      "nll_loss": 2.956960678100586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10497938096523285,
      "rewards/margins": 1.2326374053955078,
      "rewards/rejected": -1.3376166820526123,
      "step": 1916
    },
    {
      "epoch": 1.1925349922239503,
      "grad_norm": 0.5528674721717834,
      "learning_rate": 2.075e-06,
      "log_odds_chosen": 8.131418228149414,
      "log_odds_ratio": -0.12049117684364319,
      "logits/chosen": 0.35378021001815796,
      "logits/rejected": 1.0061860084533691,
      "logps/chosen": -0.9170488119125366,
      "logps/rejected": -8.42977523803711,
      "loss": 3.5667,
      "nll_loss": 3.5546751022338867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09170487523078918,
      "rewards/margins": 0.7512726783752441,
      "rewards/rejected": -0.8429775238037109,
      "step": 1917
    },
    {
      "epoch": 1.1931570762052877,
      "grad_norm": 0.46475347876548767,
      "learning_rate": 2.0500000000000003e-06,
      "log_odds_chosen": 9.595006942749023,
      "log_odds_ratio": -0.13916951417922974,
      "logits/chosen": 0.2627370357513428,
      "logits/rejected": 1.1726372241973877,
      "logps/chosen": -1.1297882795333862,
      "logps/rejected": -10.375534057617188,
      "loss": 3.1715,
      "nll_loss": 3.1576147079467773,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11297882348299026,
      "rewards/margins": 0.9245747327804565,
      "rewards/rejected": -1.0375535488128662,
      "step": 1918
    },
    {
      "epoch": 1.1937791601866252,
      "grad_norm": 0.8667030930519104,
      "learning_rate": 2.025e-06,
      "log_odds_chosen": 9.676931381225586,
      "log_odds_ratio": -0.1655566692352295,
      "logits/chosen": 0.3479841351509094,
      "logits/rejected": 1.114953637123108,
      "logps/chosen": -0.9280308485031128,
      "logps/rejected": -10.123562812805176,
      "loss": 3.0449,
      "nll_loss": 3.0283515453338623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09280309081077576,
      "rewards/margins": 0.9195532202720642,
      "rewards/rejected": -1.0123562812805176,
      "step": 1919
    },
    {
      "epoch": 1.1944012441679628,
      "grad_norm": 0.448517382144928,
      "learning_rate": 2.0000000000000003e-06,
      "log_odds_chosen": 9.664508819580078,
      "log_odds_ratio": -0.030703043565154076,
      "logits/chosen": 0.23289135098457336,
      "logits/rejected": 0.8441465497016907,
      "logps/chosen": -0.8699073791503906,
      "logps/rejected": -9.8641996383667,
      "loss": 3.4263,
      "nll_loss": 3.423267364501953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08699073642492294,
      "rewards/margins": 0.8994292616844177,
      "rewards/rejected": -0.9864200353622437,
      "step": 1920
    },
    {
      "epoch": 1.1950233281493001,
      "grad_norm": 0.5516140460968018,
      "learning_rate": 1.975e-06,
      "log_odds_chosen": 9.347766876220703,
      "log_odds_ratio": -0.031423501670360565,
      "logits/chosen": 0.2751286029815674,
      "logits/rejected": 0.904050350189209,
      "logps/chosen": -1.0117491483688354,
      "logps/rejected": -9.64893627166748,
      "loss": 3.015,
      "nll_loss": 3.0118672847747803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10117491334676743,
      "rewards/margins": 0.8637187480926514,
      "rewards/rejected": -0.964893639087677,
      "step": 1921
    },
    {
      "epoch": 1.1956454121306377,
      "grad_norm": 0.5590149760246277,
      "learning_rate": 1.95e-06,
      "log_odds_chosen": 9.179625511169434,
      "log_odds_ratio": -0.14231599867343903,
      "logits/chosen": 0.31218940019607544,
      "logits/rejected": 0.9436146020889282,
      "logps/chosen": -1.2564001083374023,
      "logps/rejected": -10.086761474609375,
      "loss": 3.4529,
      "nll_loss": 3.4386985301971436,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12564000487327576,
      "rewards/margins": 0.8830360770225525,
      "rewards/rejected": -1.0086760520935059,
      "step": 1922
    },
    {
      "epoch": 1.196267496111975,
      "grad_norm": 0.5352418422698975,
      "learning_rate": 1.925e-06,
      "log_odds_chosen": 9.404342651367188,
      "log_odds_ratio": -0.005691325291991234,
      "logits/chosen": 0.41761916875839233,
      "logits/rejected": 1.0610673427581787,
      "logps/chosen": -1.1509430408477783,
      "logps/rejected": -10.142035484313965,
      "loss": 3.3106,
      "nll_loss": 3.3100385665893555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11509430408477783,
      "rewards/margins": 0.8991092443466187,
      "rewards/rejected": -1.0142035484313965,
      "step": 1923
    },
    {
      "epoch": 1.1968895800933126,
      "grad_norm": 0.6540836095809937,
      "learning_rate": 1.9e-06,
      "log_odds_chosen": 12.195796966552734,
      "log_odds_ratio": -0.026557566598057747,
      "logits/chosen": 0.18317702412605286,
      "logits/rejected": 0.8327803611755371,
      "logps/chosen": -1.0502777099609375,
      "logps/rejected": -12.741544723510742,
      "loss": 2.6182,
      "nll_loss": 2.6155149936676025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10502777993679047,
      "rewards/margins": 1.1691266298294067,
      "rewards/rejected": -1.2741544246673584,
      "step": 1924
    },
    {
      "epoch": 1.1975116640746502,
      "grad_norm": 1.1352347135543823,
      "learning_rate": 1.875e-06,
      "log_odds_chosen": 11.687995910644531,
      "log_odds_ratio": -0.10948586463928223,
      "logits/chosen": 0.2166191041469574,
      "logits/rejected": 1.157933235168457,
      "logps/chosen": -0.9563741683959961,
      "logps/rejected": -12.046046257019043,
      "loss": 2.6403,
      "nll_loss": 2.6293962001800537,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09563742578029633,
      "rewards/margins": 1.1089673042297363,
      "rewards/rejected": -1.2046046257019043,
      "step": 1925
    },
    {
      "epoch": 1.1981337480559875,
      "grad_norm": 0.6544418334960938,
      "learning_rate": 1.85e-06,
      "log_odds_chosen": 7.253926753997803,
      "log_odds_ratio": -0.11793742328882217,
      "logits/chosen": 0.31673333048820496,
      "logits/rejected": 0.8479666709899902,
      "logps/chosen": -0.9824406504631042,
      "logps/rejected": -7.802856922149658,
      "loss": 2.611,
      "nll_loss": 2.599242687225342,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0982440710067749,
      "rewards/margins": 0.6820417046546936,
      "rewards/rejected": -0.7802857756614685,
      "step": 1926
    },
    {
      "epoch": 1.198755832037325,
      "grad_norm": 0.4343709945678711,
      "learning_rate": 1.8249999999999999e-06,
      "log_odds_chosen": 10.073572158813477,
      "log_odds_ratio": -0.23366577923297882,
      "logits/chosen": 0.3042418360710144,
      "logits/rejected": 0.5469282865524292,
      "logps/chosen": -0.8838576078414917,
      "logps/rejected": -10.546527862548828,
      "loss": 3.3211,
      "nll_loss": 3.2977826595306396,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08838576078414917,
      "rewards/margins": 0.9662670493125916,
      "rewards/rejected": -1.0546526908874512,
      "step": 1927
    },
    {
      "epoch": 1.1993779160186626,
      "grad_norm": 0.5009332299232483,
      "learning_rate": 1.8e-06,
      "log_odds_chosen": 12.118463516235352,
      "log_odds_ratio": -0.14113126695156097,
      "logits/chosen": 0.27596867084503174,
      "logits/rejected": 1.2588975429534912,
      "logps/chosen": -1.0183926820755005,
      "logps/rejected": -12.717033386230469,
      "loss": 3.1475,
      "nll_loss": 3.1334147453308105,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10183926671743393,
      "rewards/margins": 1.1698640584945679,
      "rewards/rejected": -1.2717033624649048,
      "step": 1928
    },
    {
      "epoch": 1.2,
      "grad_norm": 0.596346914768219,
      "learning_rate": 1.775e-06,
      "log_odds_chosen": 17.941753387451172,
      "log_odds_ratio": -1.1920931797249068e-07,
      "logits/chosen": 0.3379727900028229,
      "logits/rejected": 1.684706449508667,
      "logps/chosen": -1.0429511070251465,
      "logps/rejected": -18.38841438293457,
      "loss": 2.7556,
      "nll_loss": 2.7555999755859375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10429511219263077,
      "rewards/margins": 1.7345463037490845,
      "rewards/rejected": -1.838841438293457,
      "step": 1929
    },
    {
      "epoch": 1.2006220839813375,
      "grad_norm": 0.6008011102676392,
      "learning_rate": 1.7500000000000002e-06,
      "log_odds_chosen": 11.824764251708984,
      "log_odds_ratio": -0.3278602361679077,
      "logits/chosen": 0.272664338350296,
      "logits/rejected": 0.922380805015564,
      "logps/chosen": -1.0240790843963623,
      "logps/rejected": -12.52273178100586,
      "loss": 2.7195,
      "nll_loss": 2.6867575645446777,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10240790247917175,
      "rewards/margins": 1.1498652696609497,
      "rewards/rejected": -1.2522730827331543,
      "step": 1930
    },
    {
      "epoch": 1.2012441679626749,
      "grad_norm": 0.6269004344940186,
      "learning_rate": 1.7250000000000002e-06,
      "log_odds_chosen": 11.44296646118164,
      "log_odds_ratio": -0.16116118431091309,
      "logits/chosen": 0.34534481167793274,
      "logits/rejected": 0.9442611336708069,
      "logps/chosen": -0.6290194988250732,
      "logps/rejected": -11.348631858825684,
      "loss": 3.2711,
      "nll_loss": 3.2550272941589355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06290195137262344,
      "rewards/margins": 1.0719611644744873,
      "rewards/rejected": -1.134863257408142,
      "step": 1931
    },
    {
      "epoch": 1.2018662519440124,
      "grad_norm": 0.5232813954353333,
      "learning_rate": 1.7000000000000002e-06,
      "log_odds_chosen": 5.6479949951171875,
      "log_odds_ratio": -0.29262030124664307,
      "logits/chosen": 0.2010812908411026,
      "logits/rejected": 0.22653111815452576,
      "logps/chosen": -0.8776240348815918,
      "logps/rejected": -6.111688137054443,
      "loss": 3.0866,
      "nll_loss": 3.057365894317627,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08776240050792694,
      "rewards/margins": 0.5234063863754272,
      "rewards/rejected": -0.6111688017845154,
      "step": 1932
    },
    {
      "epoch": 1.20248833592535,
      "grad_norm": 0.6146247386932373,
      "learning_rate": 1.6750000000000003e-06,
      "log_odds_chosen": 11.847469329833984,
      "log_odds_ratio": -0.035451874136924744,
      "logits/chosen": 0.26600876450538635,
      "logits/rejected": 0.7060739398002625,
      "logps/chosen": -1.013869047164917,
      "logps/rejected": -12.30055046081543,
      "loss": 3.0871,
      "nll_loss": 3.0835165977478027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1013869121670723,
      "rewards/margins": 1.1286683082580566,
      "rewards/rejected": -1.2300550937652588,
      "step": 1933
    },
    {
      "epoch": 1.2031104199066873,
      "grad_norm": 0.5205911993980408,
      "learning_rate": 1.65e-06,
      "log_odds_chosen": 6.819182872772217,
      "log_odds_ratio": -0.23442748188972473,
      "logits/chosen": 0.2185009866952896,
      "logits/rejected": 0.6514573693275452,
      "logps/chosen": -0.9856964349746704,
      "logps/rejected": -7.467264652252197,
      "loss": 3.0668,
      "nll_loss": 3.0433638095855713,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09856964647769928,
      "rewards/margins": 0.6481568813323975,
      "rewards/rejected": -0.7467264533042908,
      "step": 1934
    },
    {
      "epoch": 1.2037325038880249,
      "grad_norm": 0.349277138710022,
      "learning_rate": 1.6250000000000001e-06,
      "log_odds_chosen": 11.960783004760742,
      "log_odds_ratio": -0.05431406572461128,
      "logits/chosen": 0.19932757318019867,
      "logits/rejected": 0.8002393245697021,
      "logps/chosen": -0.8757328391075134,
      "logps/rejected": -12.314295768737793,
      "loss": 3.3128,
      "nll_loss": 3.3073294162750244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08757328987121582,
      "rewards/margins": 1.1438562870025635,
      "rewards/rejected": -1.2314295768737793,
      "step": 1935
    },
    {
      "epoch": 1.2043545878693624,
      "grad_norm": 0.5340843200683594,
      "learning_rate": 1.6000000000000001e-06,
      "log_odds_chosen": 13.420051574707031,
      "log_odds_ratio": -0.00024055530957411975,
      "logits/chosen": 0.36679941415786743,
      "logits/rejected": 1.3175077438354492,
      "logps/chosen": -1.0074667930603027,
      "logps/rejected": -13.865694046020508,
      "loss": 2.8256,
      "nll_loss": 2.8255414962768555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10074667632579803,
      "rewards/margins": 1.2858227491378784,
      "rewards/rejected": -1.3865693807601929,
      "step": 1936
    },
    {
      "epoch": 1.2049766718506998,
      "grad_norm": 0.6181488037109375,
      "learning_rate": 1.5750000000000002e-06,
      "log_odds_chosen": 6.692278861999512,
      "log_odds_ratio": -0.2322840541601181,
      "logits/chosen": 0.4331367015838623,
      "logits/rejected": 0.7943879961967468,
      "logps/chosen": -0.8845175504684448,
      "logps/rejected": -6.956637859344482,
      "loss": 3.3336,
      "nll_loss": 3.3103556632995605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08845175057649612,
      "rewards/margins": 0.6072120666503906,
      "rewards/rejected": -0.6956638097763062,
      "step": 1937
    },
    {
      "epoch": 1.2055987558320373,
      "grad_norm": 0.4572494924068451,
      "learning_rate": 1.55e-06,
      "log_odds_chosen": 8.461591720581055,
      "log_odds_ratio": -0.10262424498796463,
      "logits/chosen": 0.20304995775222778,
      "logits/rejected": 0.4690617322921753,
      "logps/chosen": -0.6836486458778381,
      "logps/rejected": -8.393146514892578,
      "loss": 3.1867,
      "nll_loss": 3.176400661468506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06836486607789993,
      "rewards/margins": 0.7709497809410095,
      "rewards/rejected": -0.8393145799636841,
      "step": 1938
    },
    {
      "epoch": 1.206220839813375,
      "grad_norm": 0.641258180141449,
      "learning_rate": 1.525e-06,
      "log_odds_chosen": 13.561309814453125,
      "log_odds_ratio": -0.053132813423871994,
      "logits/chosen": 0.281563937664032,
      "logits/rejected": 1.5124027729034424,
      "logps/chosen": -0.9380850195884705,
      "logps/rejected": -14.03787612915039,
      "loss": 2.3357,
      "nll_loss": 2.330416202545166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09380850195884705,
      "rewards/margins": 1.3099792003631592,
      "rewards/rejected": -1.403787612915039,
      "step": 1939
    },
    {
      "epoch": 1.2068429237947123,
      "grad_norm": 0.6043529510498047,
      "learning_rate": 1.5e-06,
      "log_odds_chosen": 11.235591888427734,
      "log_odds_ratio": -0.08132461458444595,
      "logits/chosen": 0.11666382849216461,
      "logits/rejected": 1.035499095916748,
      "logps/chosen": -1.076815128326416,
      "logps/rejected": -11.899983406066895,
      "loss": 2.4499,
      "nll_loss": 2.441749334335327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1076815202832222,
      "rewards/margins": 1.0823167562484741,
      "rewards/rejected": -1.1899982690811157,
      "step": 1940
    },
    {
      "epoch": 1.2074650077760498,
      "grad_norm": 0.5861474871635437,
      "learning_rate": 1.475e-06,
      "log_odds_chosen": 12.166252136230469,
      "log_odds_ratio": -0.058929406106472015,
      "logits/chosen": 0.20770826935768127,
      "logits/rejected": 0.8990826606750488,
      "logps/chosen": -1.0073878765106201,
      "logps/rejected": -12.550826072692871,
      "loss": 2.6517,
      "nll_loss": 2.645765781402588,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10073879361152649,
      "rewards/margins": 1.154343843460083,
      "rewards/rejected": -1.255082607269287,
      "step": 1941
    },
    {
      "epoch": 1.2080870917573872,
      "grad_norm": 0.4842999577522278,
      "learning_rate": 1.45e-06,
      "log_odds_chosen": 12.450490951538086,
      "log_odds_ratio": -0.043036721646785736,
      "logits/chosen": 0.3495686650276184,
      "logits/rejected": 1.1103694438934326,
      "logps/chosen": -0.8654146194458008,
      "logps/rejected": -12.739120483398438,
      "loss": 3.5126,
      "nll_loss": 3.5082592964172363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08654146641492844,
      "rewards/margins": 1.1873705387115479,
      "rewards/rejected": -1.2739120721817017,
      "step": 1942
    },
    {
      "epoch": 1.2087091757387247,
      "grad_norm": 0.5130143165588379,
      "learning_rate": 1.4250000000000001e-06,
      "log_odds_chosen": 11.402581214904785,
      "log_odds_ratio": -0.1462099850177765,
      "logits/chosen": 0.26440829038619995,
      "logits/rejected": 1.2154403924942017,
      "logps/chosen": -1.113938331604004,
      "logps/rejected": -12.0927095413208,
      "loss": 3.0039,
      "nll_loss": 2.989281177520752,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11139383167028427,
      "rewards/margins": 1.0978771448135376,
      "rewards/rejected": -1.20927095413208,
      "step": 1943
    },
    {
      "epoch": 1.2093312597200623,
      "grad_norm": 0.5196940302848816,
      "learning_rate": 1.4000000000000001e-06,
      "log_odds_chosen": 13.953886985778809,
      "log_odds_ratio": -0.07984312623739243,
      "logits/chosen": 0.2587972581386566,
      "logits/rejected": 1.134250283241272,
      "logps/chosen": -1.1607818603515625,
      "logps/rejected": -14.70931625366211,
      "loss": 2.9967,
      "nll_loss": 2.988737106323242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1160781979560852,
      "rewards/margins": 1.3548533916473389,
      "rewards/rejected": -1.4709315299987793,
      "step": 1944
    },
    {
      "epoch": 1.2099533437013996,
      "grad_norm": 0.43494367599487305,
      "learning_rate": 1.3750000000000002e-06,
      "log_odds_chosen": 11.031208038330078,
      "log_odds_ratio": -0.24586418271064758,
      "logits/chosen": 0.36235183477401733,
      "logits/rejected": 1.051892638206482,
      "logps/chosen": -1.0590839385986328,
      "logps/rejected": -11.676322937011719,
      "loss": 3.7231,
      "nll_loss": 3.6984784603118896,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10590839385986328,
      "rewards/margins": 1.0617239475250244,
      "rewards/rejected": -1.1676323413848877,
      "step": 1945
    },
    {
      "epoch": 1.2105754276827372,
      "grad_norm": 0.6046903133392334,
      "learning_rate": 1.35e-06,
      "log_odds_chosen": 11.70355224609375,
      "log_odds_ratio": -0.027556756511330605,
      "logits/chosen": 0.15471243858337402,
      "logits/rejected": 0.9451410174369812,
      "logps/chosen": -1.006124496459961,
      "logps/rejected": -12.17332649230957,
      "loss": 2.3758,
      "nll_loss": 2.3730697631835938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10061244666576385,
      "rewards/margins": 1.116720199584961,
      "rewards/rejected": -1.2173326015472412,
      "step": 1946
    },
    {
      "epoch": 1.2111975116640747,
      "grad_norm": 0.7742229104042053,
      "learning_rate": 1.325e-06,
      "log_odds_chosen": 13.105299949645996,
      "log_odds_ratio": -0.09716056287288666,
      "logits/chosen": 0.22029726207256317,
      "logits/rejected": 0.7522600889205933,
      "logps/chosen": -0.7733492255210876,
      "logps/rejected": -13.077220916748047,
      "loss": 2.6818,
      "nll_loss": 2.67205810546875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.077334925532341,
      "rewards/margins": 1.2303872108459473,
      "rewards/rejected": -1.3077220916748047,
      "step": 1947
    },
    {
      "epoch": 1.211819595645412,
      "grad_norm": 0.5355544090270996,
      "learning_rate": 1.3e-06,
      "log_odds_chosen": 9.141685485839844,
      "log_odds_ratio": -0.0110908392816782,
      "logits/chosen": 0.20812612771987915,
      "logits/rejected": 0.826041042804718,
      "logps/chosen": -1.2029227018356323,
      "logps/rejected": -9.956043243408203,
      "loss": 3.0694,
      "nll_loss": 3.0682947635650635,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12029227614402771,
      "rewards/margins": 0.8753120303153992,
      "rewards/rejected": -0.9956042766571045,
      "step": 1948
    },
    {
      "epoch": 1.2124416796267496,
      "grad_norm": 0.5379774570465088,
      "learning_rate": 1.275e-06,
      "log_odds_chosen": 10.0809965133667,
      "log_odds_ratio": -0.22488044202327728,
      "logits/chosen": 0.3199789524078369,
      "logits/rejected": 1.2801084518432617,
      "logps/chosen": -1.104744791984558,
      "logps/rejected": -10.602627754211426,
      "loss": 3.0859,
      "nll_loss": 3.06339168548584,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11047448217868805,
      "rewards/margins": 0.9497882723808289,
      "rewards/rejected": -1.0602627992630005,
      "step": 1949
    },
    {
      "epoch": 1.213063763608087,
      "grad_norm": 0.47320687770843506,
      "learning_rate": 1.25e-06,
      "log_odds_chosen": 10.314563751220703,
      "log_odds_ratio": -0.11364743113517761,
      "logits/chosen": 0.17472714185714722,
      "logits/rejected": 0.5548080801963806,
      "logps/chosen": -1.1122039556503296,
      "logps/rejected": -10.97630500793457,
      "loss": 2.9175,
      "nll_loss": 2.906121253967285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11122038960456848,
      "rewards/margins": 0.9864101409912109,
      "rewards/rejected": -1.097630500793457,
      "step": 1950
    },
    {
      "epoch": 1.2136858475894245,
      "grad_norm": 0.6643481850624084,
      "learning_rate": 1.2250000000000001e-06,
      "log_odds_chosen": 7.230688571929932,
      "log_odds_ratio": -0.27975186705589294,
      "logits/chosen": 0.20719948410987854,
      "logits/rejected": 0.5046452283859253,
      "logps/chosen": -0.9272645711898804,
      "logps/rejected": -7.5637593269348145,
      "loss": 3.4042,
      "nll_loss": 3.3762478828430176,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0927264541387558,
      "rewards/margins": 0.6636494994163513,
      "rewards/rejected": -0.7563759088516235,
      "step": 1951
    },
    {
      "epoch": 1.2143079315707621,
      "grad_norm": 0.47893887758255005,
      "learning_rate": 1.2000000000000002e-06,
      "log_odds_chosen": 10.240165710449219,
      "log_odds_ratio": -0.08771882206201553,
      "logits/chosen": 0.30709195137023926,
      "logits/rejected": 0.836345911026001,
      "logps/chosen": -0.8510648608207703,
      "logps/rejected": -10.372858047485352,
      "loss": 3.2679,
      "nll_loss": 3.259153366088867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0851064920425415,
      "rewards/margins": 0.9521793127059937,
      "rewards/rejected": -1.0372858047485352,
      "step": 1952
    },
    {
      "epoch": 1.2149300155520995,
      "grad_norm": 0.4384036362171173,
      "learning_rate": 1.175e-06,
      "log_odds_chosen": 6.116580009460449,
      "log_odds_ratio": -0.0858708992600441,
      "logits/chosen": 0.20664840936660767,
      "logits/rejected": 0.5166964530944824,
      "logps/chosen": -0.8784169554710388,
      "logps/rejected": -6.095085144042969,
      "loss": 3.3228,
      "nll_loss": 3.314185857772827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08784169703722,
      "rewards/margins": 0.5216668844223022,
      "rewards/rejected": -0.6095085740089417,
      "step": 1953
    },
    {
      "epoch": 1.215552099533437,
      "grad_norm": 0.47888606786727905,
      "learning_rate": 1.15e-06,
      "log_odds_chosen": 14.013835906982422,
      "log_odds_ratio": -0.0009424424497410655,
      "logits/chosen": 0.30866438150405884,
      "logits/rejected": 1.1062043905258179,
      "logps/chosen": -0.9635977745056152,
      "logps/rejected": -14.372419357299805,
      "loss": 3.1659,
      "nll_loss": 3.1658222675323486,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09635977447032928,
      "rewards/margins": 1.3408820629119873,
      "rewards/rejected": -1.437242031097412,
      "step": 1954
    },
    {
      "epoch": 1.2161741835147746,
      "grad_norm": 0.6387030482292175,
      "learning_rate": 1.125e-06,
      "log_odds_chosen": 11.080677032470703,
      "log_odds_ratio": -0.12639693915843964,
      "logits/chosen": 0.3598368763923645,
      "logits/rejected": 1.24238121509552,
      "logps/chosen": -1.0083693265914917,
      "logps/rejected": -11.685609817504883,
      "loss": 2.8974,
      "nll_loss": 2.8847668170928955,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10083693265914917,
      "rewards/margins": 1.0677239894866943,
      "rewards/rejected": -1.1685608625411987,
      "step": 1955
    },
    {
      "epoch": 1.216796267496112,
      "grad_norm": 0.46826833486557007,
      "learning_rate": 1.1e-06,
      "log_odds_chosen": 10.808345794677734,
      "log_odds_ratio": -0.00998397171497345,
      "logits/chosen": 0.2149408757686615,
      "logits/rejected": 0.9323172569274902,
      "logps/chosen": -1.1690503358840942,
      "logps/rejected": -11.58592414855957,
      "loss": 3.2139,
      "nll_loss": 3.212857723236084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11690502613782883,
      "rewards/margins": 1.0416873693466187,
      "rewards/rejected": -1.1585923433303833,
      "step": 1956
    },
    {
      "epoch": 1.2174183514774495,
      "grad_norm": 0.5350275635719299,
      "learning_rate": 1.0749999999999999e-06,
      "log_odds_chosen": 7.807048797607422,
      "log_odds_ratio": -0.2332705557346344,
      "logits/chosen": 0.05367419868707657,
      "logits/rejected": 0.641290545463562,
      "logps/chosen": -1.2197027206420898,
      "logps/rejected": -8.814544677734375,
      "loss": 2.7277,
      "nll_loss": 2.7044012546539307,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1219702661037445,
      "rewards/margins": 0.7594842910766602,
      "rewards/rejected": -0.8814545273780823,
      "step": 1957
    },
    {
      "epoch": 1.218040435458787,
      "grad_norm": 0.4396865665912628,
      "learning_rate": 1.0500000000000001e-06,
      "log_odds_chosen": 7.691943645477295,
      "log_odds_ratio": -0.11706163734197617,
      "logits/chosen": 0.12572860717773438,
      "logits/rejected": 0.6232350468635559,
      "logps/chosen": -1.0413182973861694,
      "logps/rejected": -8.103107452392578,
      "loss": 3.0067,
      "nll_loss": 2.994955539703369,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10413183271884918,
      "rewards/margins": 0.7061789035797119,
      "rewards/rejected": -0.8103107213973999,
      "step": 1958
    },
    {
      "epoch": 1.2186625194401244,
      "grad_norm": 0.6320565342903137,
      "learning_rate": 1.0250000000000001e-06,
      "log_odds_chosen": 10.37856388092041,
      "log_odds_ratio": -0.1980902999639511,
      "logits/chosen": 0.23183494806289673,
      "logits/rejected": 0.9997614622116089,
      "logps/chosen": -1.0834324359893799,
      "logps/rejected": -11.047297477722168,
      "loss": 2.3947,
      "nll_loss": 2.3749301433563232,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10834324359893799,
      "rewards/margins": 0.9963865280151367,
      "rewards/rejected": -1.1047298908233643,
      "step": 1959
    },
    {
      "epoch": 1.219284603421462,
      "grad_norm": 0.6440889239311218,
      "learning_rate": 1.0000000000000002e-06,
      "log_odds_chosen": 9.363749504089355,
      "log_odds_ratio": -0.009031450375914574,
      "logits/chosen": 0.2922241687774658,
      "logits/rejected": 1.0417684316635132,
      "logps/chosen": -1.280368447303772,
      "logps/rejected": -10.292003631591797,
      "loss": 2.8072,
      "nll_loss": 2.806274652481079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12803685665130615,
      "rewards/margins": 0.9011635184288025,
      "rewards/rejected": -1.0292003154754639,
      "step": 1960
    },
    {
      "epoch": 1.2199066874027993,
      "grad_norm": 0.5854726433753967,
      "learning_rate": 9.75e-07,
      "log_odds_chosen": 7.83726692199707,
      "log_odds_ratio": -0.14151814579963684,
      "logits/chosen": 0.02728712186217308,
      "logits/rejected": 0.7834301590919495,
      "logps/chosen": -1.2481005191802979,
      "logps/rejected": -8.801592826843262,
      "loss": 2.4193,
      "nll_loss": 2.405181646347046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12481004744768143,
      "rewards/margins": 0.7553492188453674,
      "rewards/rejected": -0.880159318447113,
      "step": 1961
    },
    {
      "epoch": 1.2205287713841368,
      "grad_norm": 0.6133548021316528,
      "learning_rate": 9.5e-07,
      "log_odds_chosen": 17.47218894958496,
      "log_odds_ratio": -0.0755654126405716,
      "logits/chosen": 0.4042644202709198,
      "logits/rejected": 1.29765784740448,
      "logps/chosen": -0.8723544478416443,
      "logps/rejected": -17.82417869567871,
      "loss": 2.9475,
      "nll_loss": 2.939979076385498,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08723544329404831,
      "rewards/margins": 1.6951824426651,
      "rewards/rejected": -1.782417893409729,
      "step": 1962
    },
    {
      "epoch": 1.2211508553654744,
      "grad_norm": 0.6662535667419434,
      "learning_rate": 9.25e-07,
      "log_odds_chosen": 14.000879287719727,
      "log_odds_ratio": -0.0005081897252239287,
      "logits/chosen": 0.26021426916122437,
      "logits/rejected": 1.3705577850341797,
      "logps/chosen": -1.1540570259094238,
      "logps/rejected": -14.73798942565918,
      "loss": 2.3304,
      "nll_loss": 2.3303232192993164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11540570110082626,
      "rewards/margins": 1.3583931922912598,
      "rewards/rejected": -1.4737989902496338,
      "step": 1963
    },
    {
      "epoch": 1.2217729393468117,
      "grad_norm": 0.5186629295349121,
      "learning_rate": 9e-07,
      "log_odds_chosen": 14.97154426574707,
      "log_odds_ratio": -0.0003290826571173966,
      "logits/chosen": 0.1332269012928009,
      "logits/rejected": 0.7068179845809937,
      "logps/chosen": -1.1427979469299316,
      "logps/rejected": -15.563932418823242,
      "loss": 3.0262,
      "nll_loss": 3.0261826515197754,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11427979171276093,
      "rewards/margins": 1.4421135187149048,
      "rewards/rejected": -1.5563932657241821,
      "step": 1964
    },
    {
      "epoch": 1.2223950233281493,
      "grad_norm": 0.644451916217804,
      "learning_rate": 8.750000000000001e-07,
      "log_odds_chosen": 8.28154182434082,
      "log_odds_ratio": -0.18015088140964508,
      "logits/chosen": 0.14118118584156036,
      "logits/rejected": 0.36667898297309875,
      "logps/chosen": -1.0873171091079712,
      "logps/rejected": -8.973154067993164,
      "loss": 2.7051,
      "nll_loss": 2.6870474815368652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1087317168712616,
      "rewards/margins": 0.7885836958885193,
      "rewards/rejected": -0.8973153829574585,
      "step": 1965
    },
    {
      "epoch": 1.2230171073094869,
      "grad_norm": 0.6322017312049866,
      "learning_rate": 8.500000000000001e-07,
      "log_odds_chosen": 8.490213394165039,
      "log_odds_ratio": -0.034700050950050354,
      "logits/chosen": 0.14440450072288513,
      "logits/rejected": 0.6193152666091919,
      "logps/chosen": -1.045159101486206,
      "logps/rejected": -9.040973663330078,
      "loss": 2.6674,
      "nll_loss": 2.66395902633667,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1045159101486206,
      "rewards/margins": 0.7995815277099609,
      "rewards/rejected": -0.9040974378585815,
      "step": 1966
    },
    {
      "epoch": 1.2236391912908242,
      "grad_norm": 0.6075518727302551,
      "learning_rate": 8.25e-07,
      "log_odds_chosen": 14.548714637756348,
      "log_odds_ratio": -0.07951350510120392,
      "logits/chosen": 0.22518101334571838,
      "logits/rejected": 1.286879062652588,
      "logps/chosen": -1.033128261566162,
      "logps/rejected": -15.159509658813477,
      "loss": 2.7825,
      "nll_loss": 2.7745676040649414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10331282019615173,
      "rewards/margins": 1.4126381874084473,
      "rewards/rejected": -1.5159509181976318,
      "step": 1967
    },
    {
      "epoch": 1.2242612752721618,
      "grad_norm": 0.45069652795791626,
      "learning_rate": 8.000000000000001e-07,
      "log_odds_chosen": 15.424966812133789,
      "log_odds_ratio": -2.145773578376975e-06,
      "logits/chosen": 0.35158199071884155,
      "logits/rejected": 1.5012739896774292,
      "logps/chosen": -1.0200140476226807,
      "logps/rejected": -15.91152572631836,
      "loss": 3.0081,
      "nll_loss": 3.008110284805298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10200141370296478,
      "rewards/margins": 1.4891510009765625,
      "rewards/rejected": -1.591152548789978,
      "step": 1968
    },
    {
      "epoch": 1.2248833592534991,
      "grad_norm": 0.47687819600105286,
      "learning_rate": 7.75e-07,
      "log_odds_chosen": 7.983428001403809,
      "log_odds_ratio": -0.028099507093429565,
      "logits/chosen": 0.18631598353385925,
      "logits/rejected": 0.5772290825843811,
      "logps/chosen": -0.7105157971382141,
      "logps/rejected": -7.801077365875244,
      "loss": 3.1483,
      "nll_loss": 3.145503520965576,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07105158269405365,
      "rewards/margins": 0.7090562582015991,
      "rewards/rejected": -0.7801077961921692,
      "step": 1969
    },
    {
      "epoch": 1.2255054432348367,
      "grad_norm": 0.8003374338150024,
      "learning_rate": 7.5e-07,
      "log_odds_chosen": 12.410947799682617,
      "log_odds_ratio": -0.00033352465834468603,
      "logits/chosen": 0.07465855777263641,
      "logits/rejected": 0.8042261004447937,
      "logps/chosen": -1.0633723735809326,
      "logps/rejected": -12.912806510925293,
      "loss": 2.5147,
      "nll_loss": 2.5146756172180176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10633723437786102,
      "rewards/margins": 1.1849435567855835,
      "rewards/rejected": -1.2912808656692505,
      "step": 1970
    },
    {
      "epoch": 1.2261275272161742,
      "grad_norm": 2.2342381477355957,
      "learning_rate": 7.25e-07,
      "log_odds_chosen": 10.86426067352295,
      "log_odds_ratio": -0.13659894466400146,
      "logits/chosen": 0.18673838675022125,
      "logits/rejected": 0.6166978478431702,
      "logps/chosen": -1.0592784881591797,
      "logps/rejected": -11.352396011352539,
      "loss": 3.137,
      "nll_loss": 3.1233880519866943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10592785477638245,
      "rewards/margins": 1.0293117761611938,
      "rewards/rejected": -1.135239601135254,
      "step": 1971
    },
    {
      "epoch": 1.2267496111975116,
      "grad_norm": 0.6535879373550415,
      "learning_rate": 7.000000000000001e-07,
      "log_odds_chosen": 9.130402565002441,
      "log_odds_ratio": -0.1539786159992218,
      "logits/chosen": 0.32859697937965393,
      "logits/rejected": 0.9583666920661926,
      "logps/chosen": -0.966360330581665,
      "logps/rejected": -9.597227096557617,
      "loss": 2.797,
      "nll_loss": 2.7815654277801514,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09663603454828262,
      "rewards/margins": 0.8630867004394531,
      "rewards/rejected": -0.9597226977348328,
      "step": 1972
    },
    {
      "epoch": 1.2273716951788491,
      "grad_norm": 0.5714898109436035,
      "learning_rate": 6.75e-07,
      "log_odds_chosen": 13.186323165893555,
      "log_odds_ratio": -0.04164155200123787,
      "logits/chosen": 0.3834404945373535,
      "logits/rejected": 1.092491626739502,
      "logps/chosen": -1.2385993003845215,
      "logps/rejected": -13.750805854797363,
      "loss": 3.6435,
      "nll_loss": 3.6393702030181885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12385992705821991,
      "rewards/margins": 1.251220703125,
      "rewards/rejected": -1.3750805854797363,
      "step": 1973
    },
    {
      "epoch": 1.2279937791601867,
      "grad_norm": 0.48778393864631653,
      "learning_rate": 6.5e-07,
      "log_odds_chosen": 12.12580394744873,
      "log_odds_ratio": -0.11388830840587616,
      "logits/chosen": 0.17847785353660583,
      "logits/rejected": 0.7780681252479553,
      "logps/chosen": -0.8805215358734131,
      "logps/rejected": -12.4000244140625,
      "loss": 3.0103,
      "nll_loss": 2.9988701343536377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08805215358734131,
      "rewards/margins": 1.1519503593444824,
      "rewards/rejected": -1.2400025129318237,
      "step": 1974
    },
    {
      "epoch": 1.228615863141524,
      "grad_norm": 0.649593710899353,
      "learning_rate": 6.25e-07,
      "log_odds_chosen": 6.819096565246582,
      "log_odds_ratio": -0.25234541296958923,
      "logits/chosen": 0.22128456830978394,
      "logits/rejected": 0.7910210490226746,
      "logps/chosen": -1.0413517951965332,
      "logps/rejected": -7.474663257598877,
      "loss": 2.6732,
      "nll_loss": 2.647918939590454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1041351780295372,
      "rewards/margins": 0.6433311700820923,
      "rewards/rejected": -0.7474663853645325,
      "step": 1975
    },
    {
      "epoch": 1.2292379471228616,
      "grad_norm": 0.5235407948493958,
      "learning_rate": 6.000000000000001e-07,
      "log_odds_chosen": 11.304561614990234,
      "log_odds_ratio": -0.0011457885848358274,
      "logits/chosen": 0.27476418018341064,
      "logits/rejected": 1.1056790351867676,
      "logps/chosen": -0.9318090677261353,
      "logps/rejected": -11.397740364074707,
      "loss": 2.9588,
      "nll_loss": 2.9587111473083496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09318091720342636,
      "rewards/margins": 1.046593189239502,
      "rewards/rejected": -1.1397740840911865,
      "step": 1976
    },
    {
      "epoch": 1.2298600311041992,
      "grad_norm": 0.6689901947975159,
      "learning_rate": 5.75e-07,
      "log_odds_chosen": 6.795727729797363,
      "log_odds_ratio": -0.30927932262420654,
      "logits/chosen": 0.027122098952531815,
      "logits/rejected": 0.27755478024482727,
      "logps/chosen": -1.076484203338623,
      "logps/rejected": -7.535107612609863,
      "loss": 2.4418,
      "nll_loss": 2.410856246948242,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10764843225479126,
      "rewards/margins": 0.645862340927124,
      "rewards/rejected": -0.7535107731819153,
      "step": 1977
    },
    {
      "epoch": 1.2304821150855365,
      "grad_norm": 0.6304493546485901,
      "learning_rate": 5.5e-07,
      "log_odds_chosen": 9.401277542114258,
      "log_odds_ratio": -0.18153725564479828,
      "logits/chosen": 0.11142933368682861,
      "logits/rejected": 0.42696264386177063,
      "logps/chosen": -0.8474681377410889,
      "logps/rejected": -9.611903190612793,
      "loss": 2.4952,
      "nll_loss": 2.4770264625549316,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0847468227148056,
      "rewards/margins": 0.8764435648918152,
      "rewards/rejected": -0.961190402507782,
      "step": 1978
    },
    {
      "epoch": 1.231104199066874,
      "grad_norm": 0.534700870513916,
      "learning_rate": 5.250000000000001e-07,
      "log_odds_chosen": 12.72781753540039,
      "log_odds_ratio": -0.11465675383806229,
      "logits/chosen": 0.3088584244251251,
      "logits/rejected": 1.142822504043579,
      "logps/chosen": -0.8340325355529785,
      "logps/rejected": -12.834344863891602,
      "loss": 2.9313,
      "nll_loss": 2.9198219776153564,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08340325951576233,
      "rewards/margins": 1.2000312805175781,
      "rewards/rejected": -1.2834346294403076,
      "step": 1979
    },
    {
      "epoch": 1.2317262830482114,
      "grad_norm": 0.6244094967842102,
      "learning_rate": 5.000000000000001e-07,
      "log_odds_chosen": 13.20551586151123,
      "log_odds_ratio": -0.03865737095475197,
      "logits/chosen": 0.2893320918083191,
      "logits/rejected": 1.3345921039581299,
      "logps/chosen": -1.0497782230377197,
      "logps/rejected": -13.795721054077148,
      "loss": 2.7889,
      "nll_loss": 2.7850396633148193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10497783124446869,
      "rewards/margins": 1.2745941877365112,
      "rewards/rejected": -1.3795721530914307,
      "step": 1980
    },
    {
      "epoch": 1.232348367029549,
      "grad_norm": 0.532413899898529,
      "learning_rate": 4.75e-07,
      "log_odds_chosen": 10.777617454528809,
      "log_odds_ratio": -0.010713006369769573,
      "logits/chosen": 0.39420104026794434,
      "logits/rejected": 1.2114235162734985,
      "logps/chosen": -0.8548382520675659,
      "logps/rejected": -10.988439559936523,
      "loss": 3.2467,
      "nll_loss": 3.2456629276275635,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08548382669687271,
      "rewards/margins": 1.0133602619171143,
      "rewards/rejected": -1.098844051361084,
      "step": 1981
    },
    {
      "epoch": 1.2329704510108865,
      "grad_norm": 0.5516924262046814,
      "learning_rate": 4.5e-07,
      "log_odds_chosen": 12.285039901733398,
      "log_odds_ratio": -0.08736379444599152,
      "logits/chosen": 0.26371684670448303,
      "logits/rejected": 0.9414895176887512,
      "logps/chosen": -0.8180931806564331,
      "logps/rejected": -12.407268524169922,
      "loss": 2.9298,
      "nll_loss": 2.921090602874756,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08180931955575943,
      "rewards/margins": 1.1589176654815674,
      "rewards/rejected": -1.2407269477844238,
      "step": 1982
    },
    {
      "epoch": 1.2335925349922239,
      "grad_norm": 0.493353009223938,
      "learning_rate": 4.2500000000000006e-07,
      "log_odds_chosen": 13.17050552368164,
      "log_odds_ratio": -0.005692049395292997,
      "logits/chosen": 0.35018348693847656,
      "logits/rejected": 1.103072166442871,
      "logps/chosen": -1.3006165027618408,
      "logps/rejected": -13.748285293579102,
      "loss": 3.395,
      "nll_loss": 3.394420623779297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13006165623664856,
      "rewards/margins": 1.2447669506072998,
      "rewards/rejected": -1.374828577041626,
      "step": 1983
    },
    {
      "epoch": 1.2342146189735614,
      "grad_norm": 0.4653708040714264,
      "learning_rate": 4.0000000000000003e-07,
      "log_odds_chosen": 8.631146430969238,
      "log_odds_ratio": -0.19020092487335205,
      "logits/chosen": 0.43207770586013794,
      "logits/rejected": 1.0626914501190186,
      "logps/chosen": -1.5182112455368042,
      "logps/rejected": -9.975319862365723,
      "loss": 3.8365,
      "nll_loss": 3.8174636363983154,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15182113647460938,
      "rewards/margins": 0.845710813999176,
      "rewards/rejected": -0.9975318908691406,
      "step": 1984
    },
    {
      "epoch": 1.234836702954899,
      "grad_norm": 0.4419538378715515,
      "learning_rate": 3.75e-07,
      "log_odds_chosen": 7.5447492599487305,
      "log_odds_ratio": -0.01544712483882904,
      "logits/chosen": 0.24856893718242645,
      "logits/rejected": 0.7290024757385254,
      "logps/chosen": -1.1332921981811523,
      "logps/rejected": -8.262965202331543,
      "loss": 3.1574,
      "nll_loss": 3.1558544635772705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.113329216837883,
      "rewards/margins": 0.7129673361778259,
      "rewards/rejected": -0.8262965083122253,
      "step": 1985
    },
    {
      "epoch": 1.2354587869362363,
      "grad_norm": 0.4535614550113678,
      "learning_rate": 3.5000000000000004e-07,
      "log_odds_chosen": 12.693193435668945,
      "log_odds_ratio": -0.017142467200756073,
      "logits/chosen": 0.2979247272014618,
      "logits/rejected": 1.086916208267212,
      "logps/chosen": -0.9978163838386536,
      "logps/rejected": -13.19540023803711,
      "loss": 2.8112,
      "nll_loss": 2.809511661529541,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09978163242340088,
      "rewards/margins": 1.21975839138031,
      "rewards/rejected": -1.319540023803711,
      "step": 1986
    },
    {
      "epoch": 1.236080870917574,
      "grad_norm": 0.9630240797996521,
      "learning_rate": 3.25e-07,
      "log_odds_chosen": 4.148228645324707,
      "log_odds_ratio": -0.3517554998397827,
      "logits/chosen": 0.10793019831180573,
      "logits/rejected": 0.37933433055877686,
      "logps/chosen": -1.1543450355529785,
      "logps/rejected": -5.036352157592773,
      "loss": 2.5266,
      "nll_loss": 2.4914731979370117,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11543451249599457,
      "rewards/margins": 0.3882007300853729,
      "rewards/rejected": -0.5036352276802063,
      "step": 1987
    },
    {
      "epoch": 1.2367029548989112,
      "grad_norm": 0.5966618657112122,
      "learning_rate": 3.0000000000000004e-07,
      "log_odds_chosen": 12.570096015930176,
      "log_odds_ratio": -0.004863132257014513,
      "logits/chosen": 0.24723371863365173,
      "logits/rejected": 1.2845591306686401,
      "logps/chosen": -1.2328592538833618,
      "logps/rejected": -13.443431854248047,
      "loss": 2.7224,
      "nll_loss": 2.7219624519348145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1232859343290329,
      "rewards/margins": 1.2210571765899658,
      "rewards/rejected": -1.3443431854248047,
      "step": 1988
    },
    {
      "epoch": 1.2373250388802488,
      "grad_norm": 0.6358088850975037,
      "learning_rate": 2.75e-07,
      "log_odds_chosen": 6.43398380279541,
      "log_odds_ratio": -0.17213129997253418,
      "logits/chosen": 0.13488197326660156,
      "logits/rejected": 0.573233962059021,
      "logps/chosen": -0.9811330437660217,
      "logps/rejected": -6.960114002227783,
      "loss": 2.6887,
      "nll_loss": 2.6715219020843506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0981132984161377,
      "rewards/margins": 0.5978981256484985,
      "rewards/rejected": -0.6960113644599915,
      "step": 1989
    },
    {
      "epoch": 1.2379471228615864,
      "grad_norm": 0.6376218199729919,
      "learning_rate": 2.5000000000000004e-07,
      "log_odds_chosen": 8.221298217773438,
      "log_odds_ratio": -0.13431409001350403,
      "logits/chosen": 0.0446498841047287,
      "logits/rejected": 0.5792117118835449,
      "logps/chosen": -1.0903517007827759,
      "logps/rejected": -8.90846061706543,
      "loss": 2.1503,
      "nll_loss": 2.136909008026123,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10903516411781311,
      "rewards/margins": 0.7818108797073364,
      "rewards/rejected": -0.8908460736274719,
      "step": 1990
    },
    {
      "epoch": 1.2385692068429237,
      "grad_norm": 0.46342816948890686,
      "learning_rate": 2.25e-07,
      "log_odds_chosen": 9.396651268005371,
      "log_odds_ratio": -0.19147804379463196,
      "logits/chosen": 0.2942267060279846,
      "logits/rejected": 0.8135102391242981,
      "logps/chosen": -0.9783865213394165,
      "logps/rejected": -9.955822944641113,
      "loss": 3.0935,
      "nll_loss": 3.0743513107299805,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09783865511417389,
      "rewards/margins": 0.8977437019348145,
      "rewards/rejected": -0.9955822825431824,
      "step": 1991
    },
    {
      "epoch": 1.2391912908242613,
      "grad_norm": 0.6562243700027466,
      "learning_rate": 2.0000000000000002e-07,
      "log_odds_chosen": 7.678519248962402,
      "log_odds_ratio": -0.046273984014987946,
      "logits/chosen": 0.2911015748977661,
      "logits/rejected": 0.81610107421875,
      "logps/chosen": -1.0920604467391968,
      "logps/rejected": -8.079513549804688,
      "loss": 3.0501,
      "nll_loss": 3.0454328060150146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10920606553554535,
      "rewards/margins": 0.6987452507019043,
      "rewards/rejected": -0.8079513311386108,
      "step": 1992
    },
    {
      "epoch": 1.2398133748055988,
      "grad_norm": 0.6405279636383057,
      "learning_rate": 1.7500000000000002e-07,
      "log_odds_chosen": 8.547779083251953,
      "log_odds_ratio": -0.14488686621189117,
      "logits/chosen": 0.279305100440979,
      "logits/rejected": 0.7682068347930908,
      "logps/chosen": -1.0256428718566895,
      "logps/rejected": -9.043883323669434,
      "loss": 3.1503,
      "nll_loss": 3.135767936706543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10256429016590118,
      "rewards/margins": 0.8018239736557007,
      "rewards/rejected": -0.9043882489204407,
      "step": 1993
    },
    {
      "epoch": 1.2404354587869362,
      "grad_norm": 0.6466782689094543,
      "learning_rate": 1.5000000000000002e-07,
      "log_odds_chosen": 7.694208145141602,
      "log_odds_ratio": -0.07817018032073975,
      "logits/chosen": 0.18080022931098938,
      "logits/rejected": 0.88742995262146,
      "logps/chosen": -1.003135085105896,
      "logps/rejected": -8.164618492126465,
      "loss": 2.8495,
      "nll_loss": 2.8416547775268555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10031351447105408,
      "rewards/margins": 0.716148316860199,
      "rewards/rejected": -0.8164618015289307,
      "step": 1994
    },
    {
      "epoch": 1.2410575427682737,
      "grad_norm": 0.5974647402763367,
      "learning_rate": 1.2500000000000002e-07,
      "log_odds_chosen": 12.203994750976562,
      "log_odds_ratio": -0.028767168521881104,
      "logits/chosen": 0.2811827063560486,
      "logits/rejected": 0.8901752233505249,
      "logps/chosen": -0.8630846738815308,
      "logps/rejected": -12.416587829589844,
      "loss": 3.0164,
      "nll_loss": 3.0134806632995605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08630846440792084,
      "rewards/margins": 1.1553503274917603,
      "rewards/rejected": -1.2416586875915527,
      "step": 1995
    },
    {
      "epoch": 1.2416796267496113,
      "grad_norm": 0.5271542072296143,
      "learning_rate": 1.0000000000000001e-07,
      "log_odds_chosen": 15.869498252868652,
      "log_odds_ratio": -0.0005326489335857332,
      "logits/chosen": 0.31395959854125977,
      "logits/rejected": 1.5588934421539307,
      "logps/chosen": -0.9302768707275391,
      "logps/rejected": -16.15066146850586,
      "loss": 2.7436,
      "nll_loss": 2.7435102462768555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09302769601345062,
      "rewards/margins": 1.522038459777832,
      "rewards/rejected": -1.6150660514831543,
      "step": 1996
    },
    {
      "epoch": 1.2423017107309486,
      "grad_norm": 2.5741851329803467,
      "learning_rate": 7.500000000000001e-08,
      "log_odds_chosen": 11.470212936401367,
      "log_odds_ratio": -0.017528457567095757,
      "logits/chosen": 0.3122308850288391,
      "logits/rejected": 0.7689934968948364,
      "logps/chosen": -1.033614993095398,
      "logps/rejected": -11.780576705932617,
      "loss": 3.5241,
      "nll_loss": 3.5223326683044434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10336150974035263,
      "rewards/margins": 1.0746960639953613,
      "rewards/rejected": -1.1780576705932617,
      "step": 1997
    },
    {
      "epoch": 1.2429237947122862,
      "grad_norm": 0.49963220953941345,
      "learning_rate": 5.0000000000000004e-08,
      "log_odds_chosen": 13.224645614624023,
      "log_odds_ratio": -3.3678723411867395e-05,
      "logits/chosen": -0.03131323307752609,
      "logits/rejected": 0.7888933420181274,
      "logps/chosen": -0.924960732460022,
      "logps/rejected": -13.508068084716797,
      "loss": 2.3564,
      "nll_loss": 2.3563878536224365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09249607473611832,
      "rewards/margins": 1.2583107948303223,
      "rewards/rejected": -1.350806713104248,
      "step": 1998
    },
    {
      "epoch": 1.2435458786936238,
      "grad_norm": 0.5515841245651245,
      "learning_rate": 2.5000000000000002e-08,
      "log_odds_chosen": 11.417040824890137,
      "log_odds_ratio": -0.09382711350917816,
      "logits/chosen": 0.32003253698349,
      "logits/rejected": 0.9499678611755371,
      "logps/chosen": -0.9657232761383057,
      "logps/rejected": -11.697189331054688,
      "loss": 3.106,
      "nll_loss": 3.0966427326202393,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09657233208417892,
      "rewards/margins": 1.0731468200683594,
      "rewards/rejected": -1.1697189807891846,
      "step": 1999
    },
    {
      "epoch": 1.244167962674961,
      "grad_norm": 0.48643431067466736,
      "learning_rate": 0.0,
      "log_odds_chosen": 6.229976654052734,
      "log_odds_ratio": -0.18844850361347198,
      "logits/chosen": 0.3436409831047058,
      "logits/rejected": 0.5300962924957275,
      "logps/chosen": -1.046073317527771,
      "logps/rejected": -6.884661674499512,
      "loss": 3.3064,
      "nll_loss": 3.2875545024871826,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10460732877254486,
      "rewards/margins": 0.583858847618103,
      "rewards/rejected": -0.6884661912918091,
      "step": 2000
    }
  ],
  "logging_steps": 1,
  "max_steps": 2000,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}