{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.982222222222222,
  "eval_steps": 1,
  "global_step": 336,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.011851851851851851,
      "grad_norm": 62.70548815519655,
      "learning_rate": 1.4705882352941176e-08,
      "logits/chosen": 0.030916133895516396,
      "logits/rejected": 0.09742362797260284,
      "logps/chosen": -40.58351516723633,
      "logps/rejected": -58.42578887939453,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.023703703703703703,
      "grad_norm": 67.3907670519946,
      "learning_rate": 2.941176470588235e-08,
      "logits/chosen": 0.15014928579330444,
      "logits/rejected": 0.2673640847206116,
      "logps/chosen": -31.35921859741211,
      "logps/rejected": -54.71299743652344,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.035555555555555556,
      "grad_norm": 73.59075381908265,
      "learning_rate": 4.411764705882353e-08,
      "logits/chosen": 0.35403603315353394,
      "logits/rejected": 0.3630790412425995,
      "logps/chosen": -30.862504959106445,
      "logps/rejected": -43.55963897705078,
      "loss": 0.6991,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.023167992010712624,
      "rewards/margins": 0.016683291643857956,
      "rewards/rejected": 0.00648469990119338,
      "step": 3
    },
    {
      "epoch": 0.047407407407407405,
      "grad_norm": 58.78689431688176,
      "learning_rate": 5.88235294117647e-08,
      "logits/chosen": 0.3042946457862854,
      "logits/rejected": 0.25474676489830017,
      "logps/chosen": -34.22315979003906,
      "logps/rejected": -39.93827438354492,
      "loss": 0.6941,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.021718814969062805,
      "rewards/margins": -0.024322079494595528,
      "rewards/rejected": 0.0026032691821455956,
      "step": 4
    },
    {
      "epoch": 0.05925925925925926,
      "grad_norm": 59.402728518681464,
      "learning_rate": 7.352941176470588e-08,
      "logits/chosen": 0.20607078075408936,
      "logits/rejected": 0.27008742094039917,
      "logps/chosen": -40.86919403076172,
      "logps/rejected": -51.17314910888672,
      "loss": 0.6937,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02542915567755699,
      "rewards/margins": 0.01238556019961834,
      "rewards/rejected": 0.013043595477938652,
      "step": 5
    },
    {
      "epoch": 0.07111111111111111,
      "grad_norm": 60.3208791733097,
      "learning_rate": 8.823529411764706e-08,
      "logits/chosen": 0.39524325728416443,
      "logits/rejected": 0.3147166669368744,
      "logps/chosen": -45.889522552490234,
      "logps/rejected": -47.271080017089844,
      "loss": 0.6967,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.035939525812864304,
      "rewards/margins": 0.035303808748722076,
      "rewards/rejected": 0.0006357184611260891,
      "step": 6
    },
    {
      "epoch": 0.08296296296296296,
      "grad_norm": 65.84343450368385,
      "learning_rate": 1.0294117647058822e-07,
      "logits/chosen": 0.19424982368946075,
      "logits/rejected": 0.36947980523109436,
      "logps/chosen": -32.91363525390625,
      "logps/rejected": -43.79743194580078,
      "loss": 0.7058,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012993477284908295,
      "rewards/margins": -0.015153911896049976,
      "rewards/rejected": 0.028147388249635696,
      "step": 7
    },
    {
      "epoch": 0.09481481481481481,
      "grad_norm": 68.52239761198476,
      "learning_rate": 1.176470588235294e-07,
      "logits/chosen": 0.1946137249469757,
      "logits/rejected": 0.28064286708831787,
      "logps/chosen": -32.246864318847656,
      "logps/rejected": -41.628746032714844,
      "loss": 0.6735,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010772847570478916,
      "rewards/margins": 0.011450938880443573,
      "rewards/rejected": -0.022223783656954765,
      "step": 8
    },
    {
      "epoch": 0.10666666666666667,
      "grad_norm": 77.93025419789252,
      "learning_rate": 1.3235294117647057e-07,
      "logits/chosen": 0.32008448243141174,
      "logits/rejected": 0.21636219322681427,
      "logps/chosen": -40.00132369995117,
      "logps/rejected": -44.613426208496094,
      "loss": 0.6976,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008494901470839977,
      "rewards/margins": 0.011597584001719952,
      "rewards/rejected": -0.003102683462202549,
      "step": 9
    },
    {
      "epoch": 0.11851851851851852,
      "grad_norm": 60.27023294648464,
      "learning_rate": 1.4705882352941175e-07,
      "logits/chosen": 0.011551467701792717,
      "logits/rejected": 0.1401338428258896,
      "logps/chosen": -35.68666076660156,
      "logps/rejected": -47.44255065917969,
      "loss": 0.6817,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.02254348061978817,
      "rewards/margins": 0.029230808839201927,
      "rewards/rejected": -0.0517742857336998,
      "step": 10
    },
    {
      "epoch": 0.13037037037037036,
      "grad_norm": 70.77005341416117,
      "learning_rate": 1.6176470588235293e-07,
      "logits/chosen": 0.07894501090049744,
      "logits/rejected": 0.09966235607862473,
      "logps/chosen": -30.685501098632812,
      "logps/rejected": -42.800785064697266,
      "loss": 0.6773,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.006754852831363678,
      "rewards/margins": 0.031156515702605247,
      "rewards/rejected": -0.037911366671323776,
      "step": 11
    },
    {
      "epoch": 0.14222222222222222,
      "grad_norm": 58.87977165275071,
      "learning_rate": 1.764705882352941e-07,
      "logits/chosen": 0.23514162003993988,
      "logits/rejected": 0.2450232207775116,
      "logps/chosen": -41.01308822631836,
      "logps/rejected": -52.138641357421875,
      "loss": 0.6775,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.05471659079194069,
      "rewards/margins": -0.00840845424681902,
      "rewards/rejected": -0.046308137476444244,
      "step": 12
    },
    {
      "epoch": 0.15407407407407409,
      "grad_norm": 68.28764496281121,
      "learning_rate": 1.9117647058823527e-07,
      "logits/chosen": 0.13845381140708923,
      "logits/rejected": 0.06714074313640594,
      "logps/chosen": -36.72666549682617,
      "logps/rejected": -44.98724365234375,
      "loss": 0.6698,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.027201365679502487,
      "rewards/margins": 0.0457880012691021,
      "rewards/rejected": -0.07298936694860458,
      "step": 13
    },
    {
      "epoch": 0.16592592592592592,
      "grad_norm": 67.0476123108747,
      "learning_rate": 2.0588235294117645e-07,
      "logits/chosen": 0.13570713996887207,
      "logits/rejected": 0.02110590785741806,
      "logps/chosen": -39.4144287109375,
      "logps/rejected": -46.626033782958984,
      "loss": 0.6694,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.05015880987048149,
      "rewards/margins": 0.06351868808269501,
      "rewards/rejected": -0.1136775016784668,
      "step": 14
    },
    {
      "epoch": 0.17777777777777778,
      "grad_norm": 56.97224502598152,
      "learning_rate": 2.2058823529411763e-07,
      "logits/chosen": 0.14530636370182037,
      "logits/rejected": 0.22717420756816864,
      "logps/chosen": -33.9251823425293,
      "logps/rejected": -47.67527770996094,
      "loss": 0.6504,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07779295742511749,
      "rewards/margins": 0.12380316108465195,
      "rewards/rejected": -0.20159611105918884,
      "step": 15
    },
    {
      "epoch": 0.18962962962962962,
      "grad_norm": 58.43872340752561,
      "learning_rate": 2.352941176470588e-07,
      "logits/chosen": 0.1471043974161148,
      "logits/rejected": 0.26890620589256287,
      "logps/chosen": -36.48509979248047,
      "logps/rejected": -53.876888275146484,
      "loss": 0.6503,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.12628403306007385,
      "rewards/margins": 0.1476879119873047,
      "rewards/rejected": -0.27397194504737854,
      "step": 16
    },
    {
      "epoch": 0.20148148148148148,
      "grad_norm": 53.27712198861993,
      "learning_rate": 2.5e-07,
      "logits/chosen": 0.3565051555633545,
      "logits/rejected": 0.25773561000823975,
      "logps/chosen": -32.79841232299805,
      "logps/rejected": -36.324119567871094,
      "loss": 0.6414,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06849764287471771,
      "rewards/margins": 0.11378694325685501,
      "rewards/rejected": -0.18228457868099213,
      "step": 17
    },
    {
      "epoch": 0.21333333333333335,
      "grad_norm": 58.90356354190709,
      "learning_rate": 2.6470588235294114e-07,
      "logits/chosen": -0.1638399213552475,
      "logits/rejected": -0.027556225657463074,
      "logps/chosen": -35.214149475097656,
      "logps/rejected": -54.174049377441406,
      "loss": 0.6328,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.15428400039672852,
      "rewards/margins": 0.18084901571273804,
      "rewards/rejected": -0.33513307571411133,
      "step": 18
    },
    {
      "epoch": 0.22518518518518518,
      "grad_norm": 54.674461580476404,
      "learning_rate": 2.7941176470588235e-07,
      "logits/chosen": 0.2881383001804352,
      "logits/rejected": 0.32903048396110535,
      "logps/chosen": -30.270973205566406,
      "logps/rejected": -40.31577682495117,
      "loss": 0.5957,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22714650630950928,
      "rewards/margins": 0.15492179989814758,
      "rewards/rejected": -0.38206830620765686,
      "step": 19
    },
    {
      "epoch": 0.23703703703703705,
      "grad_norm": 49.124258720104926,
      "learning_rate": 2.941176470588235e-07,
      "logits/chosen": 0.1832781583070755,
      "logits/rejected": 0.22061079740524292,
      "logps/chosen": -31.4530029296875,
      "logps/rejected": -43.574642181396484,
      "loss": 0.5737,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2013498842716217,
      "rewards/margins": 0.24712926149368286,
      "rewards/rejected": -0.44847914576530457,
      "step": 20
    },
    {
      "epoch": 0.24888888888888888,
      "grad_norm": 49.31720284300177,
      "learning_rate": 3.088235294117647e-07,
      "logits/chosen": 0.049509014934301376,
      "logits/rejected": 0.09894949197769165,
      "logps/chosen": -38.46732711791992,
      "logps/rejected": -53.03515625,
      "loss": 0.5683,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.25855958461761475,
      "rewards/margins": 0.35330522060394287,
      "rewards/rejected": -0.6118648052215576,
      "step": 21
    },
    {
      "epoch": 0.2607407407407407,
      "grad_norm": 49.41172490919738,
      "learning_rate": 3.2352941176470586e-07,
      "logits/chosen": 0.09565885365009308,
      "logits/rejected": 0.13914039731025696,
      "logps/chosen": -28.3892765045166,
      "logps/rejected": -40.65375518798828,
      "loss": 0.5608,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2191687971353531,
      "rewards/margins": 0.21512456238269806,
      "rewards/rejected": -0.43429338932037354,
      "step": 22
    },
    {
      "epoch": 0.2725925925925926,
      "grad_norm": 46.290056641280934,
      "learning_rate": 3.3823529411764707e-07,
      "logits/chosen": 0.33468642830848694,
      "logits/rejected": 0.3287414312362671,
      "logps/chosen": -41.56681823730469,
      "logps/rejected": -49.69163131713867,
      "loss": 0.5531,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6832336187362671,
      "rewards/margins": 0.1509827822446823,
      "rewards/rejected": -0.8342164158821106,
      "step": 23
    },
    {
      "epoch": 0.28444444444444444,
      "grad_norm": 45.27726525885794,
      "learning_rate": 3.529411764705882e-07,
      "logits/chosen": 0.29047060012817383,
      "logits/rejected": 0.25504833459854126,
      "logps/chosen": -40.559715270996094,
      "logps/rejected": -43.744140625,
      "loss": 0.5839,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4380490183830261,
      "rewards/margins": 0.21297261118888855,
      "rewards/rejected": -0.6510215997695923,
      "step": 24
    },
    {
      "epoch": 0.2962962962962963,
      "grad_norm": 45.96606934366563,
      "learning_rate": 3.6764705882352943e-07,
      "logits/chosen": 0.19954881072044373,
      "logits/rejected": 0.24337519705295563,
      "logps/chosen": -26.71196937561035,
      "logps/rejected": -45.159339904785156,
      "loss": 0.5114,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4065704047679901,
      "rewards/margins": 0.6559778451919556,
      "rewards/rejected": -1.0625481605529785,
      "step": 25
    },
    {
      "epoch": 0.30814814814814817,
      "grad_norm": 40.51480580129527,
      "learning_rate": 3.8235294117647053e-07,
      "logits/chosen": 0.18569591641426086,
      "logits/rejected": 0.24005870521068573,
      "logps/chosen": -32.402259826660156,
      "logps/rejected": -50.5438117980957,
      "loss": 0.4871,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.35379940271377563,
      "rewards/margins": 0.88099205493927,
      "rewards/rejected": -1.2347913980484009,
      "step": 26
    },
    {
      "epoch": 0.32,
      "grad_norm": 42.88615154569158,
      "learning_rate": 3.9705882352941174e-07,
      "logits/chosen": 0.28236454725265503,
      "logits/rejected": 0.25888901948928833,
      "logps/chosen": -39.2940673828125,
      "logps/rejected": -53.0938606262207,
      "loss": 0.4142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6994470953941345,
      "rewards/margins": 1.0878021717071533,
      "rewards/rejected": -1.7872494459152222,
      "step": 27
    },
    {
      "epoch": 0.33185185185185184,
      "grad_norm": 47.29875578100757,
      "learning_rate": 4.117647058823529e-07,
      "logits/chosen": 0.3194928467273712,
      "logits/rejected": 0.32101473212242126,
      "logps/chosen": -45.893978118896484,
      "logps/rejected": -52.5146484375,
      "loss": 0.5076,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0526106357574463,
      "rewards/margins": 0.718239426612854,
      "rewards/rejected": -1.7708501815795898,
      "step": 28
    },
    {
      "epoch": 0.3437037037037037,
      "grad_norm": 37.18564900832489,
      "learning_rate": 4.264705882352941e-07,
      "logits/chosen": 0.19094619154930115,
      "logits/rejected": 0.23362189531326294,
      "logps/chosen": -36.12297439575195,
      "logps/rejected": -46.864871978759766,
      "loss": 0.4014,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7660520076751709,
      "rewards/margins": 1.0089162588119507,
      "rewards/rejected": -1.774968147277832,
      "step": 29
    },
    {
      "epoch": 0.35555555555555557,
      "grad_norm": 43.31791813857577,
      "learning_rate": 4.4117647058823526e-07,
      "logits/chosen": 0.15151675045490265,
      "logits/rejected": 0.11997775733470917,
      "logps/chosen": -38.58032989501953,
      "logps/rejected": -48.824798583984375,
      "loss": 0.4572,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2670563459396362,
      "rewards/margins": 0.831606388092041,
      "rewards/rejected": -2.098662853240967,
      "step": 30
    },
    {
      "epoch": 0.3674074074074074,
      "grad_norm": 37.063035160208685,
      "learning_rate": 4.5588235294117646e-07,
      "logits/chosen": 0.2560815215110779,
      "logits/rejected": 0.3223097026348114,
      "logps/chosen": -33.990726470947266,
      "logps/rejected": -47.65345764160156,
      "loss": 0.3806,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2836799621582031,
      "rewards/margins": 1.0427335500717163,
      "rewards/rejected": -2.326413631439209,
      "step": 31
    },
    {
      "epoch": 0.37925925925925924,
      "grad_norm": 37.90256850942511,
      "learning_rate": 4.705882352941176e-07,
      "logits/chosen": 0.18444910645484924,
      "logits/rejected": 0.27790239453315735,
      "logps/chosen": -34.04519271850586,
      "logps/rejected": -58.64192199707031,
      "loss": 0.3634,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8921431303024292,
      "rewards/margins": 1.9067623615264893,
      "rewards/rejected": -2.798905372619629,
      "step": 32
    },
    {
      "epoch": 0.39111111111111113,
      "grad_norm": 38.28825801306995,
      "learning_rate": 4.852941176470588e-07,
      "logits/chosen": 0.21531561017036438,
      "logits/rejected": 0.22173307836055756,
      "logps/chosen": -38.73649978637695,
      "logps/rejected": -51.891937255859375,
      "loss": 0.3549,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.469386339187622,
      "rewards/margins": 1.3347928524017334,
      "rewards/rejected": -2.8041794300079346,
      "step": 33
    },
    {
      "epoch": 0.40296296296296297,
      "grad_norm": 53.57688480965446,
      "learning_rate": 5e-07,
      "logits/chosen": -0.14031767845153809,
      "logits/rejected": -0.009732939302921295,
      "logps/chosen": -30.219802856445312,
      "logps/rejected": -48.51620864868164,
      "loss": 0.4584,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7153716683387756,
      "rewards/margins": 2.133821487426758,
      "rewards/rejected": -2.8491930961608887,
      "step": 34
    },
    {
      "epoch": 0.4148148148148148,
      "grad_norm": 43.6191926386301,
      "learning_rate": 4.999864732969518e-07,
      "logits/chosen": 0.2249789983034134,
      "logits/rejected": 0.2375878542661667,
      "logps/chosen": -42.989952087402344,
      "logps/rejected": -60.248451232910156,
      "loss": 0.307,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3359755277633667,
      "rewards/margins": 2.923412799835205,
      "rewards/rejected": -4.259388446807861,
      "step": 35
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 41.68496640354453,
      "learning_rate": 4.999458946515807e-07,
      "logits/chosen": 0.04158564656972885,
      "logits/rejected": 0.04120251536369324,
      "logps/chosen": -47.079593658447266,
      "logps/rejected": -64.6259536743164,
      "loss": 0.3243,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9504570960998535,
      "rewards/margins": 2.1364314556121826,
      "rewards/rejected": -4.086888313293457,
      "step": 36
    },
    {
      "epoch": 0.43851851851851853,
      "grad_norm": 37.998309675066224,
      "learning_rate": 4.998782684550491e-07,
      "logits/chosen": 0.15689387917518616,
      "logits/rejected": 0.22760489583015442,
      "logps/chosen": -31.412202835083008,
      "logps/rejected": -57.521270751953125,
      "loss": 0.3499,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.422287940979004,
      "rewards/margins": 2.10537052154541,
      "rewards/rejected": -3.527658462524414,
      "step": 37
    },
    {
      "epoch": 0.45037037037037037,
      "grad_norm": 39.14624538309661,
      "learning_rate": 4.997836020254328e-07,
      "logits/chosen": 0.09242415428161621,
      "logits/rejected": 0.12390726059675217,
      "logps/chosen": -38.68524932861328,
      "logps/rejected": -59.322471618652344,
      "loss": 0.3809,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4135229587554932,
      "rewards/margins": 2.5775866508483887,
      "rewards/rejected": -3.991109609603882,
      "step": 38
    },
    {
      "epoch": 0.4622222222222222,
      "grad_norm": 34.10903781264625,
      "learning_rate": 4.996619056069291e-07,
      "logits/chosen": 0.15454381704330444,
      "logits/rejected": 0.16882330179214478,
      "logps/chosen": -44.294654846191406,
      "logps/rejected": -66.8642578125,
      "loss": 0.3106,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.2089152336120605,
      "rewards/margins": 3.4627161026000977,
      "rewards/rejected": -5.671631336212158,
      "step": 39
    },
    {
      "epoch": 0.4740740740740741,
      "grad_norm": 41.64494289319624,
      "learning_rate": 4.995131923687487e-07,
      "logits/chosen": 0.03869347274303436,
      "logits/rejected": 0.13989922404289246,
      "logps/chosen": -48.224884033203125,
      "logps/rejected": -68.6059341430664,
      "loss": 0.3563,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.3006927967071533,
      "rewards/margins": 3.536550521850586,
      "rewards/rejected": -5.837243556976318,
      "step": 40
    },
    {
      "epoch": 0.48592592592592593,
      "grad_norm": 43.27974364485945,
      "learning_rate": 4.993374784036901e-07,
      "logits/chosen": -0.13991862535476685,
      "logits/rejected": 0.015330532565712929,
      "logps/chosen": -44.3278923034668,
      "logps/rejected": -62.52472686767578,
      "loss": 0.4348,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.4953625202178955,
      "rewards/margins": 3.03660249710083,
      "rewards/rejected": -5.5319647789001465,
      "step": 41
    },
    {
      "epoch": 0.49777777777777776,
      "grad_norm": 46.320431667154644,
      "learning_rate": 4.991347827263982e-07,
      "logits/chosen": -0.051238611340522766,
      "logits/rejected": -0.0033771172165870667,
      "logps/chosen": -43.90919876098633,
      "logps/rejected": -65.11723327636719,
      "loss": 0.4051,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.612180471420288,
      "rewards/margins": 3.3464155197143555,
      "rewards/rejected": -4.9585957527160645,
      "step": 42
    },
    {
      "epoch": 0.5096296296296297,
      "grad_norm": 52.045033376790954,
      "learning_rate": 4.989051272713069e-07,
      "logits/chosen": -0.10396721214056015,
      "logits/rejected": 0.10225249826908112,
      "logps/chosen": -45.72818374633789,
      "logps/rejected": -77.21394348144531,
      "loss": 0.3283,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.5264527797698975,
      "rewards/margins": 4.776139736175537,
      "rewards/rejected": -7.302592754364014,
      "step": 43
    },
    {
      "epoch": 0.5214814814814814,
      "grad_norm": 40.91521718681392,
      "learning_rate": 4.986485368902656e-07,
      "logits/chosen": -0.08990158140659332,
      "logits/rejected": 0.014565035700798035,
      "logps/chosen": -38.39900588989258,
      "logps/rejected": -57.75357437133789,
      "loss": 0.3584,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.275327205657959,
      "rewards/margins": 2.417283058166504,
      "rewards/rejected": -4.692610263824463,
      "step": 44
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 50.74127412580387,
      "learning_rate": 4.983650393498489e-07,
      "logits/chosen": 0.037050001323223114,
      "logits/rejected": -0.008276170119643211,
      "logps/chosen": -50.93760299682617,
      "logps/rejected": -55.42501449584961,
      "loss": 0.3717,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.5346438884735107,
      "rewards/margins": 1.2831640243530273,
      "rewards/rejected": -3.8178083896636963,
      "step": 45
    },
    {
      "epoch": 0.5451851851851852,
      "grad_norm": 38.61624054825256,
      "learning_rate": 4.980546653283537e-07,
      "logits/chosen": -0.41439855098724365,
      "logits/rejected": -0.4113887548446655,
      "logps/chosen": -41.68315124511719,
      "logps/rejected": -67.61707305908203,
      "loss": 0.3181,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8887252807617188,
      "rewards/margins": 4.549580097198486,
      "rewards/rejected": -6.438305377960205,
      "step": 46
    },
    {
      "epoch": 0.557037037037037,
      "grad_norm": 38.9863916338003,
      "learning_rate": 4.977174484124775e-07,
      "logits/chosen": -0.009788192808628082,
      "logits/rejected": -0.09325724095106125,
      "logps/chosen": -46.573936462402344,
      "logps/rejected": -59.36362838745117,
      "loss": 0.2595,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1944921016693115,
      "rewards/margins": 4.177679538726807,
      "rewards/rejected": -6.372171401977539,
      "step": 47
    },
    {
      "epoch": 0.5688888888888889,
      "grad_norm": 46.90129706189482,
      "learning_rate": 4.97353425093685e-07,
      "logits/chosen": -0.1219746470451355,
      "logits/rejected": -0.10111116617918015,
      "logps/chosen": -49.24342346191406,
      "logps/rejected": -65.33878326416016,
      "loss": 0.3779,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.4194295406341553,
      "rewards/margins": 3.601774215698242,
      "rewards/rejected": -6.021203994750977,
      "step": 48
    },
    {
      "epoch": 0.5807407407407408,
      "grad_norm": 54.305491610936826,
      "learning_rate": 4.96962634764259e-07,
      "logits/chosen": -0.13463924825191498,
      "logits/rejected": -0.09778477251529694,
      "logps/chosen": -50.75926971435547,
      "logps/rejected": -64.77645111083984,
      "loss": 0.3942,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.944636344909668,
      "rewards/margins": 3.1688618659973145,
      "rewards/rejected": -6.113498687744141,
      "step": 49
    },
    {
      "epoch": 0.5925925925925926,
      "grad_norm": 40.79377104202132,
      "learning_rate": 4.965451197130372e-07,
      "logits/chosen": -0.0598304346203804,
      "logits/rejected": 0.03133855387568474,
      "logps/chosen": -41.48918151855469,
      "logps/rejected": -72.72964477539062,
      "loss": 0.3083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6170328855514526,
      "rewards/margins": 4.565612316131592,
      "rewards/rejected": -6.182644844055176,
      "step": 50
    },
    {
      "epoch": 0.6044444444444445,
      "grad_norm": 42.877178470441834,
      "learning_rate": 4.961009251208367e-07,
      "logits/chosen": -0.014419106766581535,
      "logits/rejected": -0.018680818378925323,
      "logps/chosen": -34.062870025634766,
      "logps/rejected": -66.85511779785156,
      "loss": 0.2943,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9279564023017883,
      "rewards/margins": 6.2386088371276855,
      "rewards/rejected": -7.166565418243408,
      "step": 51
    },
    {
      "epoch": 0.6162962962962963,
      "grad_norm": 36.563663666294495,
      "learning_rate": 4.956300990555643e-07,
      "logits/chosen": -0.23208701610565186,
      "logits/rejected": -0.15281593799591064,
      "logps/chosen": -34.16992950439453,
      "logps/rejected": -48.25387954711914,
      "loss": 0.2707,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.390017032623291,
      "rewards/margins": 2.9138500690460205,
      "rewards/rejected": -4.303867340087891,
      "step": 52
    },
    {
      "epoch": 0.6281481481481481,
      "grad_norm": 48.996542745383714,
      "learning_rate": 4.951326924670147e-07,
      "logits/chosen": -0.029582835733890533,
      "logits/rejected": 0.13870403170585632,
      "logps/chosen": -46.177825927734375,
      "logps/rejected": -64.03628540039062,
      "loss": 0.4606,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.360954761505127,
      "rewards/margins": 2.338740825653076,
      "rewards/rejected": -4.699695587158203,
      "step": 53
    },
    {
      "epoch": 0.64,
      "grad_norm": 37.53257312713413,
      "learning_rate": 4.94608759181358e-07,
      "logits/chosen": -0.2242709845304489,
      "logits/rejected": -0.022289041429758072,
      "logps/chosen": -43.091976165771484,
      "logps/rejected": -56.94826126098633,
      "loss": 0.2428,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1584746837615967,
      "rewards/margins": 2.6602895259857178,
      "rewards/rejected": -3.8187644481658936,
      "step": 54
    },
    {
      "epoch": 0.6518518518518519,
      "grad_norm": 38.495616892469634,
      "learning_rate": 4.940583558953137e-07,
      "logits/chosen": -0.3163710832595825,
      "logits/rejected": -0.2686666250228882,
      "logps/chosen": -41.02560806274414,
      "logps/rejected": -75.63497924804688,
      "loss": 0.3113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4144959449768066,
      "rewards/margins": 5.705674648284912,
      "rewards/rejected": -7.120170593261719,
      "step": 55
    },
    {
      "epoch": 0.6637037037037037,
      "grad_norm": 44.72353141808658,
      "learning_rate": 4.934815421700164e-07,
      "logits/chosen": -0.28492411971092224,
      "logits/rejected": -0.2709801495075226,
      "logps/chosen": -36.71954345703125,
      "logps/rejected": -55.933624267578125,
      "loss": 0.3607,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8905187845230103,
      "rewards/margins": 3.9906599521636963,
      "rewards/rejected": -4.881179332733154,
      "step": 56
    },
    {
      "epoch": 0.6755555555555556,
      "grad_norm": 36.35665672904958,
      "learning_rate": 4.928783804245699e-07,
      "logits/chosen": 0.2555558681488037,
      "logits/rejected": 0.18786108493804932,
      "logps/chosen": -40.80218505859375,
      "logps/rejected": -54.24163055419922,
      "loss": 0.2751,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8028124570846558,
      "rewards/margins": 3.192255735397339,
      "rewards/rejected": -3.995068073272705,
      "step": 57
    },
    {
      "epoch": 0.6874074074074074,
      "grad_norm": 29.859662847563737,
      "learning_rate": 4.922489359292927e-07,
      "logits/chosen": -0.17547199130058289,
      "logits/rejected": -0.06896121799945831,
      "logps/chosen": -40.20637893676758,
      "logps/rejected": -68.45681762695312,
      "loss": 0.2406,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9151208400726318,
      "rewards/margins": 4.224143981933594,
      "rewards/rejected": -5.139265060424805,
      "step": 58
    },
    {
      "epoch": 0.6992592592592592,
      "grad_norm": 33.69014037830799,
      "learning_rate": 4.915932767986551e-07,
      "logits/chosen": -0.2176772654056549,
      "logits/rejected": -0.14603368937969208,
      "logps/chosen": -35.77494430541992,
      "logps/rejected": -56.28825378417969,
      "loss": 0.2639,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7671470642089844,
      "rewards/margins": 3.049193859100342,
      "rewards/rejected": -3.816340923309326,
      "step": 59
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 33.1631924356696,
      "learning_rate": 4.909114739839079e-07,
      "logits/chosen": -0.09617012739181519,
      "logits/rejected": -0.08796259015798569,
      "logps/chosen": -33.88630294799805,
      "logps/rejected": -55.623878479003906,
      "loss": 0.2556,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8068188428878784,
      "rewards/margins": 3.1127305030822754,
      "rewards/rejected": -3.9195497035980225,
      "step": 60
    },
    {
      "epoch": 0.7229629629629629,
      "grad_norm": 40.838783872872355,
      "learning_rate": 4.902036012654048e-07,
      "logits/chosen": 0.11093666404485703,
      "logits/rejected": 0.1355137974023819,
      "logps/chosen": -34.699256896972656,
      "logps/rejected": -55.77449035644531,
      "loss": 0.2753,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.288212776184082,
      "rewards/margins": 3.224027633666992,
      "rewards/rejected": -4.512240409851074,
      "step": 61
    },
    {
      "epoch": 0.7348148148148148,
      "grad_norm": 35.91873621159586,
      "learning_rate": 4.894697352446182e-07,
      "logits/chosen": -0.10412248969078064,
      "logits/rejected": -0.1209147572517395,
      "logps/chosen": -34.93061447143555,
      "logps/rejected": -52.149208068847656,
      "loss": 0.2958,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7685253620147705,
      "rewards/margins": 2.5662384033203125,
      "rewards/rejected": -3.334764003753662,
      "step": 62
    },
    {
      "epoch": 0.7466666666666667,
      "grad_norm": 28.279332840993643,
      "learning_rate": 4.887099553358501e-07,
      "logits/chosen": -0.1916661560535431,
      "logits/rejected": -0.14164935052394867,
      "logps/chosen": -40.58860397338867,
      "logps/rejected": -50.00385284423828,
      "loss": 0.229,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3654404282569885,
      "rewards/margins": 2.5260181427001953,
      "rewards/rejected": -2.891458511352539,
      "step": 63
    },
    {
      "epoch": 0.7585185185185185,
      "grad_norm": 43.562956759714005,
      "learning_rate": 4.879243437576383e-07,
      "logits/chosen": -0.09250672161579132,
      "logits/rejected": -0.06184221804141998,
      "logps/chosen": -33.61621856689453,
      "logps/rejected": -48.81718826293945,
      "loss": 0.286,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7941789031028748,
      "rewards/margins": 2.6042628288269043,
      "rewards/rejected": -3.398441791534424,
      "step": 64
    },
    {
      "epoch": 0.7703703703703704,
      "grad_norm": 43.71274039442652,
      "learning_rate": 4.871129855238588e-07,
      "logits/chosen": -0.1322498917579651,
      "logits/rejected": -0.05121883377432823,
      "logps/chosen": -40.41231918334961,
      "logps/rejected": -68.0873031616211,
      "loss": 0.3236,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.39779654145240784,
      "rewards/margins": 3.27742600440979,
      "rewards/rejected": -3.675222396850586,
      "step": 65
    },
    {
      "epoch": 0.7822222222222223,
      "grad_norm": 33.295271431641005,
      "learning_rate": 4.862759684345269e-07,
      "logits/chosen": -0.35007691383361816,
      "logits/rejected": -0.3479149341583252,
      "logps/chosen": -38.88081359863281,
      "logps/rejected": -49.428382873535156,
      "loss": 0.245,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.42857033014297485,
      "rewards/margins": 3.455155372619629,
      "rewards/rejected": -3.883725643157959,
      "step": 66
    },
    {
      "epoch": 0.794074074074074,
      "grad_norm": 22.368867699781628,
      "learning_rate": 4.854133830662955e-07,
      "logits/chosen": -0.26884549856185913,
      "logits/rejected": -0.27821600437164307,
      "logps/chosen": -40.386207580566406,
      "logps/rejected": -55.080604553222656,
      "loss": 0.1921,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1675169467926025,
      "rewards/margins": 4.228756904602051,
      "rewards/rejected": -5.396273136138916,
      "step": 67
    },
    {
      "epoch": 0.8059259259259259,
      "grad_norm": 32.90815737667523,
      "learning_rate": 4.845253227626536e-07,
      "logits/chosen": 0.12926915287971497,
      "logits/rejected": 0.03172997385263443,
      "logps/chosen": -56.2171516418457,
      "logps/rejected": -61.29096984863281,
      "loss": 0.2468,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6303930282592773,
      "rewards/margins": 2.693972110748291,
      "rewards/rejected": -3.3243653774261475,
      "step": 68
    },
    {
      "epoch": 0.8177777777777778,
      "grad_norm": 31.286678704407063,
      "learning_rate": 4.836118836238252e-07,
      "logits/chosen": -0.11010141670703888,
      "logits/rejected": -0.09770654886960983,
      "logps/chosen": -38.13661193847656,
      "logps/rejected": -59.40013122558594,
      "loss": 0.2512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08025422692298889,
      "rewards/margins": 2.979079246520996,
      "rewards/rejected": -2.898824691772461,
      "step": 69
    },
    {
      "epoch": 0.8296296296296296,
      "grad_norm": 27.574620445125923,
      "learning_rate": 4.826731644963704e-07,
      "logits/chosen": -0.25498491525650024,
      "logits/rejected": -0.24903275072574615,
      "logps/chosen": -32.7315788269043,
      "logps/rejected": -47.2717399597168,
      "loss": 0.222,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6683451533317566,
      "rewards/margins": 3.523477554321289,
      "rewards/rejected": -4.191822528839111,
      "step": 70
    },
    {
      "epoch": 0.8414814814814815,
      "grad_norm": 35.73348427460115,
      "learning_rate": 4.817092669624882e-07,
      "logits/chosen": -0.018255462870001793,
      "logits/rejected": -0.0020784977823495865,
      "logps/chosen": -34.584449768066406,
      "logps/rejected": -54.02947998046875,
      "loss": 0.3535,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04187864065170288,
      "rewards/margins": 3.698057174682617,
      "rewards/rejected": -3.739936113357544,
      "step": 71
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 28.886553550180775,
      "learning_rate": 4.807202953290243e-07,
      "logits/chosen": -0.2388785183429718,
      "logits/rejected": -0.1512915939092636,
      "logps/chosen": -32.676937103271484,
      "logps/rejected": -51.861934661865234,
      "loss": 0.2502,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.43571335077285767,
      "rewards/margins": 3.212143898010254,
      "rewards/rejected": -3.6478569507598877,
      "step": 72
    },
    {
      "epoch": 0.8651851851851852,
      "grad_norm": 31.329548468083193,
      "learning_rate": 4.797063566161834e-07,
      "logits/chosen": -0.016514137387275696,
      "logits/rejected": -0.02128826081752777,
      "logps/chosen": -42.21635818481445,
      "logps/rejected": -53.1080436706543,
      "loss": 0.2729,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.650065541267395,
      "rewards/margins": 1.7332451343536377,
      "rewards/rejected": -2.383310556411743,
      "step": 73
    },
    {
      "epoch": 0.8770370370370371,
      "grad_norm": 25.241915116630672,
      "learning_rate": 4.786675605459487e-07,
      "logits/chosen": -0.18203724920749664,
      "logits/rejected": -0.12511923909187317,
      "logps/chosen": -37.860111236572266,
      "logps/rejected": -67.00776672363281,
      "loss": 0.2269,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.003457695245742798,
      "rewards/margins": 4.446203708648682,
      "rewards/rejected": -4.442746162414551,
      "step": 74
    },
    {
      "epoch": 0.8888888888888888,
      "grad_norm": 26.546681076158777,
      "learning_rate": 4.776040195302079e-07,
      "logits/chosen": -0.20350059866905212,
      "logits/rejected": -0.16488413512706757,
      "logps/chosen": -29.37200164794922,
      "logps/rejected": -53.26748275756836,
      "loss": 0.2272,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.028457432985305786,
      "rewards/margins": 4.081994533538818,
      "rewards/rejected": -4.053536891937256,
      "step": 75
    },
    {
      "epoch": 0.9007407407407407,
      "grad_norm": 36.79575801377004,
      "learning_rate": 4.76515848658589e-07,
      "logits/chosen": -0.06815146654844284,
      "logits/rejected": 0.04448368400335312,
      "logps/chosen": -40.69670867919922,
      "logps/rejected": -58.95963668823242,
      "loss": 0.3201,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6759670972824097,
      "rewards/margins": 2.9853808879852295,
      "rewards/rejected": -3.6613478660583496,
      "step": 76
    },
    {
      "epoch": 0.9125925925925926,
      "grad_norm": 33.040470283627364,
      "learning_rate": 4.754031656860059e-07,
      "logits/chosen": 0.059698522090911865,
      "logits/rejected": 0.1003262847661972,
      "logps/chosen": -36.32437515258789,
      "logps/rejected": -45.55727005004883,
      "loss": 0.2451,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1681911200284958,
      "rewards/margins": 3.335202217102051,
      "rewards/rejected": -3.167011022567749,
      "step": 77
    },
    {
      "epoch": 0.9244444444444444,
      "grad_norm": 23.533686338243243,
      "learning_rate": 4.74266091019916e-07,
      "logits/chosen": -0.03607799857854843,
      "logits/rejected": -0.07650981843471527,
      "logps/chosen": -40.76026153564453,
      "logps/rejected": -51.91889190673828,
      "loss": 0.1909,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06481152772903442,
      "rewards/margins": 3.4574201107025146,
      "rewards/rejected": -3.392608642578125,
      "step": 78
    },
    {
      "epoch": 0.9362962962962963,
      "grad_norm": 34.00814256216538,
      "learning_rate": 4.7310474770728996e-07,
      "logits/chosen": -0.24094080924987793,
      "logits/rejected": -0.2244417816400528,
      "logps/chosen": -36.15470504760742,
      "logps/rejected": -50.766448974609375,
      "loss": 0.3092,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04541383683681488,
      "rewards/margins": 2.0906879901885986,
      "rewards/rejected": -2.1361019611358643,
      "step": 79
    },
    {
      "epoch": 0.9481481481481482,
      "grad_norm": 28.43036399847798,
      "learning_rate": 4.719192614212969e-07,
      "logits/chosen": 0.04508206248283386,
      "logits/rejected": 0.04841914027929306,
      "logps/chosen": -44.18794631958008,
      "logps/rejected": -74.9228515625,
      "loss": 0.1841,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0396180152893066,
      "rewards/margins": 3.7275805473327637,
      "rewards/rejected": -4.76719856262207,
      "step": 80
    },
    {
      "epoch": 0.96,
      "grad_norm": 32.15305104854747,
      "learning_rate": 4.707097604477045e-07,
      "logits/chosen": 0.10437710583209991,
      "logits/rejected": 0.10021185874938965,
      "logps/chosen": -41.348716735839844,
      "logps/rejected": -53.517127990722656,
      "loss": 0.2768,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.03640615940093994,
      "rewards/margins": 3.230294704437256,
      "rewards/rejected": -3.1938881874084473,
      "step": 81
    },
    {
      "epoch": 0.9718518518518519,
      "grad_norm": 30.563184515965578,
      "learning_rate": 4.694763756709967e-07,
      "logits/chosen": -0.17636063694953918,
      "logits/rejected": -0.21469731628894806,
      "logps/chosen": -39.04137420654297,
      "logps/rejected": -52.79186248779297,
      "loss": 0.2415,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2509702444076538,
      "rewards/margins": 3.593198776245117,
      "rewards/rejected": -3.8441689014434814,
      "step": 82
    },
    {
      "epoch": 0.9837037037037037,
      "grad_norm": 28.891738925831007,
      "learning_rate": 4.6821924056021053e-07,
      "logits/chosen": -0.11742343008518219,
      "logits/rejected": 0.009909386746585369,
      "logps/chosen": -30.184894561767578,
      "logps/rejected": -65.191650390625,
      "loss": 0.2097,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2642693817615509,
      "rewards/margins": 5.374275207519531,
      "rewards/rejected": -5.638545036315918,
      "step": 83
    },
    {
      "epoch": 0.9955555555555555,
      "grad_norm": 34.49435985459448,
      "learning_rate": 4.669384911544926e-07,
      "logits/chosen": -0.07497820258140564,
      "logits/rejected": -0.026325395330786705,
      "logps/chosen": -33.93386459350586,
      "logps/rejected": -50.840423583984375,
      "loss": 0.2826,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5045543909072876,
      "rewards/margins": 2.2327775955200195,
      "rewards/rejected": -2.7373318672180176,
      "step": 84
    },
    {
      "epoch": 1.0074074074074073,
      "grad_norm": 25.60399667566007,
      "learning_rate": 4.6563426604837817e-07,
      "logits/chosen": -0.07658643275499344,
      "logits/rejected": -0.06745781004428864,
      "logps/chosen": -45.01917266845703,
      "logps/rejected": -59.43114471435547,
      "loss": 0.1973,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.29737135767936707,
      "rewards/margins": 5.139443874359131,
      "rewards/rejected": -5.4368157386779785,
      "step": 85
    },
    {
      "epoch": 1.0192592592592593,
      "grad_norm": 21.460741017889536,
      "learning_rate": 4.6430670637679294e-07,
      "logits/chosen": -0.2205628752708435,
      "logits/rejected": -0.0864521712064743,
      "logps/chosen": -31.902761459350586,
      "logps/rejected": -51.85260009765625,
      "loss": 0.1667,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.194830521941185,
      "rewards/margins": 3.8075246810913086,
      "rewards/rejected": -4.002355098724365,
      "step": 86
    },
    {
      "epoch": 1.031111111111111,
      "grad_norm": 15.350784705262274,
      "learning_rate": 4.629559557997804e-07,
      "logits/chosen": -0.12179061770439148,
      "logits/rejected": -0.10868389904499054,
      "logps/chosen": -40.21116638183594,
      "logps/rejected": -62.20214080810547,
      "loss": 0.1195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19310609996318817,
      "rewards/margins": 4.397428512573242,
      "rewards/rejected": -4.590534687042236,
      "step": 87
    },
    {
      "epoch": 1.0429629629629629,
      "grad_norm": 13.188223717252573,
      "learning_rate": 4.615821604869563e-07,
      "logits/chosen": -0.13621510565280914,
      "logits/rejected": -0.044875748455524445,
      "logps/chosen": -39.49315643310547,
      "logps/rejected": -65.40257263183594,
      "loss": 0.1026,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10489638894796371,
      "rewards/margins": 5.503582000732422,
      "rewards/rejected": -5.608478546142578,
      "step": 88
    },
    {
      "epoch": 1.0548148148148149,
      "grad_norm": 27.4435618488601,
      "learning_rate": 4.6018546910169067e-07,
      "logits/chosen": -0.2304653376340866,
      "logits/rejected": -0.30428558588027954,
      "logps/chosen": -38.354759216308594,
      "logps/rejected": -57.675498962402344,
      "loss": 0.1946,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1100564002990723,
      "rewards/margins": 4.001605033874512,
      "rewards/rejected": -5.111660957336426,
      "step": 89
    },
    {
      "epoch": 1.0666666666666667,
      "grad_norm": 18.807247445635458,
      "learning_rate": 4.5876603278502027e-07,
      "logits/chosen": -0.12274541705846786,
      "logits/rejected": 0.009613536298274994,
      "logps/chosen": -41.16261291503906,
      "logps/rejected": -73.863037109375,
      "loss": 0.1405,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8482255935668945,
      "rewards/margins": 5.244417667388916,
      "rewards/rejected": -6.0926432609558105,
      "step": 90
    },
    {
      "epoch": 1.0785185185185184,
      "grad_norm": 24.62512250085559,
      "learning_rate": 4.573240051392935e-07,
      "logits/chosen": -0.1303870528936386,
      "logits/rejected": -0.14886482059955597,
      "logps/chosen": -39.42107009887695,
      "logps/rejected": -54.32372283935547,
      "loss": 0.1612,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6151587963104248,
      "rewards/margins": 3.3702774047851562,
      "rewards/rejected": -3.98543643951416,
      "step": 91
    },
    {
      "epoch": 1.0903703703703704,
      "grad_norm": 18.29323289164473,
      "learning_rate": 4.5585954221154853e-07,
      "logits/chosen": -0.4573056101799011,
      "logits/rejected": -0.31245023012161255,
      "logps/chosen": -32.956050872802734,
      "logps/rejected": -59.963035583496094,
      "loss": 0.1649,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5122833251953125,
      "rewards/margins": 4.091685771942139,
      "rewards/rejected": -4.603969097137451,
      "step": 92
    },
    {
      "epoch": 1.1022222222222222,
      "grad_norm": 19.321908739095367,
      "learning_rate": 4.5437280247662646e-07,
      "logits/chosen": -0.023740939795970917,
      "logits/rejected": -0.0003247186541557312,
      "logps/chosen": -39.27166748046875,
      "logps/rejected": -55.00670623779297,
      "loss": 0.1251,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7074174880981445,
      "rewards/margins": 3.3870468139648438,
      "rewards/rejected": -4.094464302062988,
      "step": 93
    },
    {
      "epoch": 1.114074074074074,
      "grad_norm": 17.03649474940601,
      "learning_rate": 4.528639468200226e-07,
      "logits/chosen": 0.2261081337928772,
      "logits/rejected": 0.2597027122974396,
      "logps/chosen": -37.43301773071289,
      "logps/rejected": -51.47749328613281,
      "loss": 0.1186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10310641676187515,
      "rewards/margins": 3.6022109985351562,
      "rewards/rejected": -3.4991047382354736,
      "step": 94
    },
    {
      "epoch": 1.125925925925926,
      "grad_norm": 16.559378309982034,
      "learning_rate": 4.5133313852047613e-07,
      "logits/chosen": -0.11725334078073502,
      "logits/rejected": -0.08741730451583862,
      "logps/chosen": -33.64775085449219,
      "logps/rejected": -55.1715087890625,
      "loss": 0.1273,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.01142565906047821,
      "rewards/margins": 3.431326150894165,
      "rewards/rejected": -3.442751407623291,
      "step": 95
    },
    {
      "epoch": 1.1377777777777778,
      "grad_norm": 20.611754629262013,
      "learning_rate": 4.4978054323230144e-07,
      "logits/chosen": 0.051543403416872025,
      "logits/rejected": 0.11705614626407623,
      "logps/chosen": -33.534278869628906,
      "logps/rejected": -51.728057861328125,
      "loss": 0.1365,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07938066124916077,
      "rewards/margins": 3.254304885864258,
      "rewards/rejected": -3.17492413520813,
      "step": 96
    },
    {
      "epoch": 1.1496296296296296,
      "grad_norm": 12.616532653018533,
      "learning_rate": 4.482063289674618e-07,
      "logits/chosen": -0.0702984482049942,
      "logits/rejected": 0.004200812429189682,
      "logps/chosen": -34.993507385253906,
      "logps/rejected": -57.94596481323242,
      "loss": 0.0914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19305679202079773,
      "rewards/margins": 4.31368350982666,
      "rewards/rejected": -4.120626449584961,
      "step": 97
    },
    {
      "epoch": 1.1614814814814816,
      "grad_norm": 12.762301209570738,
      "learning_rate": 4.466106660773884e-07,
      "logits/chosen": -0.1491287350654602,
      "logits/rejected": -0.03166097402572632,
      "logps/chosen": -39.788230895996094,
      "logps/rejected": -59.54990005493164,
      "loss": 0.094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.40045109391212463,
      "rewards/margins": 4.702445030212402,
      "rewards/rejected": -5.102896213531494,
      "step": 98
    },
    {
      "epoch": 1.1733333333333333,
      "grad_norm": 17.826250546113457,
      "learning_rate": 4.44993727234546e-07,
      "logits/chosen": 0.11196614801883698,
      "logits/rejected": 0.07824762165546417,
      "logps/chosen": -39.74019241333008,
      "logps/rejected": -49.98065185546875,
      "loss": 0.1199,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.37030303478240967,
      "rewards/margins": 2.883664608001709,
      "rewards/rejected": -3.25396728515625,
      "step": 99
    },
    {
      "epoch": 1.1851851851851851,
      "grad_norm": 16.340929612227512,
      "learning_rate": 4.4335568741374695e-07,
      "logits/chosen": -0.3065292239189148,
      "logits/rejected": -0.243222177028656,
      "logps/chosen": -39.87810516357422,
      "logps/rejected": -49.80472946166992,
      "loss": 0.1362,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.09828188270330429,
      "rewards/margins": 4.0216569900512695,
      "rewards/rejected": -4.119938850402832,
      "step": 100
    },
    {
      "epoch": 1.1970370370370371,
      "grad_norm": 15.511054375166765,
      "learning_rate": 4.4169672387321735e-07,
      "logits/chosen": -0.06340894848108292,
      "logits/rejected": -0.0665612518787384,
      "logps/chosen": -43.078765869140625,
      "logps/rejected": -65.42969512939453,
      "loss": 0.1102,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3077806830406189,
      "rewards/margins": 5.954348087310791,
      "rewards/rejected": -6.262128829956055,
      "step": 101
    },
    {
      "epoch": 1.208888888888889,
      "grad_norm": 17.09598873374456,
      "learning_rate": 4.4001701613541454e-07,
      "logits/chosen": 0.09894056618213654,
      "logits/rejected": 0.11925836652517319,
      "logps/chosen": -32.494293212890625,
      "logps/rejected": -51.34983825683594,
      "loss": 0.128,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22729653120040894,
      "rewards/margins": 4.20453405380249,
      "rewards/rejected": -4.431830406188965,
      "step": 102
    },
    {
      "epoch": 1.2207407407407407,
      "grad_norm": 22.603451847311774,
      "learning_rate": 4.383167459676008e-07,
      "logits/chosen": -0.09994232654571533,
      "logits/rejected": -0.03670894354581833,
      "logps/chosen": -34.782405853271484,
      "logps/rejected": -57.43785095214844,
      "loss": 0.1482,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.40158677101135254,
      "rewards/margins": 3.9667258262634277,
      "rewards/rejected": -4.368312358856201,
      "step": 103
    },
    {
      "epoch": 1.2325925925925927,
      "grad_norm": 17.176654931396367,
      "learning_rate": 4.365960973621734e-07,
      "logits/chosen": -0.3010917007923126,
      "logits/rejected": -0.2688751220703125,
      "logps/chosen": -30.779415130615234,
      "logps/rejected": -59.66587448120117,
      "loss": 0.1077,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5755228996276855,
      "rewards/margins": 5.500998497009277,
      "rewards/rejected": -6.076521396636963,
      "step": 104
    },
    {
      "epoch": 1.2444444444444445,
      "grad_norm": 16.54203727829074,
      "learning_rate": 4.348552565167542e-07,
      "logits/chosen": 0.03528839722275734,
      "logits/rejected": 0.025072041898965836,
      "logps/chosen": -35.11994171142578,
      "logps/rejected": -49.86817169189453,
      "loss": 0.1169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.683509349822998,
      "rewards/margins": 4.702343940734863,
      "rewards/rejected": -5.385853290557861,
      "step": 105
    },
    {
      "epoch": 1.2562962962962962,
      "grad_norm": 26.56869923226247,
      "learning_rate": 4.330944118140406e-07,
      "logits/chosen": -0.08033540099859238,
      "logits/rejected": -0.015175499022006989,
      "logps/chosen": -40.06020736694336,
      "logps/rejected": -58.90888977050781,
      "loss": 0.1479,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.02627614513039589,
      "rewards/margins": 4.818339824676514,
      "rewards/rejected": -4.7920637130737305,
      "step": 106
    },
    {
      "epoch": 1.268148148148148,
      "grad_norm": 14.298587740255858,
      "learning_rate": 4.313137538014198e-07,
      "logits/chosen": -0.08439959585666656,
      "logits/rejected": -0.15276381373405457,
      "logps/chosen": -34.021949768066406,
      "logps/rejected": -43.68275451660156,
      "loss": 0.0835,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.03106861561536789,
      "rewards/margins": 3.872872829437256,
      "rewards/rejected": -3.841804265975952,
      "step": 107
    },
    {
      "epoch": 1.28,
      "grad_norm": 25.029691995336584,
      "learning_rate": 4.295134751703492e-07,
      "logits/chosen": 0.039663467556238174,
      "logits/rejected": 0.03390619903802872,
      "logps/chosen": -50.46000671386719,
      "logps/rejected": -61.62244415283203,
      "loss": 0.1408,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.41419780254364014,
      "rewards/margins": 5.797426223754883,
      "rewards/rejected": -6.2116241455078125,
      "step": 108
    },
    {
      "epoch": 1.2918518518518518,
      "grad_norm": 23.688132743134673,
      "learning_rate": 4.276937707355044e-07,
      "logits/chosen": -0.017152896150946617,
      "logits/rejected": -0.053842976689338684,
      "logps/chosen": -41.905521392822266,
      "logps/rejected": -65.56050109863281,
      "loss": 0.1163,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.573278546333313,
      "rewards/margins": 6.835091590881348,
      "rewards/rejected": -7.408369064331055,
      "step": 109
    },
    {
      "epoch": 1.3037037037037038,
      "grad_norm": 22.26748210189321,
      "learning_rate": 4.2585483741369755e-07,
      "logits/chosen": -0.3594672679901123,
      "logits/rejected": -0.2787400782108307,
      "logps/chosen": -33.10090255737305,
      "logps/rejected": -65.98353576660156,
      "loss": 0.1206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1710546016693115,
      "rewards/margins": 5.390981197357178,
      "rewards/rejected": -6.562036037445068,
      "step": 110
    },
    {
      "epoch": 1.3155555555555556,
      "grad_norm": 12.864266715484298,
      "learning_rate": 4.239968742025684e-07,
      "logits/chosen": -0.1566499024629593,
      "logits/rejected": -0.02022075653076172,
      "logps/chosen": -31.641050338745117,
      "logps/rejected": -70.607421875,
      "loss": 0.0872,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5755561590194702,
      "rewards/margins": 5.668292999267578,
      "rewards/rejected": -6.243849277496338,
      "step": 111
    },
    {
      "epoch": 1.3274074074074074,
      "grad_norm": 11.527896741156843,
      "learning_rate": 4.2212008215905e-07,
      "logits/chosen": -0.18084248900413513,
      "logits/rejected": -0.06485521793365479,
      "logps/chosen": -32.15618133544922,
      "logps/rejected": -60.565914154052734,
      "loss": 0.0793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.524111270904541,
      "rewards/margins": 4.708486557006836,
      "rewards/rejected": -5.2325968742370605,
      "step": 112
    },
    {
      "epoch": 1.3392592592592591,
      "grad_norm": 18.981341764098254,
      "learning_rate": 4.2022466437761154e-07,
      "logits/chosen": 0.11151312291622162,
      "logits/rejected": 0.2410213202238083,
      "logps/chosen": -35.73467254638672,
      "logps/rejected": -61.45711135864258,
      "loss": 0.1104,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.03812497854232788,
      "rewards/margins": 3.9962849617004395,
      "rewards/rejected": -4.034409999847412,
      "step": 113
    },
    {
      "epoch": 1.3511111111111112,
      "grad_norm": 15.055020570658602,
      "learning_rate": 4.18310825968281e-07,
      "logits/chosen": -0.13380703330039978,
      "logits/rejected": -0.056154295802116394,
      "logps/chosen": -47.65923309326172,
      "logps/rejected": -68.50222778320312,
      "loss": 0.097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0497931241989136,
      "rewards/margins": 5.255577564239502,
      "rewards/rejected": -6.305370807647705,
      "step": 114
    },
    {
      "epoch": 1.362962962962963,
      "grad_norm": 15.94507937001582,
      "learning_rate": 4.1637877403444923e-07,
      "logits/chosen": -0.13782085478305817,
      "logits/rejected": -0.1237938329577446,
      "logps/chosen": -34.57268142700195,
      "logps/rejected": -60.96284103393555,
      "loss": 0.1328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2373143583536148,
      "rewards/margins": 5.874894142150879,
      "rewards/rejected": -5.637579917907715,
      "step": 115
    },
    {
      "epoch": 1.374814814814815,
      "grad_norm": 17.6499216351512,
      "learning_rate": 4.144287176504582e-07,
      "logits/chosen": -0.028891492635011673,
      "logits/rejected": 0.010559901595115662,
      "logps/chosen": -41.175785064697266,
      "logps/rejected": -56.148109436035156,
      "loss": 0.1233,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5991055965423584,
      "rewards/margins": 4.076951503753662,
      "rewards/rejected": -4.6760573387146,
      "step": 116
    },
    {
      "epoch": 1.3866666666666667,
      "grad_norm": 17.440881402981706,
      "learning_rate": 4.1246086783897713e-07,
      "logits/chosen": -0.07107866555452347,
      "logits/rejected": -0.05355262756347656,
      "logps/chosen": -29.337356567382812,
      "logps/rejected": -58.07263946533203,
      "loss": 0.107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25651735067367554,
      "rewards/margins": 5.097599983215332,
      "rewards/rejected": -4.841082572937012,
      "step": 117
    },
    {
      "epoch": 1.3985185185185185,
      "grad_norm": 20.485113734375798,
      "learning_rate": 4.104754375481664e-07,
      "logits/chosen": -0.027082689106464386,
      "logits/rejected": 0.0022195279598236084,
      "logps/chosen": -35.18750762939453,
      "logps/rejected": -54.038787841796875,
      "loss": 0.0963,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7344773411750793,
      "rewards/margins": 4.06456995010376,
      "rewards/rejected": -4.799046993255615,
      "step": 118
    },
    {
      "epoch": 1.4103703703703703,
      "grad_norm": 15.73045559795655,
      "learning_rate": 4.084726416286337e-07,
      "logits/chosen": -0.2252836376428604,
      "logits/rejected": -0.1367400735616684,
      "logps/chosen": -28.606430053710938,
      "logps/rejected": -54.020267486572266,
      "loss": 0.0872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.016864344477653503,
      "rewards/margins": 4.223577499389648,
      "rewards/rejected": -4.2067131996154785,
      "step": 119
    },
    {
      "epoch": 1.4222222222222223,
      "grad_norm": 18.987468125277015,
      "learning_rate": 4.0645269681018434e-07,
      "logits/chosen": -0.16107773780822754,
      "logits/rejected": -0.030207287520170212,
      "logps/chosen": -29.666810989379883,
      "logps/rejected": -61.85116958618164,
      "loss": 0.119,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.07655519992113113,
      "rewards/margins": 5.755571365356445,
      "rewards/rejected": -5.832127571105957,
      "step": 120
    },
    {
      "epoch": 1.434074074074074,
      "grad_norm": 14.503662362117073,
      "learning_rate": 4.044158216783684e-07,
      "logits/chosen": -0.515570878982544,
      "logits/rejected": -0.3124653100967407,
      "logps/chosen": -37.86850357055664,
      "logps/rejected": -69.43701934814453,
      "loss": 0.0985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2003718763589859,
      "rewards/margins": 7.45107364654541,
      "rewards/rejected": -7.6514458656311035,
      "step": 121
    },
    {
      "epoch": 1.445925925925926,
      "grad_norm": 18.569284512541163,
      "learning_rate": 4.0236223665082605e-07,
      "logits/chosen": -0.21073125302791595,
      "logits/rejected": -0.2305406928062439,
      "logps/chosen": -31.442550659179688,
      "logps/rejected": -55.32566833496094,
      "loss": 0.079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6064890027046204,
      "rewards/margins": 5.698796272277832,
      "rewards/rejected": -6.3052849769592285,
      "step": 122
    },
    {
      "epoch": 1.4577777777777778,
      "grad_norm": 17.336812678453057,
      "learning_rate": 4.0029216395343617e-07,
      "logits/chosen": -0.06248122453689575,
      "logits/rejected": -0.05541558563709259,
      "logps/chosen": -35.73994445800781,
      "logps/rejected": -60.30720901489258,
      "loss": 0.1032,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7068940997123718,
      "rewards/margins": 5.66333532333374,
      "rewards/rejected": -6.3702287673950195,
      "step": 123
    },
    {
      "epoch": 1.4696296296296296,
      "grad_norm": 14.094947491287853,
      "learning_rate": 3.982058275962682e-07,
      "logits/chosen": -0.21389123797416687,
      "logits/rejected": -0.16907303035259247,
      "logps/chosen": -28.326904296875,
      "logps/rejected": -56.3559455871582,
      "loss": 0.0846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10696560144424438,
      "rewards/margins": 4.861944675445557,
      "rewards/rejected": -4.968909740447998,
      "step": 124
    },
    {
      "epoch": 1.4814814814814814,
      "grad_norm": 16.97368933164184,
      "learning_rate": 3.9610345334934094e-07,
      "logits/chosen": -0.13031917810440063,
      "logits/rejected": -0.03798413649201393,
      "logps/chosen": -41.05464172363281,
      "logps/rejected": -65.29788208007812,
      "loss": 0.1205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.038674697279930115,
      "rewards/margins": 6.0994062423706055,
      "rewards/rejected": -6.060731887817383,
      "step": 125
    },
    {
      "epoch": 1.4933333333333334,
      "grad_norm": 15.284029740961785,
      "learning_rate": 3.939852687181915e-07,
      "logits/chosen": -0.14361430704593658,
      "logits/rejected": -0.10849830508232117,
      "logps/chosen": -36.10377502441406,
      "logps/rejected": -67.89326477050781,
      "loss": 0.0939,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7120703458786011,
      "rewards/margins": 5.921784400939941,
      "rewards/rejected": -6.633854389190674,
      "step": 126
    },
    {
      "epoch": 1.5051851851851852,
      "grad_norm": 18.417078359527594,
      "learning_rate": 3.9185150291925585e-07,
      "logits/chosen": -0.21707814931869507,
      "logits/rejected": -0.20940172672271729,
      "logps/chosen": -36.678348541259766,
      "logps/rejected": -59.70578384399414,
      "loss": 0.1107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0581213235855103,
      "rewards/margins": 5.090505123138428,
      "rewards/rejected": -6.14862585067749,
      "step": 127
    },
    {
      "epoch": 1.5170370370370372,
      "grad_norm": 15.713415294775897,
      "learning_rate": 3.8970238685506486e-07,
      "logits/chosen": -0.04535888880491257,
      "logits/rejected": 0.03404983878135681,
      "logps/chosen": -34.13862609863281,
      "logps/rejected": -66.47319793701172,
      "loss": 0.0976,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.735641360282898,
      "rewards/margins": 5.098552227020264,
      "rewards/rejected": -5.834194183349609,
      "step": 128
    },
    {
      "epoch": 1.528888888888889,
      "grad_norm": 15.290294429948705,
      "learning_rate": 3.8753815308925685e-07,
      "logits/chosen": -0.44447335600852966,
      "logits/rejected": -0.5022441148757935,
      "logps/chosen": -34.69385528564453,
      "logps/rejected": -63.007972717285156,
      "loss": 0.0908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.753921389579773,
      "rewards/margins": 6.32866907119751,
      "rewards/rejected": -7.082590579986572,
      "step": 129
    },
    {
      "epoch": 1.5407407407407407,
      "grad_norm": 22.7399302377654,
      "learning_rate": 3.8535903582141184e-07,
      "logits/chosen": -0.3744094669818878,
      "logits/rejected": -0.2060033231973648,
      "logps/chosen": -32.91036605834961,
      "logps/rejected": -61.743282318115234,
      "loss": 0.1268,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4923054873943329,
      "rewards/margins": 5.070975303649902,
      "rewards/rejected": -5.5632805824279785,
      "step": 130
    },
    {
      "epoch": 1.5525925925925925,
      "grad_norm": 19.47533309428857,
      "learning_rate": 3.8316527086170727e-07,
      "logits/chosen": -0.1633462905883789,
      "logits/rejected": -0.07288794964551926,
      "logps/chosen": -36.646484375,
      "logps/rejected": -59.22307205200195,
      "loss": 0.125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2637479305267334,
      "rewards/margins": 5.560683727264404,
      "rewards/rejected": -5.824431419372559,
      "step": 131
    },
    {
      "epoch": 1.5644444444444443,
      "grad_norm": 18.594859296902843,
      "learning_rate": 3.809570956054003e-07,
      "logits/chosen": -0.5365747213363647,
      "logits/rejected": -0.39433979988098145,
      "logps/chosen": -31.459264755249023,
      "logps/rejected": -62.012969970703125,
      "loss": 0.1097,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.048090934753418,
      "rewards/margins": 5.831287860870361,
      "rewards/rejected": -6.879378318786621,
      "step": 132
    },
    {
      "epoch": 1.5762962962962963,
      "grad_norm": 14.282275106755108,
      "learning_rate": 3.787347490071389e-07,
      "logits/chosen": -0.20027217268943787,
      "logits/rejected": -0.11141454428434372,
      "logps/chosen": -39.01911163330078,
      "logps/rejected": -64.89835357666016,
      "loss": 0.0991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3214702308177948,
      "rewards/margins": 5.419382095336914,
      "rewards/rejected": -5.740852355957031,
      "step": 133
    },
    {
      "epoch": 1.5881481481481483,
      "grad_norm": 13.42088885786826,
      "learning_rate": 3.764984715551031e-07,
      "logits/chosen": -0.12170754373073578,
      "logits/rejected": -0.046029090881347656,
      "logps/chosen": -29.234134674072266,
      "logps/rejected": -60.45512771606445,
      "loss": 0.0895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1851249635219574,
      "rewards/margins": 6.193130970001221,
      "rewards/rejected": -6.378255844116211,
      "step": 134
    },
    {
      "epoch": 1.6,
      "grad_norm": 17.391251462773063,
      "learning_rate": 3.7424850524498113e-07,
      "logits/chosen": -0.18073627352714539,
      "logits/rejected": -0.05371435731649399,
      "logps/chosen": -35.178985595703125,
      "logps/rejected": -62.41261672973633,
      "loss": 0.112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.027589425444602966,
      "rewards/margins": 5.411984443664551,
      "rewards/rejected": -5.384395599365234,
      "step": 135
    },
    {
      "epoch": 1.6118518518518519,
      "grad_norm": 13.216285367583016,
      "learning_rate": 3.7198509355378207e-07,
      "logits/chosen": -0.3801528811454773,
      "logits/rejected": -0.3222590982913971,
      "logps/chosen": -40.775054931640625,
      "logps/rejected": -52.93413543701172,
      "loss": 0.1169,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5116928815841675,
      "rewards/margins": 3.9808902740478516,
      "rewards/rejected": -5.49258279800415,
      "step": 136
    },
    {
      "epoch": 1.6237037037037036,
      "grad_norm": 20.45643460429083,
      "learning_rate": 3.6970848141348855e-07,
      "logits/chosen": -0.17568367719650269,
      "logits/rejected": -0.11519981920719147,
      "logps/chosen": -39.35060501098633,
      "logps/rejected": -59.42844009399414,
      "loss": 0.1259,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02011704444885254,
      "rewards/margins": 5.978552341461182,
      "rewards/rejected": -5.998669624328613,
      "step": 137
    },
    {
      "epoch": 1.6355555555555554,
      "grad_norm": 20.16029126070904,
      "learning_rate": 3.6741891518455146e-07,
      "logits/chosen": -0.16909295320510864,
      "logits/rejected": -0.11791606992483139,
      "logps/chosen": -39.2324333190918,
      "logps/rejected": -67.33000183105469,
      "loss": 0.0818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9399681091308594,
      "rewards/margins": 5.435766220092773,
      "rewards/rejected": -6.375733375549316,
      "step": 138
    },
    {
      "epoch": 1.6474074074074074,
      "grad_norm": 16.83558679733193,
      "learning_rate": 3.6511664262923094e-07,
      "logits/chosen": -0.2512515187263489,
      "logits/rejected": -0.12882237136363983,
      "logps/chosen": -27.65873908996582,
      "logps/rejected": -61.08441925048828,
      "loss": 0.0919,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.27784237265586853,
      "rewards/margins": 6.476930141448975,
      "rewards/rejected": -6.754772663116455,
      "step": 139
    },
    {
      "epoch": 1.6592592592592592,
      "grad_norm": 12.155079766899926,
      "learning_rate": 3.6280191288478435e-07,
      "logits/chosen": -0.15503238141536713,
      "logits/rejected": -0.06224162131547928,
      "logps/chosen": -34.50273895263672,
      "logps/rejected": -62.98163604736328,
      "loss": 0.0725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17935335636138916,
      "rewards/margins": 5.786401271820068,
      "rewards/rejected": -5.965755462646484,
      "step": 140
    },
    {
      "epoch": 1.6711111111111112,
      "grad_norm": 14.838007532934355,
      "learning_rate": 3.604749764365069e-07,
      "logits/chosen": -0.2061130404472351,
      "logits/rejected": -0.11659687012434006,
      "logps/chosen": -27.322792053222656,
      "logps/rejected": -62.887542724609375,
      "loss": 0.0849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2183775156736374,
      "rewards/margins": 7.025249481201172,
      "rewards/rejected": -7.243628025054932,
      "step": 141
    },
    {
      "epoch": 1.682962962962963,
      "grad_norm": 18.816292977620716,
      "learning_rate": 3.5813608509062526e-07,
      "logits/chosen": -0.21707522869110107,
      "logits/rejected": -0.09010873734951019,
      "logps/chosen": -36.751190185546875,
      "logps/rejected": -75.8062744140625,
      "loss": 0.1009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9384999871253967,
      "rewards/margins": 6.734908580780029,
      "rewards/rejected": -7.673408508300781,
      "step": 142
    },
    {
      "epoch": 1.6948148148148148,
      "grad_norm": 19.8821437700751,
      "learning_rate": 3.557854919470491e-07,
      "logits/chosen": -0.27580782771110535,
      "logits/rejected": -0.22370710968971252,
      "logps/chosen": -44.46840286254883,
      "logps/rejected": -56.556922912597656,
      "loss": 0.1165,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3054481744766235,
      "rewards/margins": 3.6606221199035645,
      "rewards/rejected": -4.966070175170898,
      "step": 143
    },
    {
      "epoch": 1.7066666666666666,
      "grad_norm": 16.302116776290895,
      "learning_rate": 3.5342345137198206e-07,
      "logits/chosen": -0.12199485301971436,
      "logits/rejected": -0.08353496342897415,
      "logps/chosen": -38.30879211425781,
      "logps/rejected": -51.09149932861328,
      "loss": 0.112,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.1275089532136917,
      "rewards/margins": 3.7598049640655518,
      "rewards/rejected": -3.8873140811920166,
      "step": 144
    },
    {
      "epoch": 1.7185185185185186,
      "grad_norm": 15.133588204814354,
      "learning_rate": 3.510502189703954e-07,
      "logits/chosen": -0.10772836208343506,
      "logits/rejected": -0.06885837763547897,
      "logps/chosen": -40.77737045288086,
      "logps/rejected": -69.48592376708984,
      "loss": 0.0693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8564466834068298,
      "rewards/margins": 8.315703392028809,
      "rewards/rejected": -9.172150611877441,
      "step": 145
    },
    {
      "epoch": 1.7303703703703703,
      "grad_norm": 19.362732647667297,
      "learning_rate": 3.486660515583691e-07,
      "logits/chosen": -0.2726586163043976,
      "logits/rejected": -0.21814075112342834,
      "logps/chosen": -30.717233657836914,
      "logps/rejected": -65.25225830078125,
      "loss": 0.0885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21597516536712646,
      "rewards/margins": 6.636316776275635,
      "rewards/rejected": -6.852292060852051,
      "step": 146
    },
    {
      "epoch": 1.7422222222222223,
      "grad_norm": 11.230644499357242,
      "learning_rate": 3.4627120713529983e-07,
      "logits/chosen": -0.2115684449672699,
      "logits/rejected": -0.11108352243900299,
      "logps/chosen": -34.891475677490234,
      "logps/rejected": -77.10374450683594,
      "loss": 0.0803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.222525954246521,
      "rewards/margins": 9.00995922088623,
      "rewards/rejected": -10.232484817504883,
      "step": 147
    },
    {
      "epoch": 1.7540740740740741,
      "grad_norm": 13.137309578079025,
      "learning_rate": 3.438659448559825e-07,
      "logits/chosen": -0.13276290893554688,
      "logits/rejected": -0.10878665745258331,
      "logps/chosen": -37.001060485839844,
      "logps/rejected": -73.28951263427734,
      "loss": 0.067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.598874568939209,
      "rewards/margins": 7.266829013824463,
      "rewards/rejected": -7.865704536437988,
      "step": 148
    },
    {
      "epoch": 1.765925925925926,
      "grad_norm": 21.677182436717583,
      "learning_rate": 3.414505250025659e-07,
      "logits/chosen": -0.04767221957445145,
      "logits/rejected": 0.07465275377035141,
      "logps/chosen": -40.81106185913086,
      "logps/rejected": -67.47488403320312,
      "loss": 0.0942,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7800440788269043,
      "rewards/margins": 5.173450469970703,
      "rewards/rejected": -5.953495025634766,
      "step": 149
    },
    {
      "epoch": 1.7777777777777777,
      "grad_norm": 16.148540981867313,
      "learning_rate": 3.390252089563867e-07,
      "logits/chosen": -0.18858963251113892,
      "logits/rejected": -0.20340172946453094,
      "logps/chosen": -36.109954833984375,
      "logps/rejected": -55.27710723876953,
      "loss": 0.0796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0519292950630188,
      "rewards/margins": 5.729560852050781,
      "rewards/rejected": -5.677631855010986,
      "step": 150
    },
    {
      "epoch": 1.7896296296296297,
      "grad_norm": 24.312535239343674,
      "learning_rate": 3.3659025916968475e-07,
      "logits/chosen": -0.2818453013896942,
      "logits/rejected": -0.19897550344467163,
      "logps/chosen": -38.93151092529297,
      "logps/rejected": -76.9036636352539,
      "loss": 0.1537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5907295346260071,
      "rewards/margins": 6.914373397827148,
      "rewards/rejected": -7.505102634429932,
      "step": 151
    },
    {
      "epoch": 1.8014814814814815,
      "grad_norm": 20.831076322049565,
      "learning_rate": 3.3414593913720155e-07,
      "logits/chosen": -0.22526244819164276,
      "logits/rejected": -0.18128839135169983,
      "logps/chosen": -34.0900764465332,
      "logps/rejected": -55.26824951171875,
      "loss": 0.126,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0966421514749527,
      "rewards/margins": 5.949459075927734,
      "rewards/rejected": -6.046100616455078,
      "step": 152
    },
    {
      "epoch": 1.8133333333333335,
      "grad_norm": 11.740893074967538,
      "learning_rate": 3.3169251336766697e-07,
      "logits/chosen": -0.1995951235294342,
      "logits/rejected": -0.08064538240432739,
      "logps/chosen": -34.26948165893555,
      "logps/rejected": -62.14799499511719,
      "loss": 0.0776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.565490484237671,
      "rewards/margins": 5.796334266662598,
      "rewards/rejected": -7.361824989318848,
      "step": 153
    },
    {
      "epoch": 1.8251851851851852,
      "grad_norm": 16.192262072429642,
      "learning_rate": 3.2923024735517567e-07,
      "logits/chosen": -0.3225496709346771,
      "logits/rejected": -0.23174233734607697,
      "logps/chosen": -31.906171798706055,
      "logps/rejected": -58.93853759765625,
      "loss": 0.0956,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.29122716188430786,
      "rewards/margins": 5.078674793243408,
      "rewards/rejected": -5.369902610778809,
      "step": 154
    },
    {
      "epoch": 1.837037037037037,
      "grad_norm": 20.0031978017647,
      "learning_rate": 3.2675940755045713e-07,
      "logits/chosen": 0.008214278146624565,
      "logits/rejected": 0.14055991172790527,
      "logps/chosen": -46.526527404785156,
      "logps/rejected": -81.31307983398438,
      "loss": 0.1503,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6333274245262146,
      "rewards/margins": 6.6069817543029785,
      "rewards/rejected": -7.240309238433838,
      "step": 155
    },
    {
      "epoch": 1.8488888888888888,
      "grad_norm": 21.076223385192634,
      "learning_rate": 3.242802613320418e-07,
      "logits/chosen": 0.0031320489943027496,
      "logits/rejected": 0.012668165378272533,
      "logps/chosen": -37.56488037109375,
      "logps/rejected": -64.01717376708984,
      "loss": 0.1549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7534648180007935,
      "rewards/margins": 6.474049091339111,
      "rewards/rejected": -7.227513313293457,
      "step": 156
    },
    {
      "epoch": 1.8607407407407406,
      "grad_norm": 16.652570591848395,
      "learning_rate": 3.217930769773275e-07,
      "logits/chosen": -0.35275697708129883,
      "logits/rejected": -0.24555784463882446,
      "logps/chosen": -33.54517364501953,
      "logps/rejected": -62.51717758178711,
      "loss": 0.0959,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.42744529247283936,
      "rewards/margins": 6.722561836242676,
      "rewards/rejected": -7.150007247924805,
      "step": 157
    },
    {
      "epoch": 1.8725925925925926,
      "grad_norm": 12.597802388134623,
      "learning_rate": 3.1929812363354764e-07,
      "logits/chosen": -0.2830018103122711,
      "logits/rejected": -0.1911703646183014,
      "logps/chosen": -34.85630416870117,
      "logps/rejected": -65.27281188964844,
      "loss": 0.0536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6973313689231873,
      "rewards/margins": 5.912741184234619,
      "rewards/rejected": -6.610072612762451,
      "step": 158
    },
    {
      "epoch": 1.8844444444444446,
      "grad_norm": 12.10851684482882,
      "learning_rate": 3.167956712886463e-07,
      "logits/chosen": -0.11961568146944046,
      "logits/rejected": -0.07325749099254608,
      "logps/chosen": -39.96379470825195,
      "logps/rejected": -55.56465148925781,
      "loss": 0.061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3007912635803223,
      "rewards/margins": 4.756231307983398,
      "rewards/rejected": -6.0570220947265625,
      "step": 159
    },
    {
      "epoch": 1.8962962962962964,
      "grad_norm": 20.233134180855856,
      "learning_rate": 3.142859907420615e-07,
      "logits/chosen": -0.10496459901332855,
      "logits/rejected": 0.02405383251607418,
      "logps/chosen": -33.706703186035156,
      "logps/rejected": -68.788818359375,
      "loss": 0.1011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.46070706844329834,
      "rewards/margins": 5.680697917938232,
      "rewards/rejected": -6.14140510559082,
      "step": 160
    },
    {
      "epoch": 1.9081481481481481,
      "grad_norm": 19.119673878649753,
      "learning_rate": 3.117693535754213e-07,
      "logits/chosen": -0.10953935980796814,
      "logits/rejected": -0.043600670993328094,
      "logps/chosen": -32.68844985961914,
      "logps/rejected": -67.97357177734375,
      "loss": 0.098,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.133280411362648,
      "rewards/margins": 7.29306173324585,
      "rewards/rejected": -7.159781455993652,
      "step": 161
    },
    {
      "epoch": 1.92,
      "grad_norm": 20.235536301916813,
      "learning_rate": 3.092460321231547e-07,
      "logits/chosen": -0.25959259271621704,
      "logits/rejected": -0.20939543843269348,
      "logps/chosen": -35.41835021972656,
      "logps/rejected": -67.97672271728516,
      "loss": 0.1272,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.018587589263916,
      "rewards/margins": 7.818422317504883,
      "rewards/rejected": -8.837010383605957,
      "step": 162
    },
    {
      "epoch": 1.9318518518518517,
      "grad_norm": 13.909230300778022,
      "learning_rate": 3.0671629944302164e-07,
      "logits/chosen": -0.12026870250701904,
      "logits/rejected": -0.1089366227388382,
      "logps/chosen": -36.70647048950195,
      "logps/rejected": -53.47910690307617,
      "loss": 0.0742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5273434519767761,
      "rewards/margins": 5.643020153045654,
      "rewards/rejected": -6.170363903045654,
      "step": 163
    },
    {
      "epoch": 1.9437037037037037,
      "grad_norm": 12.864589613039078,
      "learning_rate": 3.0418042928656415e-07,
      "logits/chosen": -0.19225530326366425,
      "logits/rejected": -0.10623307526111603,
      "logps/chosen": -30.59122085571289,
      "logps/rejected": -59.536502838134766,
      "loss": 0.0836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13015054166316986,
      "rewards/margins": 5.6972761154174805,
      "rewards/rejected": -5.827426910400391,
      "step": 164
    },
    {
      "epoch": 1.9555555555555557,
      "grad_norm": 15.975537356292229,
      "learning_rate": 3.016386960694827e-07,
      "logits/chosen": -0.41196244955062866,
      "logits/rejected": -0.28604307770729065,
      "logps/chosen": -39.74413299560547,
      "logps/rejected": -67.16585540771484,
      "loss": 0.1075,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3397538661956787,
      "rewards/margins": 5.61893892288208,
      "rewards/rejected": -6.958693027496338,
      "step": 165
    },
    {
      "epoch": 1.9674074074074075,
      "grad_norm": 32.234428734857076,
      "learning_rate": 2.990913748419411e-07,
      "logits/chosen": 0.07510136812925339,
      "logits/rejected": 0.11186552792787552,
      "logps/chosen": -41.38081741333008,
      "logps/rejected": -67.78083038330078,
      "loss": 0.1778,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3183833360671997,
      "rewards/margins": 5.662228107452393,
      "rewards/rejected": -5.980611801147461,
      "step": 166
    },
    {
      "epoch": 1.9792592592592593,
      "grad_norm": 19.969450441352706,
      "learning_rate": 2.9653874125880167e-07,
      "logits/chosen": -0.17725233733654022,
      "logits/rejected": -0.10895150154829025,
      "logps/chosen": -36.67229461669922,
      "logps/rejected": -61.394744873046875,
      "loss": 0.1436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03877316415309906,
      "rewards/margins": 5.38054895401001,
      "rewards/rejected": -5.341775894165039,
      "step": 167
    },
    {
      "epoch": 1.991111111111111,
      "grad_norm": 11.405884683324148,
      "learning_rate": 2.9398107154979634e-07,
      "logits/chosen": -0.21582955121994019,
      "logits/rejected": -0.1547984778881073,
      "logps/chosen": -33.66050338745117,
      "logps/rejected": -71.53202056884766,
      "loss": 0.0568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2750725746154785,
      "rewards/margins": 6.011247158050537,
      "rewards/rejected": -6.286320209503174,
      "step": 168
    },
    {
      "epoch": 2.002962962962963,
      "grad_norm": 10.684152961615103,
      "learning_rate": 2.9141864248963427e-07,
      "logits/chosen": -0.43596649169921875,
      "logits/rejected": -0.3842291235923767,
      "logps/chosen": -39.145042419433594,
      "logps/rejected": -52.33580780029297,
      "loss": 0.0686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2999729812145233,
      "rewards/margins": 5.18059778213501,
      "rewards/rejected": -5.4805707931518555,
      "step": 169
    },
    {
      "epoch": 2.0148148148148146,
      "grad_norm": 7.120772046885603,
      "learning_rate": 2.8885173136805125e-07,
      "logits/chosen": -0.0826089009642601,
      "logits/rejected": 0.052907198667526245,
      "logps/chosen": -34.04817199707031,
      "logps/rejected": -73.21731567382812,
      "loss": 0.0483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0369320884346962,
      "rewards/margins": 5.923883438110352,
      "rewards/rejected": -5.960815906524658,
      "step": 170
    },
    {
      "epoch": 2.026666666666667,
      "grad_norm": 6.919197934532035,
      "learning_rate": 2.862806159598032e-07,
      "logits/chosen": -0.45804017782211304,
      "logits/rejected": -0.408170610666275,
      "logps/chosen": -35.05866241455078,
      "logps/rejected": -58.95463562011719,
      "loss": 0.0433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28513020277023315,
      "rewards/margins": 6.706875324249268,
      "rewards/rejected": -6.421745300292969,
      "step": 171
    },
    {
      "epoch": 2.0385185185185186,
      "grad_norm": 7.6734571457046075,
      "learning_rate": 2.837055744946072e-07,
      "logits/chosen": -0.20093482732772827,
      "logits/rejected": -0.15153169631958008,
      "logps/chosen": -26.53860092163086,
      "logps/rejected": -60.75783157348633,
      "loss": 0.0522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1763535887002945,
      "rewards/margins": 6.684775352478027,
      "rewards/rejected": -6.508421421051025,
      "step": 172
    },
    {
      "epoch": 2.0503703703703704,
      "grad_norm": 7.089127360949598,
      "learning_rate": 2.811268856270332e-07,
      "logits/chosen": -0.19816571474075317,
      "logits/rejected": -0.17036175727844238,
      "logps/chosen": -30.929105758666992,
      "logps/rejected": -63.191619873046875,
      "loss": 0.0436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12159377336502075,
      "rewards/margins": 6.226848602294922,
      "rewards/rejected": -6.348442077636719,
      "step": 173
    },
    {
      "epoch": 2.062222222222222,
      "grad_norm": 6.879444744601849,
      "learning_rate": 2.7854482840634965e-07,
      "logits/chosen": -0.3685060143470764,
      "logits/rejected": -0.2508692145347595,
      "logps/chosen": -30.003223419189453,
      "logps/rejected": -66.00552368164062,
      "loss": 0.0503,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6152498126029968,
      "rewards/margins": 7.741688251495361,
      "rewards/rejected": -8.356938362121582,
      "step": 174
    },
    {
      "epoch": 2.074074074074074,
      "grad_norm": 7.35254551338382,
      "learning_rate": 2.759596822463267e-07,
      "logits/chosen": -0.23846019804477692,
      "logits/rejected": -0.21598272025585175,
      "logps/chosen": -35.864341735839844,
      "logps/rejected": -60.58774948120117,
      "loss": 0.0493,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5300172567367554,
      "rewards/margins": 6.528465747833252,
      "rewards/rejected": -7.0584821701049805,
      "step": 175
    },
    {
      "epoch": 2.0859259259259257,
      "grad_norm": 6.0224009777317535,
      "learning_rate": 2.73371726895e-07,
      "logits/chosen": -0.4624573588371277,
      "logits/rejected": -0.3582268953323364,
      "logps/chosen": -38.6595573425293,
      "logps/rejected": -68.8594970703125,
      "loss": 0.035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1498335599899292,
      "rewards/margins": 6.414737224578857,
      "rewards/rejected": -6.564570903778076,
      "step": 176
    },
    {
      "epoch": 2.097777777777778,
      "grad_norm": 5.373444495677693,
      "learning_rate": 2.7078124240439793e-07,
      "logits/chosen": -0.293597012758255,
      "logits/rejected": -0.18846404552459717,
      "logps/chosen": -37.71804428100586,
      "logps/rejected": -79.11727905273438,
      "loss": 0.0295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9366405606269836,
      "rewards/margins": 8.907712936401367,
      "rewards/rejected": -9.844353675842285,
      "step": 177
    },
    {
      "epoch": 2.1096296296296297,
      "grad_norm": 6.426554871574788,
      "learning_rate": 2.68188509100236e-07,
      "logits/chosen": -0.07860371470451355,
      "logits/rejected": -0.032592758536338806,
      "logps/chosen": -36.26288604736328,
      "logps/rejected": -67.72529602050781,
      "loss": 0.041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021683871746063232,
      "rewards/margins": 6.0264506340026855,
      "rewards/rejected": -6.048134803771973,
      "step": 178
    },
    {
      "epoch": 2.1214814814814815,
      "grad_norm": 7.4465908422134435,
      "learning_rate": 2.6559380755158206e-07,
      "logits/chosen": -0.2125643938779831,
      "logits/rejected": -0.11938470602035522,
      "logps/chosen": -41.93673324584961,
      "logps/rejected": -67.91219329833984,
      "loss": 0.0526,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.314428448677063,
      "rewards/margins": 6.577666759490967,
      "rewards/rejected": -6.892095565795898,
      "step": 179
    },
    {
      "epoch": 2.1333333333333333,
      "grad_norm": 8.207906364980415,
      "learning_rate": 2.629974185404951e-07,
      "logits/chosen": -0.19172216951847076,
      "logits/rejected": -0.09138239920139313,
      "logps/chosen": -33.569427490234375,
      "logps/rejected": -83.35604858398438,
      "loss": 0.0522,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.1560230553150177,
      "rewards/margins": 6.664412975311279,
      "rewards/rejected": -6.820435523986816,
      "step": 180
    },
    {
      "epoch": 2.145185185185185,
      "grad_norm": 7.056836784079578,
      "learning_rate": 2.603996230316402e-07,
      "logits/chosen": 0.04577064514160156,
      "logits/rejected": 0.027345050126314163,
      "logps/chosen": -28.81571388244629,
      "logps/rejected": -50.763343811035156,
      "loss": 0.0463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12452198565006256,
      "rewards/margins": 5.144538879394531,
      "rewards/rejected": -5.269061088562012,
      "step": 181
    },
    {
      "epoch": 2.157037037037037,
      "grad_norm": 11.894887100183427,
      "learning_rate": 2.5780070214188474e-07,
      "logits/chosen": -0.2564762532711029,
      "logits/rejected": -0.17662659287452698,
      "logps/chosen": -44.12029266357422,
      "logps/rejected": -67.84626007080078,
      "loss": 0.067,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7224655151367188,
      "rewards/margins": 6.115478038787842,
      "rewards/rejected": -6.837944030761719,
      "step": 182
    },
    {
      "epoch": 2.168888888888889,
      "grad_norm": 7.773592692412279,
      "learning_rate": 2.552009371098778e-07,
      "logits/chosen": -0.22470326721668243,
      "logits/rejected": -0.16598555445671082,
      "logps/chosen": -40.50588607788086,
      "logps/rejected": -66.48469543457031,
      "loss": 0.0511,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6875913143157959,
      "rewards/margins": 6.013528823852539,
      "rewards/rejected": -6.7011213302612305,
      "step": 183
    },
    {
      "epoch": 2.180740740740741,
      "grad_norm": 6.760102453494981,
      "learning_rate": 2.5260060926561604e-07,
      "logits/chosen": -0.10662063956260681,
      "logits/rejected": 0.02468992955982685,
      "logps/chosen": -30.439607620239258,
      "logps/rejected": -68.47404479980469,
      "loss": 0.042,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.1491369903087616,
      "rewards/margins": 7.288580894470215,
      "rewards/rejected": -7.437718391418457,
      "step": 184
    },
    {
      "epoch": 2.1925925925925926,
      "grad_norm": 8.50819275884591,
      "learning_rate": 2.5e-07,
      "logits/chosen": -0.314007043838501,
      "logits/rejected": -0.2440987378358841,
      "logps/chosen": -40.95098114013672,
      "logps/rejected": -58.714630126953125,
      "loss": 0.061,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0700465440750122,
      "rewards/margins": 4.418706893920898,
      "rewards/rejected": -5.488753795623779,
      "step": 185
    },
    {
      "epoch": 2.2044444444444444,
      "grad_norm": 7.253010502441156,
      "learning_rate": 2.4739939073438393e-07,
      "logits/chosen": -0.3739926815032959,
      "logits/rejected": -0.29570120573043823,
      "logps/chosen": -46.3231201171875,
      "logps/rejected": -73.8995361328125,
      "loss": 0.0448,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3167559504508972,
      "rewards/margins": 5.942593574523926,
      "rewards/rejected": -6.259350299835205,
      "step": 186
    },
    {
      "epoch": 2.216296296296296,
      "grad_norm": 10.218903811635839,
      "learning_rate": 2.4479906289012216e-07,
      "logits/chosen": -0.23329459130764008,
      "logits/rejected": -0.16335441172122955,
      "logps/chosen": -41.237979888916016,
      "logps/rejected": -61.02803039550781,
      "loss": 0.0585,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.29829394817352295,
      "rewards/margins": 6.824221611022949,
      "rewards/rejected": -6.525927543640137,
      "step": 187
    },
    {
      "epoch": 2.228148148148148,
      "grad_norm": 5.706051711082172,
      "learning_rate": 2.421992978581152e-07,
      "logits/chosen": -0.20098957419395447,
      "logits/rejected": -0.1625077724456787,
      "logps/chosen": -33.27192306518555,
      "logps/rejected": -63.34418869018555,
      "loss": 0.0339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16527414321899414,
      "rewards/margins": 7.369152069091797,
      "rewards/rejected": -7.534427165985107,
      "step": 188
    },
    {
      "epoch": 2.24,
      "grad_norm": 5.574632829097292,
      "learning_rate": 2.3960037696835987e-07,
      "logits/chosen": -0.15070542693138123,
      "logits/rejected": -0.09224209934473038,
      "logps/chosen": -36.12839126586914,
      "logps/rejected": -78.54652404785156,
      "loss": 0.0296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1367915868759155,
      "rewards/margins": 9.016799926757812,
      "rewards/rejected": -10.15359115600586,
      "step": 189
    },
    {
      "epoch": 2.251851851851852,
      "grad_norm": 9.222402573182322,
      "learning_rate": 2.3700258145950493e-07,
      "logits/chosen": -0.16236115992069244,
      "logits/rejected": -0.21426168084144592,
      "logps/chosen": -32.171287536621094,
      "logps/rejected": -65.74127960205078,
      "loss": 0.0555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.31431764364242554,
      "rewards/margins": 7.022213459014893,
      "rewards/rejected": -7.336531162261963,
      "step": 190
    },
    {
      "epoch": 2.2637037037037038,
      "grad_norm": 7.127001970520486,
      "learning_rate": 2.3440619244841794e-07,
      "logits/chosen": -0.2093840390443802,
      "logits/rejected": -0.20409150421619415,
      "logps/chosen": -31.866199493408203,
      "logps/rejected": -57.632957458496094,
      "loss": 0.0419,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5825138092041016,
      "rewards/margins": 5.033719539642334,
      "rewards/rejected": -5.616233825683594,
      "step": 191
    },
    {
      "epoch": 2.2755555555555556,
      "grad_norm": 7.894992729942256,
      "learning_rate": 2.3181149089976404e-07,
      "logits/chosen": -0.07013247907161713,
      "logits/rejected": -0.04877481237053871,
      "logps/chosen": -33.34114456176758,
      "logps/rejected": -54.995853424072266,
      "loss": 0.0496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.006869621574878693,
      "rewards/margins": 6.3635053634643555,
      "rewards/rejected": -6.356635570526123,
      "step": 192
    },
    {
      "epoch": 2.2874074074074073,
      "grad_norm": 11.705188167009563,
      "learning_rate": 2.2921875759560207e-07,
      "logits/chosen": -0.13075098395347595,
      "logits/rejected": -0.20485468208789825,
      "logps/chosen": -47.33964157104492,
      "logps/rejected": -67.68054962158203,
      "loss": 0.0641,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.68022620677948,
      "rewards/margins": 6.593554973602295,
      "rewards/rejected": -7.273780822753906,
      "step": 193
    },
    {
      "epoch": 2.299259259259259,
      "grad_norm": 7.134458814614045,
      "learning_rate": 2.2662827310499995e-07,
      "logits/chosen": -0.2494003027677536,
      "logits/rejected": -0.20588865876197815,
      "logps/chosen": -36.393733978271484,
      "logps/rejected": -58.504547119140625,
      "loss": 0.042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11634793132543564,
      "rewards/margins": 6.01495885848999,
      "rewards/rejected": -6.1313066482543945,
      "step": 194
    },
    {
      "epoch": 2.311111111111111,
      "grad_norm": 12.206709819986875,
      "learning_rate": 2.2404031775367332e-07,
      "logits/chosen": -0.29956668615341187,
      "logits/rejected": -0.21865390241146088,
      "logps/chosen": -32.450687408447266,
      "logps/rejected": -69.80120086669922,
      "loss": 0.0409,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12443738430738449,
      "rewards/margins": 8.257219314575195,
      "rewards/rejected": -8.132781982421875,
      "step": 195
    },
    {
      "epoch": 2.322962962962963,
      "grad_norm": 6.1999786310250675,
      "learning_rate": 2.2145517159365043e-07,
      "logits/chosen": -0.5200955271720886,
      "logits/rejected": -0.4296617805957794,
      "logps/chosen": -36.10881805419922,
      "logps/rejected": -63.84122848510742,
      "loss": 0.0278,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5274075865745544,
      "rewards/margins": 6.768378257751465,
      "rewards/rejected": -7.295785903930664,
      "step": 196
    },
    {
      "epoch": 2.334814814814815,
      "grad_norm": 12.570655654481419,
      "learning_rate": 2.1887311437296684e-07,
      "logits/chosen": -0.31551796197891235,
      "logits/rejected": -0.281019389629364,
      "logps/chosen": -29.494091033935547,
      "logps/rejected": -46.46226501464844,
      "loss": 0.0761,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2344491183757782,
      "rewards/margins": 4.557922840118408,
      "rewards/rejected": -4.3234734535217285,
      "step": 197
    },
    {
      "epoch": 2.3466666666666667,
      "grad_norm": 7.628823812638006,
      "learning_rate": 2.162944255053928e-07,
      "logits/chosen": -0.31029197573661804,
      "logits/rejected": -0.24161145091056824,
      "logps/chosen": -29.65079689025879,
      "logps/rejected": -57.47043228149414,
      "loss": 0.0435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.04058557003736496,
      "rewards/margins": 6.636114597320557,
      "rewards/rejected": -6.595529079437256,
      "step": 198
    },
    {
      "epoch": 2.3585185185185185,
      "grad_norm": 8.85295248102532,
      "learning_rate": 2.137193840401968e-07,
      "logits/chosen": -0.3979605734348297,
      "logits/rejected": -0.32846879959106445,
      "logps/chosen": -34.3480224609375,
      "logps/rejected": -58.354896545410156,
      "loss": 0.0533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3420531153678894,
      "rewards/margins": 5.625160217285156,
      "rewards/rejected": -5.283106803894043,
      "step": 199
    },
    {
      "epoch": 2.3703703703703702,
      "grad_norm": 6.562343692053369,
      "learning_rate": 2.1114826863194878e-07,
      "logits/chosen": -0.25852352380752563,
      "logits/rejected": -0.17369653284549713,
      "logps/chosen": -35.14963912963867,
      "logps/rejected": -68.84390258789062,
      "loss": 0.0393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06890328228473663,
      "rewards/margins": 7.833587169647217,
      "rewards/rejected": -7.902491092681885,
      "step": 200
    },
    {
      "epoch": 2.3822222222222225,
      "grad_norm": 7.374090556871228,
      "learning_rate": 2.0858135751036568e-07,
      "logits/chosen": -0.347494900226593,
      "logits/rejected": -0.366567462682724,
      "logps/chosen": -48.93431091308594,
      "logps/rejected": -69.73056030273438,
      "loss": 0.033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1343318223953247,
      "rewards/margins": 7.379057884216309,
      "rewards/rejected": -8.51339054107666,
      "step": 201
    },
    {
      "epoch": 2.3940740740740742,
      "grad_norm": 7.264335195847015,
      "learning_rate": 2.060189284502037e-07,
      "logits/chosen": -0.28864482045173645,
      "logits/rejected": -0.14989601075649261,
      "logps/chosen": -36.92799377441406,
      "logps/rejected": -68.40786743164062,
      "loss": 0.039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6765543818473816,
      "rewards/margins": 6.945480823516846,
      "rewards/rejected": -7.622035503387451,
      "step": 202
    },
    {
      "epoch": 2.405925925925926,
      "grad_norm": 7.5132642347332155,
      "learning_rate": 2.0346125874119838e-07,
      "logits/chosen": -0.35054826736450195,
      "logits/rejected": -0.35159796476364136,
      "logps/chosen": -35.19864273071289,
      "logps/rejected": -67.63153839111328,
      "loss": 0.0423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2543597221374512,
      "rewards/margins": 7.7994771003723145,
      "rewards/rejected": -9.053837776184082,
      "step": 203
    },
    {
      "epoch": 2.417777777777778,
      "grad_norm": 10.04344534438614,
      "learning_rate": 2.0090862515805895e-07,
      "logits/chosen": -0.13007110357284546,
      "logits/rejected": -0.1110701858997345,
      "logps/chosen": -43.942832946777344,
      "logps/rejected": -60.823211669921875,
      "loss": 0.0693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1511293649673462,
      "rewards/margins": 6.567122936248779,
      "rewards/rejected": -7.718252658843994,
      "step": 204
    },
    {
      "epoch": 2.4296296296296296,
      "grad_norm": 9.49554839896552,
      "learning_rate": 1.983613039305173e-07,
      "logits/chosen": -0.4052940905094147,
      "logits/rejected": -0.23975247144699097,
      "logps/chosen": -27.93557357788086,
      "logps/rejected": -65.16735076904297,
      "loss": 0.0571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3964582085609436,
      "rewards/margins": 8.21355152130127,
      "rewards/rejected": -8.610010147094727,
      "step": 205
    },
    {
      "epoch": 2.4414814814814814,
      "grad_norm": 7.396028421519832,
      "learning_rate": 1.9581957071343588e-07,
      "logits/chosen": -0.3185134828090668,
      "logits/rejected": -0.1753259003162384,
      "logps/chosen": -45.08576202392578,
      "logps/rejected": -88.92870330810547,
      "loss": 0.047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9330806732177734,
      "rewards/margins": 8.258587837219238,
      "rewards/rejected": -10.191668510437012,
      "step": 206
    },
    {
      "epoch": 2.453333333333333,
      "grad_norm": 6.455933266927346,
      "learning_rate": 1.9328370055697832e-07,
      "logits/chosen": -0.20377328991889954,
      "logits/rejected": -0.09884392470121384,
      "logps/chosen": -31.141347885131836,
      "logps/rejected": -68.2867431640625,
      "loss": 0.032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21240955591201782,
      "rewards/margins": 7.815065860748291,
      "rewards/rejected": -8.027475357055664,
      "step": 207
    },
    {
      "epoch": 2.4651851851851854,
      "grad_norm": 9.373669301801096,
      "learning_rate": 1.907539678768453e-07,
      "logits/chosen": -0.5164112448692322,
      "logits/rejected": -0.4492265284061432,
      "logps/chosen": -31.97795295715332,
      "logps/rejected": -73.8591079711914,
      "loss": 0.0554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6608830094337463,
      "rewards/margins": 7.822652816772461,
      "rewards/rejected": -8.483535766601562,
      "step": 208
    },
    {
      "epoch": 2.477037037037037,
      "grad_norm": 8.80264276423414,
      "learning_rate": 1.8823064642457876e-07,
      "logits/chosen": -0.19362421333789825,
      "logits/rejected": -0.09952510893344879,
      "logps/chosen": -36.87741470336914,
      "logps/rejected": -76.23078918457031,
      "loss": 0.0465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9911991357803345,
      "rewards/margins": 7.124609470367432,
      "rewards/rejected": -8.115808486938477,
      "step": 209
    },
    {
      "epoch": 2.488888888888889,
      "grad_norm": 6.872370358967133,
      "learning_rate": 1.8571400925793852e-07,
      "logits/chosen": -0.28052157163619995,
      "logits/rejected": -0.18670235574245453,
      "logps/chosen": -34.98965835571289,
      "logps/rejected": -62.01945495605469,
      "loss": 0.036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1422077715396881,
      "rewards/margins": 7.268811225891113,
      "rewards/rejected": -7.126603126525879,
      "step": 210
    },
    {
      "epoch": 2.5007407407407407,
      "grad_norm": 6.102992921343477,
      "learning_rate": 1.8320432871135376e-07,
      "logits/chosen": -0.012273239903151989,
      "logits/rejected": 0.11726081371307373,
      "logps/chosen": -41.02720642089844,
      "logps/rejected": -70.41509246826172,
      "loss": 0.0326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5696543455123901,
      "rewards/margins": 6.976672649383545,
      "rewards/rejected": -7.546327114105225,
      "step": 211
    },
    {
      "epoch": 2.5125925925925925,
      "grad_norm": 7.523877695462096,
      "learning_rate": 1.8070187636645237e-07,
      "logits/chosen": -0.25425052642822266,
      "logits/rejected": -0.20172733068466187,
      "logps/chosen": -29.920835494995117,
      "logps/rejected": -58.127830505371094,
      "loss": 0.0442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9108222126960754,
      "rewards/margins": 5.636702060699463,
      "rewards/rejected": -6.547523498535156,
      "step": 212
    },
    {
      "epoch": 2.5244444444444447,
      "grad_norm": 8.233825538587578,
      "learning_rate": 1.782069230226725e-07,
      "logits/chosen": -0.1111406460404396,
      "logits/rejected": -0.07830701768398285,
      "logps/chosen": -36.62953567504883,
      "logps/rejected": -69.00572204589844,
      "loss": 0.0489,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7046716809272766,
      "rewards/margins": 8.098491668701172,
      "rewards/rejected": -8.803162574768066,
      "step": 213
    },
    {
      "epoch": 2.536296296296296,
      "grad_norm": 7.26503668987421,
      "learning_rate": 1.7571973866795813e-07,
      "logits/chosen": -0.3010156750679016,
      "logits/rejected": -0.14240717887878418,
      "logps/chosen": -28.9267635345459,
      "logps/rejected": -62.530731201171875,
      "loss": 0.0385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.38880711793899536,
      "rewards/margins": 7.034039497375488,
      "rewards/rejected": -7.422846794128418,
      "step": 214
    },
    {
      "epoch": 2.5481481481481483,
      "grad_norm": 5.978089867317629,
      "learning_rate": 1.7324059244954292e-07,
      "logits/chosen": -0.4227255582809448,
      "logits/rejected": -0.40319374203681946,
      "logps/chosen": -33.19075012207031,
      "logps/rejected": -64.02904510498047,
      "loss": 0.0325,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5749123096466064,
      "rewards/margins": 9.057025909423828,
      "rewards/rejected": -9.631938934326172,
      "step": 215
    },
    {
      "epoch": 2.56,
      "grad_norm": 9.696303087236457,
      "learning_rate": 1.7076975264482433e-07,
      "logits/chosen": -0.37839898467063904,
      "logits/rejected": -0.273608922958374,
      "logps/chosen": -36.7985954284668,
      "logps/rejected": -64.79552459716797,
      "loss": 0.0498,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2072323560714722,
      "rewards/margins": 6.241967678070068,
      "rewards/rejected": -7.449199676513672,
      "step": 216
    },
    {
      "epoch": 2.571851851851852,
      "grad_norm": 7.034579383121281,
      "learning_rate": 1.6830748663233303e-07,
      "logits/chosen": -0.25258129835128784,
      "logits/rejected": -0.23720452189445496,
      "logps/chosen": -31.781192779541016,
      "logps/rejected": -62.308006286621094,
      "loss": 0.0353,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21115940809249878,
      "rewards/margins": 7.114487648010254,
      "rewards/rejected": -7.325647354125977,
      "step": 217
    },
    {
      "epoch": 2.5837037037037036,
      "grad_norm": 8.754965473386916,
      "learning_rate": 1.6585406086279846e-07,
      "logits/chosen": -0.43037766218185425,
      "logits/rejected": -0.375767320394516,
      "logps/chosen": -40.90904998779297,
      "logps/rejected": -76.52076721191406,
      "loss": 0.0544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7350015044212341,
      "rewards/margins": 8.066032409667969,
      "rewards/rejected": -8.801033973693848,
      "step": 218
    },
    {
      "epoch": 2.5955555555555554,
      "grad_norm": 5.4726077818919565,
      "learning_rate": 1.6340974083031523e-07,
      "logits/chosen": -0.29419374465942383,
      "logits/rejected": -0.28271955251693726,
      "logps/chosen": -30.226680755615234,
      "logps/rejected": -55.334014892578125,
      "loss": 0.0302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.04286186397075653,
      "rewards/margins": 6.613791465759277,
      "rewards/rejected": -6.570930004119873,
      "step": 219
    },
    {
      "epoch": 2.6074074074074076,
      "grad_norm": 7.600745751196765,
      "learning_rate": 1.6097479104361326e-07,
      "logits/chosen": -0.38003548979759216,
      "logits/rejected": -0.20779910683631897,
      "logps/chosen": -26.411277770996094,
      "logps/rejected": -65.45418548583984,
      "loss": 0.0318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08172351866960526,
      "rewards/margins": 7.309746742248535,
      "rewards/rejected": -7.228023052215576,
      "step": 220
    },
    {
      "epoch": 2.6192592592592594,
      "grad_norm": 5.529941543826629,
      "learning_rate": 1.5854947499743413e-07,
      "logits/chosen": -0.2649455666542053,
      "logits/rejected": -0.1322605013847351,
      "logps/chosen": -28.605796813964844,
      "logps/rejected": -67.70567321777344,
      "loss": 0.0316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8953390717506409,
      "rewards/margins": 7.937841892242432,
      "rewards/rejected": -8.83318042755127,
      "step": 221
    },
    {
      "epoch": 2.631111111111111,
      "grad_norm": 7.828964734675065,
      "learning_rate": 1.5613405514401757e-07,
      "logits/chosen": -0.4999098479747772,
      "logits/rejected": -0.46459028124809265,
      "logps/chosen": -32.34528350830078,
      "logps/rejected": -64.39063262939453,
      "loss": 0.0417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7131352424621582,
      "rewards/margins": 7.366223335266113,
      "rewards/rejected": -9.07935905456543,
      "step": 222
    },
    {
      "epoch": 2.642962962962963,
      "grad_norm": 6.684903510896331,
      "learning_rate": 1.537287928647002e-07,
      "logits/chosen": -0.33326905965805054,
      "logits/rejected": -0.2772333025932312,
      "logps/chosen": -33.04732894897461,
      "logps/rejected": -55.9017448425293,
      "loss": 0.034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4501342475414276,
      "rewards/margins": 6.482940196990967,
      "rewards/rejected": -6.933073997497559,
      "step": 223
    },
    {
      "epoch": 2.6548148148148147,
      "grad_norm": 7.774671778619875,
      "learning_rate": 1.513339484416309e-07,
      "logits/chosen": -0.350558876991272,
      "logits/rejected": -0.28618156909942627,
      "logps/chosen": -49.11450958251953,
      "logps/rejected": -80.40065002441406,
      "loss": 0.0421,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9198662042617798,
      "rewards/margins": 8.049423217773438,
      "rewards/rejected": -9.969290733337402,
      "step": 224
    },
    {
      "epoch": 2.6666666666666665,
      "grad_norm": 6.147372232877328,
      "learning_rate": 1.489497810296046e-07,
      "logits/chosen": -0.2636696696281433,
      "logits/rejected": -0.1582798808813095,
      "logps/chosen": -33.32222366333008,
      "logps/rejected": -88.90251159667969,
      "loss": 0.0347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8391042947769165,
      "rewards/margins": 9.376372337341309,
      "rewards/rejected": -10.215476036071777,
      "step": 225
    },
    {
      "epoch": 2.6785185185185183,
      "grad_norm": 6.058104127594901,
      "learning_rate": 1.4657654862801797e-07,
      "logits/chosen": -0.3205685317516327,
      "logits/rejected": -0.24160242080688477,
      "logps/chosen": -28.52737045288086,
      "logps/rejected": -70.28912353515625,
      "loss": 0.0233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.24154254794120789,
      "rewards/margins": 6.4893083572387695,
      "rewards/rejected": -6.730850696563721,
      "step": 226
    },
    {
      "epoch": 2.6903703703703705,
      "grad_norm": 10.148811706386386,
      "learning_rate": 1.4421450805295082e-07,
      "logits/chosen": -0.33272168040275574,
      "logits/rejected": -0.3017561733722687,
      "logps/chosen": -40.76533889770508,
      "logps/rejected": -58.748046875,
      "loss": 0.0433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.23764045536518097,
      "rewards/margins": 5.5487141609191895,
      "rewards/rejected": -5.7863545417785645,
      "step": 227
    },
    {
      "epoch": 2.7022222222222223,
      "grad_norm": 7.063093696385122,
      "learning_rate": 1.418639149093748e-07,
      "logits/chosen": -0.5206415057182312,
      "logits/rejected": -0.42945098876953125,
      "logps/chosen": -35.94019317626953,
      "logps/rejected": -51.362579345703125,
      "loss": 0.0385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5331210494041443,
      "rewards/margins": 5.205883502960205,
      "rewards/rejected": -5.739004611968994,
      "step": 228
    },
    {
      "epoch": 2.714074074074074,
      "grad_norm": 7.873488891465637,
      "learning_rate": 1.3952502356349323e-07,
      "logits/chosen": -0.2090906947851181,
      "logits/rejected": -0.116541787981987,
      "logps/chosen": -36.972110748291016,
      "logps/rejected": -70.80089569091797,
      "loss": 0.0445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16982126235961914,
      "rewards/margins": 8.14144515991211,
      "rewards/rejected": -8.311266899108887,
      "step": 229
    },
    {
      "epoch": 2.725925925925926,
      "grad_norm": 6.247579493903217,
      "learning_rate": 1.371980871152157e-07,
      "logits/chosen": -0.14566001296043396,
      "logits/rejected": -0.19308951497077942,
      "logps/chosen": -41.63805389404297,
      "logps/rejected": -75.58000183105469,
      "loss": 0.0357,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.45496970415115356,
      "rewards/margins": 8.866785049438477,
      "rewards/rejected": -9.321755409240723,
      "step": 230
    },
    {
      "epoch": 2.7377777777777776,
      "grad_norm": 6.023624860692066,
      "learning_rate": 1.3488335737076911e-07,
      "logits/chosen": -0.24541382491588593,
      "logits/rejected": -0.2724686861038208,
      "logps/chosen": -33.94440841674805,
      "logps/rejected": -54.19181823730469,
      "loss": 0.0349,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.641716718673706,
      "rewards/margins": 6.787499904632568,
      "rewards/rejected": -7.4292168617248535,
      "step": 231
    },
    {
      "epoch": 2.74962962962963,
      "grad_norm": 6.829922502232146,
      "learning_rate": 1.3258108481544847e-07,
      "logits/chosen": -0.2750440537929535,
      "logits/rejected": -0.2242782562971115,
      "logps/chosen": -47.98163986206055,
      "logps/rejected": -71.17113494873047,
      "loss": 0.0381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.330321192741394,
      "rewards/margins": 5.847900390625,
      "rewards/rejected": -7.178221702575684,
      "step": 232
    },
    {
      "epoch": 2.7614814814814816,
      "grad_norm": 9.939842224750613,
      "learning_rate": 1.3029151858651143e-07,
      "logits/chosen": -0.4768088757991791,
      "logits/rejected": -0.3678171634674072,
      "logps/chosen": -28.488832473754883,
      "logps/rejected": -72.2415542602539,
      "loss": 0.0563,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4457303285598755,
      "rewards/margins": 7.388967514038086,
      "rewards/rejected": -7.834697723388672,
      "step": 233
    },
    {
      "epoch": 2.7733333333333334,
      "grad_norm": 5.772137373566735,
      "learning_rate": 1.2801490644621788e-07,
      "logits/chosen": -0.10860705375671387,
      "logits/rejected": -0.09419623762369156,
      "logps/chosen": -41.14183807373047,
      "logps/rejected": -73.03929138183594,
      "loss": 0.0334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0385979413986206,
      "rewards/margins": 9.18422794342041,
      "rewards/rejected": -10.222824096679688,
      "step": 234
    },
    {
      "epoch": 2.785185185185185,
      "grad_norm": 6.872437338157428,
      "learning_rate": 1.257514947550189e-07,
      "logits/chosen": -0.32558369636535645,
      "logits/rejected": -0.23454414308071136,
      "logps/chosen": -27.442285537719727,
      "logps/rejected": -47.84419250488281,
      "loss": 0.0369,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5560548305511475,
      "rewards/margins": 5.780346870422363,
      "rewards/rejected": -6.336400985717773,
      "step": 235
    },
    {
      "epoch": 2.797037037037037,
      "grad_norm": 9.526509052512719,
      "learning_rate": 1.2350152844489688e-07,
      "logits/chosen": -0.3666895925998688,
      "logits/rejected": -0.22046907246112823,
      "logps/chosen": -38.18906021118164,
      "logps/rejected": -68.36451721191406,
      "loss": 0.0521,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1897006034851074,
      "rewards/margins": 6.763792991638184,
      "rewards/rejected": -7.953493118286133,
      "step": 236
    },
    {
      "epoch": 2.8088888888888888,
      "grad_norm": 6.2325582014169205,
      "learning_rate": 1.2126525099286108e-07,
      "logits/chosen": -0.3752056956291199,
      "logits/rejected": -0.29132628440856934,
      "logps/chosen": -39.4105110168457,
      "logps/rejected": -70.36639404296875,
      "loss": 0.0344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.190868854522705,
      "rewards/margins": 7.782485008239746,
      "rewards/rejected": -8.973353385925293,
      "step": 237
    },
    {
      "epoch": 2.8207407407407405,
      "grad_norm": 6.813356776563027,
      "learning_rate": 1.1904290439459971e-07,
      "logits/chosen": -0.4055876135826111,
      "logits/rejected": -0.34407296776771545,
      "logps/chosen": -36.34687805175781,
      "logps/rejected": -64.90667724609375,
      "loss": 0.029,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6599448323249817,
      "rewards/margins": 7.470705509185791,
      "rewards/rejected": -8.130650520324707,
      "step": 238
    },
    {
      "epoch": 2.8325925925925928,
      "grad_norm": 8.170401197759237,
      "learning_rate": 1.1683472913829284e-07,
      "logits/chosen": -0.20514726638793945,
      "logits/rejected": -0.11336632817983627,
      "logps/chosen": -51.73873519897461,
      "logps/rejected": -71.9918441772461,
      "loss": 0.0379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8510772585868835,
      "rewards/margins": 6.703639507293701,
      "rewards/rejected": -7.554717063903809,
      "step": 239
    },
    {
      "epoch": 2.8444444444444446,
      "grad_norm": 8.191853793447038,
      "learning_rate": 1.146409641785882e-07,
      "logits/chosen": -0.16900832951068878,
      "logits/rejected": -0.16538989543914795,
      "logps/chosen": -44.017845153808594,
      "logps/rejected": -54.396583557128906,
      "loss": 0.0415,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.656957983970642,
      "rewards/margins": 4.629300117492676,
      "rewards/rejected": -6.286258697509766,
      "step": 240
    },
    {
      "epoch": 2.8562962962962963,
      "grad_norm": 7.381077568622568,
      "learning_rate": 1.1246184691074314e-07,
      "logits/chosen": -0.24956950545310974,
      "logits/rejected": -0.24476227164268494,
      "logps/chosen": -42.77149963378906,
      "logps/rejected": -79.72845458984375,
      "loss": 0.0398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6535344123840332,
      "rewards/margins": 9.295120239257812,
      "rewards/rejected": -9.948655128479004,
      "step": 241
    },
    {
      "epoch": 2.868148148148148,
      "grad_norm": 8.160459964737315,
      "learning_rate": 1.1029761314493518e-07,
      "logits/chosen": -0.30951178073883057,
      "logits/rejected": -0.3085937201976776,
      "logps/chosen": -40.81586456298828,
      "logps/rejected": -60.63238525390625,
      "loss": 0.0367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3215899467468262,
      "rewards/margins": 6.157529830932617,
      "rewards/rejected": -7.479120254516602,
      "step": 242
    },
    {
      "epoch": 2.88,
      "grad_norm": 8.037040216975743,
      "learning_rate": 1.0814849708074414e-07,
      "logits/chosen": -0.30921998620033264,
      "logits/rejected": -0.3065870404243469,
      "logps/chosen": -51.514556884765625,
      "logps/rejected": -66.55127716064453,
      "loss": 0.0324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17993119359016418,
      "rewards/margins": 6.790800094604492,
      "rewards/rejected": -6.970730781555176,
      "step": 243
    },
    {
      "epoch": 2.891851851851852,
      "grad_norm": 9.656051990182503,
      "learning_rate": 1.0601473128180854e-07,
      "logits/chosen": -0.2805531620979309,
      "logits/rejected": -0.2533468008041382,
      "logps/chosen": -46.80023193359375,
      "logps/rejected": -68.6700668334961,
      "loss": 0.0383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8430509567260742,
      "rewards/margins": 8.124579429626465,
      "rewards/rejected": -8.967630386352539,
      "step": 244
    },
    {
      "epoch": 2.9037037037037035,
      "grad_norm": 8.425061421071412,
      "learning_rate": 1.0389654665065908e-07,
      "logits/chosen": -0.2665305435657501,
      "logits/rejected": -0.30099448561668396,
      "logps/chosen": -36.76901626586914,
      "logps/rejected": -62.380638122558594,
      "loss": 0.0436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7122980952262878,
      "rewards/margins": 7.467397689819336,
      "rewards/rejected": -8.179695129394531,
      "step": 245
    },
    {
      "epoch": 2.9155555555555557,
      "grad_norm": 7.412242548453604,
      "learning_rate": 1.0179417240373182e-07,
      "logits/chosen": -0.32122743129730225,
      "logits/rejected": -0.22737433016300201,
      "logps/chosen": -52.4939079284668,
      "logps/rejected": -86.98854064941406,
      "loss": 0.0332,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.623575210571289,
      "rewards/margins": 8.667156219482422,
      "rewards/rejected": -11.290731430053711,
      "step": 246
    },
    {
      "epoch": 2.9274074074074075,
      "grad_norm": 6.343263474138892,
      "learning_rate": 9.970783604656383e-08,
      "logits/chosen": -0.40669649839401245,
      "logits/rejected": -0.2913494408130646,
      "logps/chosen": -40.742733001708984,
      "logps/rejected": -70.4676513671875,
      "loss": 0.0345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4790029525756836,
      "rewards/margins": 7.971775054931641,
      "rewards/rejected": -8.450777053833008,
      "step": 247
    },
    {
      "epoch": 2.9392592592592592,
      "grad_norm": 8.350168639074793,
      "learning_rate": 9.763776334917398e-08,
      "logits/chosen": -0.28063714504241943,
      "logits/rejected": -0.2614033818244934,
      "logps/chosen": -39.683170318603516,
      "logps/rejected": -56.151100158691406,
      "loss": 0.0442,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2253810167312622,
      "rewards/margins": 5.419277667999268,
      "rewards/rejected": -6.644659042358398,
      "step": 248
    },
    {
      "epoch": 2.951111111111111,
      "grad_norm": 6.1204430667008864,
      "learning_rate": 9.558417832163162e-08,
      "logits/chosen": -0.13153290748596191,
      "logits/rejected": -0.19138801097869873,
      "logps/chosen": -38.79569625854492,
      "logps/rejected": -55.324180603027344,
      "loss": 0.0339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.37145036458969116,
      "rewards/margins": 6.06140661239624,
      "rewards/rejected": -6.432857513427734,
      "step": 249
    },
    {
      "epoch": 2.962962962962963,
      "grad_norm": 7.267482062373579,
      "learning_rate": 9.354730318981561e-08,
      "logits/chosen": -0.4541955590248108,
      "logits/rejected": -0.3866829574108124,
      "logps/chosen": -31.26105308532715,
      "logps/rejected": -66.00698852539062,
      "loss": 0.0438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9516152739524841,
      "rewards/margins": 7.78145170211792,
      "rewards/rejected": -8.73306655883789,
      "step": 250
    },
    {
      "epoch": 2.974814814814815,
      "grad_norm": 5.662458264963396,
      "learning_rate": 9.15273583713663e-08,
      "logits/chosen": -0.37392503023147583,
      "logits/rejected": -0.29210343956947327,
      "logps/chosen": -47.48450469970703,
      "logps/rejected": -90.92308044433594,
      "loss": 0.0351,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.219913959503174,
      "rewards/margins": 12.272329330444336,
      "rewards/rejected": -14.492244720458984,
      "step": 251
    },
    {
      "epoch": 2.986666666666667,
      "grad_norm": 5.97436680259246,
      "learning_rate": 8.95245624518336e-08,
      "logits/chosen": -0.3016185760498047,
      "logits/rejected": -0.2964284420013428,
      "logps/chosen": -34.3846321105957,
      "logps/rejected": -68.51025390625,
      "loss": 0.0343,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7737685441970825,
      "rewards/margins": 7.51839017868042,
      "rewards/rejected": -8.292159080505371,
      "step": 252
    },
    {
      "epoch": 2.9985185185185186,
      "grad_norm": 6.809722463225596,
      "learning_rate": 8.753913216102285e-08,
      "logits/chosen": -0.26927924156188965,
      "logits/rejected": -0.09772679954767227,
      "logps/chosen": -39.219181060791016,
      "logps/rejected": -83.87496185302734,
      "loss": 0.0378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.874872088432312,
      "rewards/margins": 7.590030193328857,
      "rewards/rejected": -9.464902877807617,
      "step": 253
    },
    {
      "epoch": 3.0103703703703704,
      "grad_norm": 5.7996860529913565,
      "learning_rate": 8.557128234954189e-08,
      "logits/chosen": -0.40512141585350037,
      "logits/rejected": -0.3163852393627167,
      "logps/chosen": -29.341394424438477,
      "logps/rejected": -70.40370178222656,
      "loss": 0.0283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9042503833770752,
      "rewards/margins": 8.99386978149414,
      "rewards/rejected": -9.898119926452637,
      "step": 254
    },
    {
      "epoch": 3.022222222222222,
      "grad_norm": 5.286377326614028,
      "learning_rate": 8.362122596555088e-08,
      "logits/chosen": -0.47114288806915283,
      "logits/rejected": -0.3825288712978363,
      "logps/chosen": -33.60106658935547,
      "logps/rejected": -76.53120422363281,
      "loss": 0.0282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.207690954208374,
      "rewards/margins": 10.646196365356445,
      "rewards/rejected": -11.853886604309082,
      "step": 255
    },
    {
      "epoch": 3.034074074074074,
      "grad_norm": 6.253088250116567,
      "learning_rate": 8.16891740317189e-08,
      "logits/chosen": -0.3761712908744812,
      "logits/rejected": -0.35195398330688477,
      "logps/chosen": -33.14177322387695,
      "logps/rejected": -55.4194450378418,
      "loss": 0.0385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6758537888526917,
      "rewards/margins": 6.090365409851074,
      "rewards/rejected": -6.766219615936279,
      "step": 256
    },
    {
      "epoch": 3.0459259259259257,
      "grad_norm": 6.986196206894695,
      "learning_rate": 7.977533562238838e-08,
      "logits/chosen": -0.4037611186504364,
      "logits/rejected": -0.3634166419506073,
      "logps/chosen": -34.40519332885742,
      "logps/rejected": -70.6146240234375,
      "loss": 0.0372,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2543308138847351,
      "rewards/margins": 7.034722328186035,
      "rewards/rejected": -7.289052963256836,
      "step": 257
    },
    {
      "epoch": 3.057777777777778,
      "grad_norm": 5.505207911608369,
      "learning_rate": 7.787991784094999e-08,
      "logits/chosen": -0.2387389987707138,
      "logits/rejected": -0.08904880285263062,
      "logps/chosen": -36.03857421875,
      "logps/rejected": -89.65563201904297,
      "loss": 0.0278,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6398571133613586,
      "rewards/margins": 8.411537170410156,
      "rewards/rejected": -9.051393508911133,
      "step": 258
    },
    {
      "epoch": 3.0696296296296297,
      "grad_norm": 6.3788891492475415,
      "learning_rate": 7.60031257974316e-08,
      "logits/chosen": -0.36758318543434143,
      "logits/rejected": -0.28655973076820374,
      "logps/chosen": -34.49348831176758,
      "logps/rejected": -75.51551818847656,
      "loss": 0.0368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3124196529388428,
      "rewards/margins": 9.141406059265137,
      "rewards/rejected": -10.453825950622559,
      "step": 259
    },
    {
      "epoch": 3.0814814814814815,
      "grad_norm": 7.213670504047207,
      "learning_rate": 7.414516258630244e-08,
      "logits/chosen": -0.2934122681617737,
      "logits/rejected": -0.2819547653198242,
      "logps/chosen": -49.67085647583008,
      "logps/rejected": -82.62693786621094,
      "loss": 0.0342,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0997469425201416,
      "rewards/margins": 9.53528118133545,
      "rewards/rejected": -10.635027885437012,
      "step": 260
    },
    {
      "epoch": 3.0933333333333333,
      "grad_norm": 5.692432592478289,
      "learning_rate": 7.230622926449564e-08,
      "logits/chosen": -0.2739347219467163,
      "logits/rejected": -0.20775115489959717,
      "logps/chosen": -37.56914520263672,
      "logps/rejected": -65.73677062988281,
      "loss": 0.0333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4430259466171265,
      "rewards/margins": 8.155195236206055,
      "rewards/rejected": -9.598221778869629,
      "step": 261
    },
    {
      "epoch": 3.105185185185185,
      "grad_norm": 7.076581419281233,
      "learning_rate": 7.048652482965078e-08,
      "logits/chosen": -0.18068230152130127,
      "logits/rejected": -0.1852649450302124,
      "logps/chosen": -46.34651184082031,
      "logps/rejected": -68.5006103515625,
      "loss": 0.037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1328610181808472,
      "rewards/margins": 7.682218551635742,
      "rewards/rejected": -8.815078735351562,
      "step": 262
    },
    {
      "epoch": 3.117037037037037,
      "grad_norm": 6.338236077273577,
      "learning_rate": 6.868624619858021e-08,
      "logits/chosen": -0.2783002257347107,
      "logits/rejected": -0.3345209062099457,
      "logps/chosen": -36.772254943847656,
      "logps/rejected": -83.5664291381836,
      "loss": 0.0277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4274479150772095,
      "rewards/margins": 7.226326942443848,
      "rewards/rejected": -7.653774261474609,
      "step": 263
    },
    {
      "epoch": 3.128888888888889,
      "grad_norm": 5.495669924723658,
      "learning_rate": 6.690558818595943e-08,
      "logits/chosen": -0.34206265211105347,
      "logits/rejected": -0.19553421437740326,
      "logps/chosen": -32.38424301147461,
      "logps/rejected": -85.89747619628906,
      "loss": 0.0276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1448084115982056,
      "rewards/margins": 9.975635528564453,
      "rewards/rejected": -11.120444297790527,
      "step": 264
    },
    {
      "epoch": 3.140740740740741,
      "grad_norm": 5.0666203766090385,
      "learning_rate": 6.514474348324581e-08,
      "logits/chosen": -0.38811901211738586,
      "logits/rejected": -0.2747833728790283,
      "logps/chosen": -48.448951721191406,
      "logps/rejected": -77.10838317871094,
      "loss": 0.028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6878550052642822,
      "rewards/margins": 7.391786575317383,
      "rewards/rejected": -9.079641342163086,
      "step": 265
    },
    {
      "epoch": 3.1525925925925926,
      "grad_norm": 8.885007920801026,
      "learning_rate": 6.340390263782655e-08,
      "logits/chosen": -0.5093058347702026,
      "logits/rejected": -0.3832343816757202,
      "logps/chosen": -32.23210144042969,
      "logps/rejected": -76.0536117553711,
      "loss": 0.047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.038543462753296,
      "rewards/margins": 9.122876167297363,
      "rewards/rejected": -10.161420822143555,
      "step": 266
    },
    {
      "epoch": 3.1644444444444444,
      "grad_norm": 7.984860081690398,
      "learning_rate": 6.168325403239913e-08,
      "logits/chosen": -0.4433887004852295,
      "logits/rejected": -0.3792242109775543,
      "logps/chosen": -29.287479400634766,
      "logps/rejected": -58.2768669128418,
      "loss": 0.0387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.46901828050613403,
      "rewards/margins": 7.602441787719727,
      "rewards/rejected": -8.071460723876953,
      "step": 267
    },
    {
      "epoch": 3.176296296296296,
      "grad_norm": 4.958032867566694,
      "learning_rate": 5.998298386458545e-08,
      "logits/chosen": -0.22974154353141785,
      "logits/rejected": -0.19992095232009888,
      "logps/chosen": -38.70039367675781,
      "logps/rejected": -72.84990692138672,
      "loss": 0.0231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0671855211257935,
      "rewards/margins": 7.9272379875183105,
      "rewards/rejected": -8.994423866271973,
      "step": 268
    },
    {
      "epoch": 3.188148148148148,
      "grad_norm": 5.988243698878292,
      "learning_rate": 5.830327612678265e-08,
      "logits/chosen": -0.32180365920066833,
      "logits/rejected": -0.3045603632926941,
      "logps/chosen": -43.980316162109375,
      "logps/rejected": -83.64070129394531,
      "loss": 0.0285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.710637331008911,
      "rewards/margins": 9.31506061553955,
      "rewards/rejected": -12.025696754455566,
      "step": 269
    },
    {
      "epoch": 3.2,
      "grad_norm": 5.602823035809897,
      "learning_rate": 5.6644312586253044e-08,
      "logits/chosen": -0.0014043133705854416,
      "logits/rejected": -0.04007536917924881,
      "logps/chosen": -63.08719253540039,
      "logps/rejected": -80.3905029296875,
      "loss": 0.0292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9075740575790405,
      "rewards/margins": 7.626714706420898,
      "rewards/rejected": -9.53428840637207,
      "step": 270
    },
    {
      "epoch": 3.211851851851852,
      "grad_norm": 7.326638872216747,
      "learning_rate": 5.5006272765454056e-08,
      "logits/chosen": -0.43287378549575806,
      "logits/rejected": -0.27151188254356384,
      "logps/chosen": -33.51972579956055,
      "logps/rejected": -58.82566452026367,
      "loss": 0.0425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.519016444683075,
      "rewards/margins": 6.466248512268066,
      "rewards/rejected": -6.985265731811523,
      "step": 271
    },
    {
      "epoch": 3.2237037037037037,
      "grad_norm": 4.755215217001098,
      "learning_rate": 5.338933392261158e-08,
      "logits/chosen": -0.2298121452331543,
      "logits/rejected": -0.17249788343906403,
      "logps/chosen": -37.40292739868164,
      "logps/rejected": -64.0435562133789,
      "loss": 0.0245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.865021288394928,
      "rewards/margins": 6.909914016723633,
      "rewards/rejected": -7.774934768676758,
      "step": 272
    },
    {
      "epoch": 3.2355555555555555,
      "grad_norm": 7.328735036408531,
      "learning_rate": 5.1793671032538206e-08,
      "logits/chosen": -0.5466493368148804,
      "logits/rejected": -0.49364450573921204,
      "logps/chosen": -31.749622344970703,
      "logps/rejected": -76.33462524414062,
      "loss": 0.0351,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5585615634918213,
      "rewards/margins": 8.347299575805664,
      "rewards/rejected": -8.905860900878906,
      "step": 273
    },
    {
      "epoch": 3.2474074074074073,
      "grad_norm": 4.866161229053689,
      "learning_rate": 5.021945676769859e-08,
      "logits/chosen": -0.5478118658065796,
      "logits/rejected": -0.38123688101768494,
      "logps/chosen": -26.91775131225586,
      "logps/rejected": -66.9200668334961,
      "loss": 0.0256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.39309221506118774,
      "rewards/margins": 8.147336959838867,
      "rewards/rejected": -8.540430068969727,
      "step": 274
    },
    {
      "epoch": 3.259259259259259,
      "grad_norm": 5.465123706910723,
      "learning_rate": 4.866686147952387e-08,
      "logits/chosen": -0.15793977677822113,
      "logits/rejected": -0.13396653532981873,
      "logps/chosen": -38.39078140258789,
      "logps/rejected": -68.49137115478516,
      "loss": 0.0332,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5564990639686584,
      "rewards/margins": 7.465914726257324,
      "rewards/rejected": -8.022414207458496,
      "step": 275
    },
    {
      "epoch": 3.2711111111111113,
      "grad_norm": 6.892669002618101,
      "learning_rate": 4.71360531799774e-08,
      "logits/chosen": -0.17291544377803802,
      "logits/rejected": -0.11444761604070663,
      "logps/chosen": -52.285491943359375,
      "logps/rejected": -84.1561508178711,
      "loss": 0.0411,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8530375957489014,
      "rewards/margins": 7.457571983337402,
      "rewards/rejected": -10.310609817504883,
      "step": 276
    },
    {
      "epoch": 3.282962962962963,
      "grad_norm": 4.92957758138627,
      "learning_rate": 4.562719752337349e-08,
      "logits/chosen": -0.47689568996429443,
      "logits/rejected": -0.38014689087867737,
      "logps/chosen": -51.07635498046875,
      "logps/rejected": -94.97547149658203,
      "loss": 0.0246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.258345603942871,
      "rewards/margins": 8.710776329040527,
      "rewards/rejected": -10.969121932983398,
      "step": 277
    },
    {
      "epoch": 3.294814814814815,
      "grad_norm": 5.454251196984929,
      "learning_rate": 4.4140457788451434e-08,
      "logits/chosen": -0.3425113260746002,
      "logits/rejected": -0.25056517124176025,
      "logps/chosen": -31.684978485107422,
      "logps/rejected": -69.71234130859375,
      "loss": 0.0282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.24049748480319977,
      "rewards/margins": 7.845020771026611,
      "rewards/rejected": -8.085518836975098,
      "step": 278
    },
    {
      "epoch": 3.3066666666666666,
      "grad_norm": 5.883268469101933,
      "learning_rate": 4.267599486070647e-08,
      "logits/chosen": -0.172508105635643,
      "logits/rejected": -0.20862886309623718,
      "logps/chosen": -39.89122772216797,
      "logps/rejected": -52.314510345458984,
      "loss": 0.0292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3147379755973816,
      "rewards/margins": 5.7033185958862305,
      "rewards/rejected": -6.0180559158325195,
      "step": 279
    },
    {
      "epoch": 3.3185185185185184,
      "grad_norm": 6.523804321940008,
      "learning_rate": 4.1233967214979764e-08,
      "logits/chosen": -0.3144129812717438,
      "logits/rejected": -0.22518262267112732,
      "logps/chosen": -42.5799560546875,
      "logps/rejected": -53.63530349731445,
      "loss": 0.0332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.022313117980957,
      "rewards/margins": 4.446239471435547,
      "rewards/rejected": -5.468553066253662,
      "step": 280
    },
    {
      "epoch": 3.33037037037037,
      "grad_norm": 5.61217781900571,
      "learning_rate": 3.9814530898309356e-08,
      "logits/chosen": -0.2805265784263611,
      "logits/rejected": -0.17468589544296265,
      "logps/chosen": -36.89150619506836,
      "logps/rejected": -73.11276245117188,
      "loss": 0.0278,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6258598566055298,
      "rewards/margins": 8.552447319030762,
      "rewards/rejected": -9.17830753326416,
      "step": 281
    },
    {
      "epoch": 3.3422222222222224,
      "grad_norm": 5.921953356785593,
      "learning_rate": 3.8417839513043646e-08,
      "logits/chosen": -0.28441232442855835,
      "logits/rejected": -0.20493623614311218,
      "logps/chosen": -41.19779586791992,
      "logps/rejected": -60.01893615722656,
      "loss": 0.0339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5952982902526855,
      "rewards/margins": 5.5062713623046875,
      "rewards/rejected": -7.101569652557373,
      "step": 282
    },
    {
      "epoch": 3.354074074074074,
      "grad_norm": 5.347638108561962,
      "learning_rate": 3.704404420021956e-08,
      "logits/chosen": -0.3048914670944214,
      "logits/rejected": -0.17332229018211365,
      "logps/chosen": -33.9517936706543,
      "logps/rejected": -71.91447448730469,
      "loss": 0.0351,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8167021870613098,
      "rewards/margins": 8.507079124450684,
      "rewards/rejected": -9.32378101348877,
      "step": 283
    },
    {
      "epoch": 3.365925925925926,
      "grad_norm": 5.238374490213889,
      "learning_rate": 3.569329362320708e-08,
      "logits/chosen": -0.2728411853313446,
      "logits/rejected": -0.2584533393383026,
      "logps/chosen": -30.238557815551758,
      "logps/rejected": -75.02142333984375,
      "loss": 0.0269,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.49139711260795593,
      "rewards/margins": 9.32203197479248,
      "rewards/rejected": -9.813429832458496,
      "step": 284
    },
    {
      "epoch": 3.3777777777777778,
      "grad_norm": 6.525469512599618,
      "learning_rate": 3.436573395162179e-08,
      "logits/chosen": -0.3524834215641022,
      "logits/rejected": -0.30102020502090454,
      "logps/chosen": -30.73918914794922,
      "logps/rejected": -59.49354553222656,
      "loss": 0.0396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09267859905958176,
      "rewards/margins": 6.891367435455322,
      "rewards/rejected": -6.984046459197998,
      "step": 285
    },
    {
      "epoch": 3.3896296296296295,
      "grad_norm": 5.564012623506871,
      "learning_rate": 3.306150884550732e-08,
      "logits/chosen": -0.3768519461154938,
      "logits/rejected": -0.3264191448688507,
      "logps/chosen": -41.36799240112305,
      "logps/rejected": -67.31135559082031,
      "loss": 0.0329,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0806761980056763,
      "rewards/margins": 7.03513240814209,
      "rewards/rejected": -8.115808486938477,
      "step": 286
    },
    {
      "epoch": 3.4014814814814813,
      "grad_norm": 5.931359172922728,
      "learning_rate": 3.17807594397895e-08,
      "logits/chosen": -0.3008241653442383,
      "logits/rejected": -0.1854274868965149,
      "logps/chosen": -35.424800872802734,
      "logps/rejected": -75.77165222167969,
      "loss": 0.0242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3784024715423584,
      "rewards/margins": 9.62315845489502,
      "rewards/rejected": -11.00156021118164,
      "step": 287
    },
    {
      "epoch": 3.413333333333333,
      "grad_norm": 5.696220945908767,
      "learning_rate": 3.052362432900332e-08,
      "logits/chosen": -0.4203820526599884,
      "logits/rejected": -0.3877807557582855,
      "logps/chosen": -37.054630279541016,
      "logps/rejected": -61.031986236572266,
      "loss": 0.0281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9897009134292603,
      "rewards/margins": 7.574882984161377,
      "rewards/rejected": -8.564583778381348,
      "step": 288
    },
    {
      "epoch": 3.4251851851851853,
      "grad_norm": 5.040781372610454,
      "learning_rate": 2.9290239552295538e-08,
      "logits/chosen": -0.04420602694153786,
      "logits/rejected": -0.0779787227511406,
      "logps/chosen": -49.03828811645508,
      "logps/rejected": -64.86919403076172,
      "loss": 0.0233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1655118465423584,
      "rewards/margins": 7.464972496032715,
      "rewards/rejected": -8.630484580993652,
      "step": 289
    },
    {
      "epoch": 3.437037037037037,
      "grad_norm": 6.3238698051370825,
      "learning_rate": 2.8080738578703052e-08,
      "logits/chosen": -0.2396240085363388,
      "logits/rejected": -0.18526090681552887,
      "logps/chosen": -35.554222106933594,
      "logps/rejected": -80.34398651123047,
      "loss": 0.0385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15909190475940704,
      "rewards/margins": 11.187956809997559,
      "rewards/rejected": -11.347049713134766,
      "step": 290
    },
    {
      "epoch": 3.448888888888889,
      "grad_norm": 6.096416193339527,
      "learning_rate": 2.6895252292709974e-08,
      "logits/chosen": -0.3143896460533142,
      "logits/rejected": -0.2961388826370239,
      "logps/chosen": -45.88547897338867,
      "logps/rejected": -72.7055892944336,
      "loss": 0.0357,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5558788776397705,
      "rewards/margins": 7.8544135093688965,
      "rewards/rejected": -9.41029167175293,
      "step": 291
    },
    {
      "epoch": 3.4607407407407407,
      "grad_norm": 6.109539533433742,
      "learning_rate": 2.5733908980083984e-08,
      "logits/chosen": -0.20717650651931763,
      "logits/rejected": -0.1371382474899292,
      "logps/chosen": -34.01182556152344,
      "logps/rejected": -69.25910949707031,
      "loss": 0.0275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.003832221031189,
      "rewards/margins": 6.742988586425781,
      "rewards/rejected": -7.74682092666626,
      "step": 292
    },
    {
      "epoch": 3.4725925925925925,
      "grad_norm": 5.397666624928578,
      "learning_rate": 2.4596834313994037e-08,
      "logits/chosen": -0.2090422511100769,
      "logits/rejected": -0.1905803680419922,
      "logps/chosen": -37.257659912109375,
      "logps/rejected": -59.028656005859375,
      "loss": 0.0267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.009922012686729431,
      "rewards/margins": 7.401028633117676,
      "rewards/rejected": -7.410951614379883,
      "step": 293
    },
    {
      "epoch": 3.4844444444444447,
      "grad_norm": 4.277274237028717,
      "learning_rate": 2.3484151341411018e-08,
      "logits/chosen": -0.28495097160339355,
      "logits/rejected": -0.13665924966335297,
      "logps/chosen": -30.814395904541016,
      "logps/rejected": -71.38455963134766,
      "loss": 0.0167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.883634090423584,
      "rewards/margins": 8.096288681030273,
      "rewards/rejected": -8.979923248291016,
      "step": 294
    },
    {
      "epoch": 3.4962962962962965,
      "grad_norm": 5.969698336918465,
      "learning_rate": 2.23959804697921e-08,
      "logits/chosen": 0.0012427568435668945,
      "logits/rejected": -0.05258895084261894,
      "logps/chosen": -43.88676071166992,
      "logps/rejected": -69.91567993164062,
      "loss": 0.0243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0892465114593506,
      "rewards/margins": 8.158609390258789,
      "rewards/rejected": -9.247856140136719,
      "step": 295
    },
    {
      "epoch": 3.5081481481481482,
      "grad_norm": 4.0211602970173095,
      "learning_rate": 2.1332439454051277e-08,
      "logits/chosen": -0.20002827048301697,
      "logits/rejected": -0.1429349184036255,
      "logps/chosen": -34.301002502441406,
      "logps/rejected": -55.76853561401367,
      "loss": 0.0213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09878039360046387,
      "rewards/margins": 6.826313495635986,
      "rewards/rejected": -6.925093650817871,
      "step": 296
    },
    {
      "epoch": 3.52,
      "grad_norm": 5.481157431560272,
      "learning_rate": 2.029364338381656e-08,
      "logits/chosen": -0.38807621598243713,
      "logits/rejected": -0.3801242709159851,
      "logps/chosen": -46.397727966308594,
      "logps/rejected": -55.0689582824707,
      "loss": 0.0289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3208569288253784,
      "rewards/margins": 5.903377056121826,
      "rewards/rejected": -6.224234580993652,
      "step": 297
    },
    {
      "epoch": 3.531851851851852,
      "grad_norm": 5.352166370096892,
      "learning_rate": 1.9279704670975726e-08,
      "logits/chosen": -0.2355162501335144,
      "logits/rejected": -0.07986889034509659,
      "logps/chosen": -34.6320686340332,
      "logps/rejected": -71.03166198730469,
      "loss": 0.0233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7058801651000977,
      "rewards/margins": 7.573249816894531,
      "rewards/rejected": -8.279129981994629,
      "step": 298
    },
    {
      "epoch": 3.5437037037037036,
      "grad_norm": 5.786741095008315,
      "learning_rate": 1.829073303751172e-08,
      "logits/chosen": -0.3151942193508148,
      "logits/rejected": -0.33904606103897095,
      "logps/chosen": -29.33087921142578,
      "logps/rejected": -61.94608688354492,
      "loss": 0.0277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2769862413406372,
      "rewards/margins": 8.623825073242188,
      "rewards/rejected": -9.900812149047852,
      "step": 299
    },
    {
      "epoch": 3.5555555555555554,
      "grad_norm": 4.36518877931545,
      "learning_rate": 1.732683550362954e-08,
      "logits/chosen": -0.23127204179763794,
      "logits/rejected": -0.15411251783370972,
      "logps/chosen": -50.591552734375,
      "logps/rejected": -77.29461669921875,
      "loss": 0.0216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3554518222808838,
      "rewards/margins": 7.547949314117432,
      "rewards/rejected": -8.903400421142578,
      "step": 300
    },
    {
      "epoch": 3.5674074074074076,
      "grad_norm": 4.847754834407613,
      "learning_rate": 1.6388116376174765e-08,
      "logits/chosen": -0.3548241853713989,
      "logits/rejected": -0.2721732556819916,
      "logps/chosen": -38.101295471191406,
      "logps/rejected": -78.74886322021484,
      "loss": 0.0229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5179784297943115,
      "rewards/margins": 8.753091812133789,
      "rewards/rejected": -10.27107048034668,
      "step": 301
    },
    {
      "epoch": 3.5792592592592594,
      "grad_norm": 7.781488916643506,
      "learning_rate": 1.5474677237346468e-08,
      "logits/chosen": -0.3061152994632721,
      "logits/rejected": -0.3108983337879181,
      "logps/chosen": -41.504512786865234,
      "logps/rejected": -78.45973205566406,
      "loss": 0.0451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2482175827026367,
      "rewards/margins": 8.759658813476562,
      "rewards/rejected": -10.0078763961792,
      "step": 302
    },
    {
      "epoch": 3.591111111111111,
      "grad_norm": 6.94980194588748,
      "learning_rate": 1.4586616933704527e-08,
      "logits/chosen": -0.018258891999721527,
      "logits/rejected": -0.007322182413190603,
      "logps/chosen": -54.701812744140625,
      "logps/rejected": -73.30741882324219,
      "loss": 0.0412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3399360179901123,
      "rewards/margins": 6.371276378631592,
      "rewards/rejected": -7.711213111877441,
      "step": 303
    },
    {
      "epoch": 3.602962962962963,
      "grad_norm": 6.423577076131564,
      "learning_rate": 1.372403156547311e-08,
      "logits/chosen": -0.43270713090896606,
      "logits/rejected": -0.36236703395843506,
      "logps/chosen": -33.90000534057617,
      "logps/rejected": -60.915306091308594,
      "loss": 0.0339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.318174958229065,
      "rewards/margins": 7.222588062286377,
      "rewards/rejected": -8.540762901306152,
      "step": 304
    },
    {
      "epoch": 3.6148148148148147,
      "grad_norm": 4.091095308871669,
      "learning_rate": 1.2887014476141212e-08,
      "logits/chosen": -0.30502232909202576,
      "logits/rejected": -0.36349910497665405,
      "logps/chosen": -40.1556510925293,
      "logps/rejected": -68.93075561523438,
      "loss": 0.0244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.034249335527420044,
      "rewards/margins": 9.958725929260254,
      "rewards/rejected": -9.992975234985352,
      "step": 305
    },
    {
      "epoch": 3.626666666666667,
      "grad_norm": 6.177303284695512,
      "learning_rate": 1.2075656242361732e-08,
      "logits/chosen": -0.2732085585594177,
      "logits/rejected": -0.1902616024017334,
      "logps/chosen": -34.206607818603516,
      "logps/rejected": -68.56214904785156,
      "loss": 0.0255,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0075308084487915,
      "rewards/margins": 8.230666160583496,
      "rewards/rejected": -9.238195419311523,
      "step": 306
    },
    {
      "epoch": 3.6385185185185183,
      "grad_norm": 4.816450379471901,
      "learning_rate": 1.1290044664149873e-08,
      "logits/chosen": -0.11257211118936539,
      "logits/rejected": -0.13122713565826416,
      "logps/chosen": -48.688873291015625,
      "logps/rejected": -74.23355102539062,
      "loss": 0.021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2951655387878418,
      "rewards/margins": 8.786190032958984,
      "rewards/rejected": -10.081355094909668,
      "step": 307
    },
    {
      "epoch": 3.6503703703703705,
      "grad_norm": 6.851539041964001,
      "learning_rate": 1.0530264755381824e-08,
      "logits/chosen": -0.4197385013103485,
      "logits/rejected": -0.43813198804855347,
      "logps/chosen": -33.74197769165039,
      "logps/rejected": -56.243553161621094,
      "loss": 0.0363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6660629510879517,
      "rewards/margins": 5.48460054397583,
      "rewards/rejected": -6.150663375854492,
      "step": 308
    },
    {
      "epoch": 3.6622222222222223,
      "grad_norm": 4.955677182707875,
      "learning_rate": 9.796398734595284e-09,
      "logits/chosen": -0.26788032054901123,
      "logits/rejected": -0.26230883598327637,
      "logps/chosen": -28.0145206451416,
      "logps/rejected": -51.17897033691406,
      "loss": 0.0243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11930375546216965,
      "rewards/margins": 5.247139930725098,
      "rewards/rejected": -5.366443634033203,
      "step": 309
    },
    {
      "epoch": 3.674074074074074,
      "grad_norm": 5.676351317770164,
      "learning_rate": 9.088526016092141e-09,
      "logits/chosen": -0.3300250470638275,
      "logits/rejected": -0.31215977668762207,
      "logps/chosen": -33.89133834838867,
      "logps/rejected": -67.8143539428711,
      "loss": 0.033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08850759267807007,
      "rewards/margins": 9.25622272491455,
      "rewards/rejected": -9.344730377197266,
      "step": 310
    },
    {
      "epoch": 3.685925925925926,
      "grad_norm": 5.070558881430833,
      "learning_rate": 8.40672320134489e-09,
      "logits/chosen": -0.305147260427475,
      "logits/rejected": -0.17320549488067627,
      "logps/chosen": -33.947872161865234,
      "logps/rejected": -74.02790832519531,
      "loss": 0.0242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0814896896481514,
      "rewards/margins": 9.62078857421875,
      "rewards/rejected": -9.539299011230469,
      "step": 311
    },
    {
      "epoch": 3.6977777777777776,
      "grad_norm": 4.665245148624672,
      "learning_rate": 7.751064070707247e-09,
      "logits/chosen": -0.44625863432884216,
      "logits/rejected": -0.4555772542953491,
      "logps/chosen": -42.39094161987305,
      "logps/rejected": -67.60736846923828,
      "loss": 0.0215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6547796130180359,
      "rewards/margins": 7.189840793609619,
      "rewards/rejected": -7.844620227813721,
      "step": 312
    },
    {
      "epoch": 3.70962962962963,
      "grad_norm": 7.083642027360033,
      "learning_rate": 7.12161957543006e-09,
      "logits/chosen": -0.251234769821167,
      "logits/rejected": -0.13808919489383698,
      "logps/chosen": -54.0128288269043,
      "logps/rejected": -92.3724365234375,
      "loss": 0.035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6177242994308472,
      "rewards/margins": 8.311200141906738,
      "rewards/rejected": -9.928923606872559,
      "step": 313
    },
    {
      "epoch": 3.7214814814814816,
      "grad_norm": 4.713181664530663,
      "learning_rate": 6.518457829983559e-09,
      "logits/chosen": -0.3703707158565521,
      "logits/rejected": -0.2648147940635681,
      "logps/chosen": -48.93006896972656,
      "logps/rejected": -66.18186950683594,
      "loss": 0.0258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7085801362991333,
      "rewards/margins": 5.302105903625488,
      "rewards/rejected": -6.010685920715332,
      "step": 314
    },
    {
      "epoch": 3.7333333333333334,
      "grad_norm": 4.802422252346654,
      "learning_rate": 5.9416441046862555e-09,
      "logits/chosen": -0.3589542508125305,
      "logits/rejected": -0.3287428319454193,
      "logps/chosen": -27.614389419555664,
      "logps/rejected": -59.38009262084961,
      "loss": 0.0301,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17799943685531616,
      "rewards/margins": 7.3723649978637695,
      "rewards/rejected": -7.550364017486572,
      "step": 315
    },
    {
      "epoch": 3.745185185185185,
      "grad_norm": 5.504211939139539,
      "learning_rate": 5.3912408186420064e-09,
      "logits/chosen": -0.19225972890853882,
      "logits/rejected": -0.24219948053359985,
      "logps/chosen": -39.883174896240234,
      "logps/rejected": -57.19430160522461,
      "loss": 0.026,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7933384776115417,
      "rewards/margins": 7.422837257385254,
      "rewards/rejected": -8.21617603302002,
      "step": 316
    },
    {
      "epoch": 3.757037037037037,
      "grad_norm": 5.560326027847558,
      "learning_rate": 4.867307532985227e-09,
      "logits/chosen": -0.48098868131637573,
      "logits/rejected": -0.3568829894065857,
      "logps/chosen": -54.420379638671875,
      "logps/rejected": -85.24766540527344,
      "loss": 0.0297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9853860139846802,
      "rewards/margins": 7.260770320892334,
      "rewards/rejected": -9.246155738830566,
      "step": 317
    },
    {
      "epoch": 3.7688888888888887,
      "grad_norm": 7.196198216327019,
      "learning_rate": 4.369900944435734e-09,
      "logits/chosen": -0.16600579023361206,
      "logits/rejected": -0.0450252965092659,
      "logps/chosen": -42.486473083496094,
      "logps/rejected": -81.37212371826172,
      "loss": 0.0389,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9354689121246338,
      "rewards/margins": 7.651963233947754,
      "rewards/rejected": -8.587431907653809,
      "step": 318
    },
    {
      "epoch": 3.7807407407407405,
      "grad_norm": 6.279239735558416,
      "learning_rate": 3.899074879163244e-09,
      "logits/chosen": -0.40112194418907166,
      "logits/rejected": -0.339353084564209,
      "logps/chosen": -34.84368133544922,
      "logps/rejected": -61.614341735839844,
      "loss": 0.0348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0974161624908447,
      "rewards/margins": 7.075118064880371,
      "rewards/rejected": -8.172533988952637,
      "step": 319
    },
    {
      "epoch": 3.7925925925925927,
      "grad_norm": 6.9892724428640705,
      "learning_rate": 3.4548802869627804e-09,
      "logits/chosen": -0.3009003698825836,
      "logits/rejected": -0.21556389331817627,
      "logps/chosen": -41.47652816772461,
      "logps/rejected": -67.13237762451172,
      "loss": 0.0368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2825208604335785,
      "rewards/margins": 4.948946952819824,
      "rewards/rejected": -5.231468200683594,
      "step": 320
    },
    {
      "epoch": 3.8044444444444445,
      "grad_norm": 6.856578977189586,
      "learning_rate": 3.037365235741024e-09,
      "logits/chosen": -0.16049635410308838,
      "logits/rejected": -0.12033607065677643,
      "logps/chosen": -32.831031799316406,
      "logps/rejected": -62.16967010498047,
      "loss": 0.0433,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.41414308547973633,
      "rewards/margins": 7.311939239501953,
      "rewards/rejected": -7.726081371307373,
      "step": 321
    },
    {
      "epoch": 3.8162962962962963,
      "grad_norm": 9.951539288327684,
      "learning_rate": 2.6465749063149245e-09,
      "logits/chosen": -0.6789449453353882,
      "logits/rejected": -0.6332409381866455,
      "logps/chosen": -36.879947662353516,
      "logps/rejected": -80.12794494628906,
      "loss": 0.0538,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3466782569885254,
      "rewards/margins": 9.836599349975586,
      "rewards/rejected": -11.183279037475586,
      "step": 322
    },
    {
      "epoch": 3.828148148148148,
      "grad_norm": 6.85068630218398,
      "learning_rate": 2.282551587522441e-09,
      "logits/chosen": -0.5462524890899658,
      "logits/rejected": -0.43087050318717957,
      "logps/chosen": -32.245567321777344,
      "logps/rejected": -56.410179138183594,
      "loss": 0.0455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1382488012313843,
      "rewards/margins": 6.717283248901367,
      "rewards/rejected": -7.855532169342041,
      "step": 323
    },
    {
      "epoch": 3.84,
      "grad_norm": 6.505661948537336,
      "learning_rate": 1.9453346716462316e-09,
      "logits/chosen": -0.3759889602661133,
      "logits/rejected": -0.4021185040473938,
      "logps/chosen": -36.99543380737305,
      "logps/rejected": -46.82232666015625,
      "loss": 0.0393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.881350576877594,
      "rewards/margins": 4.79249382019043,
      "rewards/rejected": -5.673844337463379,
      "step": 324
    },
    {
      "epoch": 3.851851851851852,
      "grad_norm": 6.937575318919963,
      "learning_rate": 1.6349606501509794e-09,
      "logits/chosen": -0.2532769441604614,
      "logits/rejected": -0.2831670045852661,
      "logps/chosen": -41.752403259277344,
      "logps/rejected": -52.2098388671875,
      "loss": 0.0363,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7358890771865845,
      "rewards/margins": 6.947170734405518,
      "rewards/rejected": -7.6830596923828125,
      "step": 325
    },
    {
      "epoch": 3.863703703703704,
      "grad_norm": 3.940918067532272,
      "learning_rate": 1.351463109734441e-09,
      "logits/chosen": -0.6125096082687378,
      "logits/rejected": -0.3610725700855255,
      "logps/chosen": -36.78853225708008,
      "logps/rejected": -68.35499572753906,
      "loss": 0.0198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8846864700317383,
      "rewards/margins": 8.920228004455566,
      "rewards/rejected": -9.804913520812988,
      "step": 326
    },
    {
      "epoch": 3.8755555555555556,
      "grad_norm": 5.731819862714682,
      "learning_rate": 1.0948727286930192e-09,
      "logits/chosen": -0.07925964891910553,
      "logits/rejected": -0.04111175611615181,
      "logps/chosen": -35.71060562133789,
      "logps/rejected": -57.488651275634766,
      "loss": 0.0309,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09710303694009781,
      "rewards/margins": 6.214367866516113,
      "rewards/rejected": -6.311470985412598,
      "step": 327
    },
    {
      "epoch": 3.8874074074074074,
      "grad_norm": 7.4093470103902845,
      "learning_rate": 8.652172736017816e-10,
      "logits/chosen": -0.2285485416650772,
      "logits/rejected": -0.22180640697479248,
      "logps/chosen": -45.58677673339844,
      "logps/rejected": -74.19960021972656,
      "loss": 0.0412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.34262484312057495,
      "rewards/margins": 6.549678325653076,
      "rewards/rejected": -6.892302989959717,
      "step": 328
    },
    {
      "epoch": 3.899259259259259,
      "grad_norm": 5.597791664125433,
      "learning_rate": 6.625215963098896e-10,
      "logits/chosen": -0.22412040829658508,
      "logits/rejected": -0.25815126299858093,
      "logps/chosen": -39.15158462524414,
      "logps/rejected": -52.841060638427734,
      "loss": 0.0284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1085604429244995,
      "rewards/margins": 5.7370734214782715,
      "rewards/rejected": -6.845633506774902,
      "step": 329
    },
    {
      "epoch": 3.911111111111111,
      "grad_norm": 4.652252051429055,
      "learning_rate": 4.868076312512515e-10,
      "logits/chosen": -0.3777186870574951,
      "logits/rejected": -0.3193652629852295,
      "logps/chosen": -32.07898712158203,
      "logps/rejected": -64.57249450683594,
      "loss": 0.0241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5756787657737732,
      "rewards/margins": 7.689653396606445,
      "rewards/rejected": -8.265332221984863,
      "step": 330
    },
    {
      "epoch": 3.9229629629629628,
      "grad_norm": 5.81932820439485,
      "learning_rate": 3.3809439307086463e-10,
      "logits/chosen": -0.20917584002017975,
      "logits/rejected": -0.12255613505840302,
      "logps/chosen": -31.15184783935547,
      "logps/rejected": -63.176910400390625,
      "loss": 0.0335,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12342405319213867,
      "rewards/margins": 6.832948684692383,
      "rewards/rejected": -6.956372261047363,
      "step": 331
    },
    {
      "epoch": 3.934814814814815,
      "grad_norm": 6.583884307182356,
      "learning_rate": 2.1639797456723952e-10,
      "logits/chosen": -0.3113446831703186,
      "logits/rejected": -0.34424495697021484,
      "logps/chosen": -50.414649963378906,
      "logps/rejected": -66.15986633300781,
      "loss": 0.0305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8464060425758362,
      "rewards/margins": 7.252386093139648,
      "rewards/rejected": -8.098793029785156,
      "step": 332
    },
    {
      "epoch": 3.9466666666666668,
      "grad_norm": 5.902458355798929,
      "learning_rate": 1.21731544950876e-10,
      "logits/chosen": -0.23517660796642303,
      "logits/rejected": -0.3104686737060547,
      "logps/chosen": -45.34723663330078,
      "logps/rejected": -86.45767974853516,
      "loss": 0.0269,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3326948881149292,
      "rewards/margins": 9.871392250061035,
      "rewards/rejected": -11.204087257385254,
      "step": 333
    },
    {
      "epoch": 3.9585185185185185,
      "grad_norm": 5.025564856386988,
      "learning_rate": 5.4105348419264394e-11,
      "logits/chosen": -0.6560889482498169,
      "logits/rejected": -0.6381913423538208,
      "logps/chosen": -31.789920806884766,
      "logps/rejected": -57.232948303222656,
      "loss": 0.0281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.43755847215652466,
      "rewards/margins": 6.743692398071289,
      "rewards/rejected": -7.181251525878906,
      "step": 334
    },
    {
      "epoch": 3.9703703703703703,
      "grad_norm": 4.328140885361808,
      "learning_rate": 1.3526703048216682e-11,
      "logits/chosen": -0.5039613246917725,
      "logits/rejected": -0.4086998999118805,
      "logps/chosen": -34.5649528503418,
      "logps/rejected": -85.53738403320312,
      "loss": 0.0214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7040150165557861,
      "rewards/margins": 11.698722839355469,
      "rewards/rejected": -12.402737617492676,
      "step": 335
    },
    {
      "epoch": 3.982222222222222,
      "grad_norm": 5.989193672621692,
      "learning_rate": 0.0,
      "logits/chosen": -0.3169300854206085,
      "logits/rejected": -0.32203078269958496,
      "logps/chosen": -37.71720886230469,
      "logps/rejected": -64.16942596435547,
      "loss": 0.0329,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6902213096618652,
      "rewards/margins": 7.074714183807373,
      "rewards/rejected": -7.76493501663208,
      "step": 336
    },
    {
      "epoch": 3.982222222222222,
      "step": 336,
      "total_flos": 0.0,
      "train_loss": 0.14947671072912358,
      "train_runtime": 64472.8667,
      "train_samples_per_second": 0.67,
      "train_steps_per_second": 0.005
    }
  ],
  "logging_steps": 1,
  "max_steps": 336,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 200,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}