{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 300,
  "global_step": 3821,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.3054830287206268e-08,
      "logits/chosen": -2.7604618072509766,
      "logits/rejected": -2.686812162399292,
      "logps/chosen": -516.73779296875,
      "logps/rejected": -458.60467529296875,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.3054830287206266e-07,
      "logits/chosen": -2.4880800247192383,
      "logits/rejected": -2.4930832386016846,
      "logps/chosen": -338.7858581542969,
      "logps/rejected": -404.5611572265625,
      "loss": 0.6929,
      "rewards/accuracies": 0.4027777910232544,
      "rewards/chosen": 0.00024087271594908088,
      "rewards/margins": 0.0006852700607851148,
      "rewards/rejected": -0.00044439738849177957,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.610966057441253e-07,
      "logits/chosen": -2.475435733795166,
      "logits/rejected": -2.4197583198547363,
      "logps/chosen": -327.35919189453125,
      "logps/rejected": -443.83868408203125,
      "loss": 0.6931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.7859845178900287e-05,
      "rewards/margins": 1.1227629329368938e-05,
      "rewards/rejected": -4.9087520892499015e-05,
      "step": 20
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.9164490861618804e-07,
      "logits/chosen": -2.5618598461151123,
      "logits/rejected": -2.5939595699310303,
      "logps/chosen": -348.56982421875,
      "logps/rejected": -416.8001403808594,
      "loss": 0.692,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.00023523520212620497,
      "rewards/margins": 0.0015929860528558493,
      "rewards/rejected": -0.0013577509671449661,
      "step": 30
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.221932114882506e-07,
      "logits/chosen": -2.5246434211730957,
      "logits/rejected": -2.4987733364105225,
      "logps/chosen": -376.7454528808594,
      "logps/rejected": -427.66729736328125,
      "loss": 0.692,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0012155056465417147,
      "rewards/margins": 0.0037051704712212086,
      "rewards/rejected": -0.00492067588493228,
      "step": 40
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.527415143603135e-07,
      "logits/chosen": -2.4959325790405273,
      "logits/rejected": -2.4452037811279297,
      "logps/chosen": -290.552001953125,
      "logps/rejected": -383.3431701660156,
      "loss": 0.69,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0027376641519367695,
      "rewards/margins": 0.005442826543003321,
      "rewards/rejected": -0.008180489763617516,
      "step": 50
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.832898172323761e-07,
      "logits/chosen": -2.4295783042907715,
      "logits/rejected": -2.3901355266571045,
      "logps/chosen": -377.3544006347656,
      "logps/rejected": -410.72991943359375,
      "loss": 0.6909,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.006509931292384863,
      "rewards/margins": 0.004867006093263626,
      "rewards/rejected": -0.011376937851309776,
      "step": 60
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.138381201044387e-07,
      "logits/chosen": -2.310166835784912,
      "logits/rejected": -2.279524803161621,
      "logps/chosen": -279.5904846191406,
      "logps/rejected": -370.0677795410156,
      "loss": 0.6889,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.007495372090488672,
      "rewards/margins": 0.007953675463795662,
      "rewards/rejected": -0.015449047088623047,
      "step": 70
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0443864229765013e-06,
      "logits/chosen": -2.36948561668396,
      "logits/rejected": -2.3835222721099854,
      "logps/chosen": -342.13653564453125,
      "logps/rejected": -447.1036682128906,
      "loss": 0.6878,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.010767060332000256,
      "rewards/margins": 0.013605493120849133,
      "rewards/rejected": -0.024372553452849388,
      "step": 80
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.1749347258485642e-06,
      "logits/chosen": -2.472949266433716,
      "logits/rejected": -2.3902525901794434,
      "logps/chosen": -325.2154541015625,
      "logps/rejected": -401.51751708984375,
      "loss": 0.6844,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.012460511177778244,
      "rewards/margins": 0.01636466197669506,
      "rewards/rejected": -0.028825175017118454,
      "step": 90
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.305483028720627e-06,
      "logits/chosen": -2.5286309719085693,
      "logits/rejected": -2.5337207317352295,
      "logps/chosen": -365.7882080078125,
      "logps/rejected": -409.24261474609375,
      "loss": 0.6836,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.017355522140860558,
      "rewards/margins": 0.019160564988851547,
      "rewards/rejected": -0.036516088992357254,
      "step": 100
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.4360313315926894e-06,
      "logits/chosen": -2.485241651535034,
      "logits/rejected": -2.473548412322998,
      "logps/chosen": -337.1002197265625,
      "logps/rejected": -444.09832763671875,
      "loss": 0.6793,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.023945586755871773,
      "rewards/margins": 0.02508346177637577,
      "rewards/rejected": -0.049029044806957245,
      "step": 110
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5665796344647521e-06,
      "logits/chosen": -2.437514066696167,
      "logits/rejected": -2.439671516418457,
      "logps/chosen": -343.27777099609375,
      "logps/rejected": -444.11639404296875,
      "loss": 0.6747,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.028381770476698875,
      "rewards/margins": 0.039804860949516296,
      "rewards/rejected": -0.06818662583827972,
      "step": 120
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.6971279373368146e-06,
      "logits/chosen": -2.4196202754974365,
      "logits/rejected": -2.329251527786255,
      "logps/chosen": -380.3607482910156,
      "logps/rejected": -435.93896484375,
      "loss": 0.6724,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.03702790290117264,
      "rewards/margins": 0.049176327884197235,
      "rewards/rejected": -0.08620421588420868,
      "step": 130
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8276762402088774e-06,
      "logits/chosen": -2.436796188354492,
      "logits/rejected": -2.4008259773254395,
      "logps/chosen": -364.23687744140625,
      "logps/rejected": -456.6722106933594,
      "loss": 0.6653,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.04911624267697334,
      "rewards/margins": 0.06083670258522034,
      "rewards/rejected": -0.10995294898748398,
      "step": 140
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9582245430809403e-06,
      "logits/chosen": -2.465663433074951,
      "logits/rejected": -2.4756152629852295,
      "logps/chosen": -344.59808349609375,
      "logps/rejected": -450.5936584472656,
      "loss": 0.6615,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.07244648039340973,
      "rewards/margins": 0.07592395693063736,
      "rewards/rejected": -0.1483704298734665,
      "step": 150
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0887728459530026e-06,
      "logits/chosen": -2.365227222442627,
      "logits/rejected": -2.3483099937438965,
      "logps/chosen": -365.65509033203125,
      "logps/rejected": -459.9405822753906,
      "loss": 0.6494,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.088950976729393,
      "rewards/margins": 0.0978541225194931,
      "rewards/rejected": -0.1868050992488861,
      "step": 160
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.2193211488250653e-06,
      "logits/chosen": -2.5415358543395996,
      "logits/rejected": -2.4844305515289307,
      "logps/chosen": -417.45013427734375,
      "logps/rejected": -502.04888916015625,
      "loss": 0.6513,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.12002843618392944,
      "rewards/margins": 0.09641442447900772,
      "rewards/rejected": -0.21644285321235657,
      "step": 170
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.3498694516971284e-06,
      "logits/chosen": -2.373419761657715,
      "logits/rejected": -2.328564167022705,
      "logps/chosen": -308.98297119140625,
      "logps/rejected": -423.2518005371094,
      "loss": 0.6334,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.10061755031347275,
      "rewards/margins": 0.14423246681690216,
      "rewards/rejected": -0.2448500096797943,
      "step": 180
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4804177545691907e-06,
      "logits/chosen": -2.4381210803985596,
      "logits/rejected": -2.4739632606506348,
      "logps/chosen": -366.0082092285156,
      "logps/rejected": -444.33233642578125,
      "loss": 0.6237,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.15544722974300385,
      "rewards/margins": 0.15099851787090302,
      "rewards/rejected": -0.30644577741622925,
      "step": 190
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.610966057441254e-06,
      "logits/chosen": -2.3546931743621826,
      "logits/rejected": -2.3614845275878906,
      "logps/chosen": -413.4657287597656,
      "logps/rejected": -487.9469299316406,
      "loss": 0.6049,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.21883895993232727,
      "rewards/margins": 0.22747401893138885,
      "rewards/rejected": -0.44631296396255493,
      "step": 200
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.741514360313316e-06,
      "logits/chosen": -2.3287081718444824,
      "logits/rejected": -2.2180962562561035,
      "logps/chosen": -397.4319763183594,
      "logps/rejected": -475.10614013671875,
      "loss": 0.6212,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2562285363674164,
      "rewards/margins": 0.17265436053276062,
      "rewards/rejected": -0.4288829267024994,
      "step": 210
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.872062663185379e-06,
      "logits/chosen": -2.3937861919403076,
      "logits/rejected": -2.323655366897583,
      "logps/chosen": -393.7010498046875,
      "logps/rejected": -540.32958984375,
      "loss": 0.5708,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.31672877073287964,
      "rewards/margins": 0.32215583324432373,
      "rewards/rejected": -0.6388846039772034,
      "step": 220
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.0026109660574416e-06,
      "logits/chosen": -2.2199885845184326,
      "logits/rejected": -2.1488921642303467,
      "logps/chosen": -362.84710693359375,
      "logps/rejected": -462.92755126953125,
      "loss": 0.57,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.4027708172798157,
      "rewards/margins": 0.28824615478515625,
      "rewards/rejected": -0.6910169124603271,
      "step": 230
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1331592689295043e-06,
      "logits/chosen": -2.1226742267608643,
      "logits/rejected": -2.128727674484253,
      "logps/chosen": -443.38226318359375,
      "logps/rejected": -544.7005615234375,
      "loss": 0.5704,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.5918055176734924,
      "rewards/margins": 0.33963826298713684,
      "rewards/rejected": -0.9314438104629517,
      "step": 240
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.263707571801567e-06,
      "logits/chosen": -2.2666714191436768,
      "logits/rejected": -2.192073106765747,
      "logps/chosen": -407.093505859375,
      "logps/rejected": -539.2747802734375,
      "loss": 0.53,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.6421019434928894,
      "rewards/margins": 0.4279584288597107,
      "rewards/rejected": -1.0700603723526,
      "step": 250
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.3942558746736293e-06,
      "logits/chosen": -2.1499412059783936,
      "logits/rejected": -2.0672426223754883,
      "logps/chosen": -427.7845764160156,
      "logps/rejected": -526.4454956054688,
      "loss": 0.5201,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.7411731481552124,
      "rewards/margins": 0.33993035554885864,
      "rewards/rejected": -1.0811034440994263,
      "step": 260
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.524804177545692e-06,
      "logits/chosen": -2.0998997688293457,
      "logits/rejected": -2.022254467010498,
      "logps/chosen": -459.3646545410156,
      "logps/rejected": -625.1860961914062,
      "loss": 0.5132,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.9105283617973328,
      "rewards/margins": 0.5623170137405396,
      "rewards/rejected": -1.472845435142517,
      "step": 270
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.6553524804177547e-06,
      "logits/chosen": -2.058182716369629,
      "logits/rejected": -1.990330457687378,
      "logps/chosen": -422.2322692871094,
      "logps/rejected": -509.48504638671875,
      "loss": 0.5447,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.8607247471809387,
      "rewards/margins": 0.5177969932556152,
      "rewards/rejected": -1.3785216808319092,
      "step": 280
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.7859007832898174e-06,
      "logits/chosen": -1.908735990524292,
      "logits/rejected": -1.7987762689590454,
      "logps/chosen": -532.5482788085938,
      "logps/rejected": -675.1096801757812,
      "loss": 0.4906,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1365267038345337,
      "rewards/margins": 0.7340434789657593,
      "rewards/rejected": -1.870570421218872,
      "step": 290
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.9164490861618806e-06,
      "logits/chosen": -1.999489188194275,
      "logits/rejected": -1.8826462030410767,
      "logps/chosen": -510.0675354003906,
      "logps/rejected": -654.4682006835938,
      "loss": 0.4906,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1803276538848877,
      "rewards/margins": 0.692645788192749,
      "rewards/rejected": -1.8729734420776367,
      "step": 300
    },
    {
      "epoch": 0.08,
      "eval_logits/chosen": -1.7535996437072754,
      "eval_logits/rejected": -1.6233811378479004,
      "eval_logps/chosen": -473.8013610839844,
      "eval_logps/rejected": -603.2532958984375,
      "eval_loss": 0.5339562892913818,
      "eval_rewards/accuracies": 0.7310000061988831,
      "eval_rewards/chosen": -1.181405782699585,
      "eval_rewards/margins": 0.661102831363678,
      "eval_rewards/rejected": -1.8425085544586182,
      "eval_runtime": 1389.4464,
      "eval_samples_per_second": 1.439,
      "eval_steps_per_second": 0.36,
      "step": 300
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.046997389033943e-06,
      "logits/chosen": -1.8269140720367432,
      "logits/rejected": -1.690843939781189,
      "logps/chosen": -498.0406799316406,
      "logps/rejected": -633.5381469726562,
      "loss": 0.5566,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.2140557765960693,
      "rewards/margins": 0.597601592540741,
      "rewards/rejected": -1.811657190322876,
      "step": 310
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.177545691906005e-06,
      "logits/chosen": -1.767525315284729,
      "logits/rejected": -1.5722942352294922,
      "logps/chosen": -499.41839599609375,
      "logps/rejected": -658.9512329101562,
      "loss": 0.5073,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2438334226608276,
      "rewards/margins": 0.7738653421401978,
      "rewards/rejected": -2.0176987648010254,
      "step": 320
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.308093994778068e-06,
      "logits/chosen": -1.8579235076904297,
      "logits/rejected": -1.7731454372406006,
      "logps/chosen": -443.87689208984375,
      "logps/rejected": -572.9539184570312,
      "loss": 0.563,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.1261751651763916,
      "rewards/margins": 0.6167136430740356,
      "rewards/rejected": -1.7428886890411377,
      "step": 330
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.4386422976501306e-06,
      "logits/chosen": -1.7547829151153564,
      "logits/rejected": -1.6789696216583252,
      "logps/chosen": -377.6453552246094,
      "logps/rejected": -584.998779296875,
      "loss": 0.4792,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9642307162284851,
      "rewards/margins": 0.8439178466796875,
      "rewards/rejected": -1.8081486225128174,
      "step": 340
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.569190600522193e-06,
      "logits/chosen": -1.6814041137695312,
      "logits/rejected": -1.4768749475479126,
      "logps/chosen": -506.594482421875,
      "logps/rejected": -591.21142578125,
      "loss": 0.5415,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.2887766361236572,
      "rewards/margins": 0.5483629107475281,
      "rewards/rejected": -1.8371394872665405,
      "step": 350
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.699738903394257e-06,
      "logits/chosen": -1.6767174005508423,
      "logits/rejected": -1.5448577404022217,
      "logps/chosen": -424.47357177734375,
      "logps/rejected": -592.1474609375,
      "loss": 0.4852,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.9117814302444458,
      "rewards/margins": 0.8897954225540161,
      "rewards/rejected": -1.801576852798462,
      "step": 360
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.8302872062663196e-06,
      "logits/chosen": -1.7710018157958984,
      "logits/rejected": -1.6090304851531982,
      "logps/chosen": -496.84796142578125,
      "logps/rejected": -633.0993041992188,
      "loss": 0.5396,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.243511438369751,
      "rewards/margins": 0.702240526676178,
      "rewards/rejected": -1.9457519054412842,
      "step": 370
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.9608355091383814e-06,
      "logits/chosen": -1.4778488874435425,
      "logits/rejected": -1.3310272693634033,
      "logps/chosen": -402.7732849121094,
      "logps/rejected": -492.5380859375,
      "loss": 0.5017,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.9786807298660278,
      "rewards/margins": 0.623943567276001,
      "rewards/rejected": -1.6026241779327393,
      "step": 380
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.9999488562447675e-06,
      "logits/chosen": -1.40001380443573,
      "logits/rejected": -1.2227522134780884,
      "logps/chosen": -532.0943603515625,
      "logps/rejected": -664.1284790039062,
      "loss": 0.6002,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.533832311630249,
      "rewards/margins": 0.5622987747192383,
      "rewards/rejected": -2.0961310863494873,
      "step": 390
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999698361256577e-06,
      "logits/chosen": -1.461018443107605,
      "logits/rejected": -1.1940746307373047,
      "logps/chosen": -509.679443359375,
      "logps/rejected": -636.8277587890625,
      "loss": 0.5079,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1675481796264648,
      "rewards/margins": 0.7109408974647522,
      "rewards/rejected": -1.8784888982772827,
      "step": 400
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999239142174581e-06,
      "logits/chosen": -1.3779613971710205,
      "logits/rejected": -1.1279867887496948,
      "logps/chosen": -455.8907165527344,
      "logps/rejected": -604.815185546875,
      "loss": 0.4635,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2613023519515991,
      "rewards/margins": 0.855317234992981,
      "rewards/rejected": -2.11661958694458,
      "step": 410
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.99857123734344e-06,
      "logits/chosen": -1.3608977794647217,
      "logits/rejected": -0.901921272277832,
      "logps/chosen": -503.4825134277344,
      "logps/rejected": -629.4042358398438,
      "loss": 0.4903,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.280937671661377,
      "rewards/margins": 0.8498051762580872,
      "rewards/rejected": -2.1307430267333984,
      "step": 420
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997694702533016e-06,
      "logits/chosen": -1.29677414894104,
      "logits/rejected": -1.0692778825759888,
      "logps/chosen": -548.71826171875,
      "logps/rejected": -724.1248779296875,
      "loss": 0.466,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5205175876617432,
      "rewards/margins": 0.9235566854476929,
      "rewards/rejected": -2.4440743923187256,
      "step": 430
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.996609610933713e-06,
      "logits/chosen": -1.3901933431625366,
      "logits/rejected": -1.1403733491897583,
      "logps/chosen": -505.74139404296875,
      "logps/rejected": -676.5526123046875,
      "loss": 0.5209,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.319435715675354,
      "rewards/margins": 0.8349205255508423,
      "rewards/rejected": -2.1543564796447754,
      "step": 440
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995316053150366e-06,
      "logits/chosen": -1.3645076751708984,
      "logits/rejected": -1.0972566604614258,
      "logps/chosen": -509.98016357421875,
      "logps/rejected": -682.1041259765625,
      "loss": 0.5307,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.315499186515808,
      "rewards/margins": 0.8478175401687622,
      "rewards/rejected": -2.163316488265991,
      "step": 450
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9938141371946815e-06,
      "logits/chosen": -1.3621938228607178,
      "logits/rejected": -0.9139550924301147,
      "logps/chosen": -506.2652893066406,
      "logps/rejected": -685.5975341796875,
      "loss": 0.4647,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.2151587009429932,
      "rewards/margins": 0.93292635679245,
      "rewards/rejected": -2.148085117340088,
      "step": 460
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.992103988476206e-06,
      "logits/chosen": -1.175817847251892,
      "logits/rejected": -0.6028262376785278,
      "logps/chosen": -476.334716796875,
      "logps/rejected": -651.3397216796875,
      "loss": 0.4581,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2715951204299927,
      "rewards/margins": 0.8561094999313354,
      "rewards/rejected": -2.127704381942749,
      "step": 470
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990185749791866e-06,
      "logits/chosen": -0.8731945753097534,
      "logits/rejected": -0.5909063220024109,
      "logps/chosen": -551.8997802734375,
      "logps/rejected": -691.8196411132812,
      "loss": 0.4837,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4918056726455688,
      "rewards/margins": 1.0931782722473145,
      "rewards/rejected": -2.5849835872650146,
      "step": 480
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9880595813140395e-06,
      "logits/chosen": -0.8462217450141907,
      "logits/rejected": -0.4468405246734619,
      "logps/chosen": -490.5753479003906,
      "logps/rejected": -612.1764526367188,
      "loss": 0.4908,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.316733956336975,
      "rewards/margins": 0.7920147180557251,
      "rewards/rejected": -2.108748435974121,
      "step": 490
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.985725660577184e-06,
      "logits/chosen": -0.6493757963180542,
      "logits/rejected": -0.6190133690834045,
      "logps/chosen": -496.7198181152344,
      "logps/rejected": -738.5999145507812,
      "loss": 0.4759,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4519720077514648,
      "rewards/margins": 1.00557541847229,
      "rewards/rejected": -2.457547664642334,
      "step": 500
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.983184182463009e-06,
      "logits/chosen": -0.6732873916625977,
      "logits/rejected": -0.4988276958465576,
      "logps/chosen": -510.90106201171875,
      "logps/rejected": -625.2664794921875,
      "loss": 0.4795,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.3072543144226074,
      "rewards/margins": 0.8223368525505066,
      "rewards/rejected": -2.1295909881591797,
      "step": 510
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.980435359184203e-06,
      "logits/chosen": -1.0567286014556885,
      "logits/rejected": -0.561193585395813,
      "logps/chosen": -564.832275390625,
      "logps/rejected": -718.8677368164062,
      "loss": 0.5194,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5574848651885986,
      "rewards/margins": 0.938727855682373,
      "rewards/rejected": -2.496212959289551,
      "step": 520
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9774794202667236e-06,
      "logits/chosen": -0.9730321168899536,
      "logits/rejected": -0.3585650324821472,
      "logps/chosen": -440.9288024902344,
      "logps/rejected": -633.592041015625,
      "loss": 0.3516,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.1980558633804321,
      "rewards/margins": 1.1983206272125244,
      "rewards/rejected": -2.396376132965088,
      "step": 530
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.974316612530615e-06,
      "logits/chosen": -0.5258805751800537,
      "logits/rejected": -0.13965483009815216,
      "logps/chosen": -500.12091064453125,
      "logps/rejected": -707.7438354492188,
      "loss": 0.4002,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.656359314918518,
      "rewards/margins": 1.194873571395874,
      "rewards/rejected": -2.8512330055236816,
      "step": 540
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.970947200069416e-06,
      "logits/chosen": -0.8342685699462891,
      "logits/rejected": 0.23774346709251404,
      "logps/chosen": -462.1539611816406,
      "logps/rejected": -631.550537109375,
      "loss": 0.5032,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.5009446144104004,
      "rewards/margins": 1.0611143112182617,
      "rewards/rejected": -2.562058925628662,
      "step": 550
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.967371464228096e-06,
      "logits/chosen": -0.44802480936050415,
      "logits/rejected": -0.0694938451051712,
      "logps/chosen": -492.4813537597656,
      "logps/rejected": -663.474365234375,
      "loss": 0.4973,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5952883958816528,
      "rewards/margins": 1.0112850666046143,
      "rewards/rejected": -2.6065733432769775,
      "step": 560
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.963589703579569e-06,
      "logits/chosen": -0.5677907466888428,
      "logits/rejected": 0.08190581947565079,
      "logps/chosen": -503.6414489746094,
      "logps/rejected": -645.163330078125,
      "loss": 0.4839,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4353834390640259,
      "rewards/margins": 0.922812819480896,
      "rewards/rejected": -2.358196496963501,
      "step": 570
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9596022338997615e-06,
      "logits/chosen": -0.5412989854812622,
      "logits/rejected": 0.47167086601257324,
      "logps/chosen": -497.56103515625,
      "logps/rejected": -681.288818359375,
      "loss": 0.4859,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6671890020370483,
      "rewards/margins": 1.1972028017044067,
      "rewards/rejected": -2.864391803741455,
      "step": 580
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.955409388141243e-06,
      "logits/chosen": -0.8228802680969238,
      "logits/rejected": 0.08221787214279175,
      "logps/chosen": -544.5670776367188,
      "logps/rejected": -733.0662231445312,
      "loss": 0.5465,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6897475719451904,
      "rewards/margins": 1.3086159229278564,
      "rewards/rejected": -2.998363971710205,
      "step": 590
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.951011516405429e-06,
      "logits/chosen": -0.3682901859283447,
      "logits/rejected": 0.21380114555358887,
      "logps/chosen": -472.1390686035156,
      "logps/rejected": -650.9225463867188,
      "loss": 0.4794,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.3374059200286865,
      "rewards/margins": 0.9791024923324585,
      "rewards/rejected": -2.3165085315704346,
      "step": 600
    },
    {
      "epoch": 0.16,
      "eval_logits/chosen": 0.4449966549873352,
      "eval_logits/rejected": 1.2460291385650635,
      "eval_logps/chosen": -494.4773254394531,
      "eval_logps/rejected": -666.9945068359375,
      "eval_loss": 0.4700670540332794,
      "eval_rewards/accuracies": 0.7699999809265137,
      "eval_rewards/chosen": -1.38816499710083,
      "eval_rewards/margins": 1.0917549133300781,
      "eval_rewards/rejected": -2.479919910430908,
      "eval_runtime": 1372.1838,
      "eval_samples_per_second": 1.458,
      "eval_steps_per_second": 0.364,
      "step": 600
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.946408985913344e-06,
      "logits/chosen": -1.1787288188934326,
      "logits/rejected": 0.032404374331235886,
      "logps/chosen": -540.3707275390625,
      "logps/rejected": -690.0636596679688,
      "loss": 0.4104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2028136253356934,
      "rewards/margins": 1.3079878091812134,
      "rewards/rejected": -2.510801315307617,
      "step": 610
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.941602180974958e-06,
      "logits/chosen": -0.5976434946060181,
      "logits/rejected": 0.23231466114521027,
      "logps/chosen": -513.5987548828125,
      "logps/rejected": -683.162353515625,
      "loss": 0.4517,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.503683090209961,
      "rewards/margins": 1.2033047676086426,
      "rewards/rejected": -2.7069880962371826,
      "step": 620
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.936591502957101e-06,
      "logits/chosen": -0.6624077558517456,
      "logits/rejected": 0.3608100712299347,
      "logps/chosen": -447.55230712890625,
      "logps/rejected": -660.681640625,
      "loss": 0.3989,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.3714964389801025,
      "rewards/margins": 1.2194106578826904,
      "rewards/rejected": -2.590907335281372,
      "step": 630
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.931377370249946e-06,
      "logits/chosen": -0.27748388051986694,
      "logits/rejected": 0.2729637026786804,
      "logps/chosen": -528.355224609375,
      "logps/rejected": -733.2789916992188,
      "loss": 0.4943,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8842767477035522,
      "rewards/margins": 1.311034917831421,
      "rewards/rejected": -3.1953113079071045,
      "step": 640
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.925960218232073e-06,
      "logits/chosen": -0.4966405928134918,
      "logits/rejected": 0.048351895064115524,
      "logps/chosen": -491.5657653808594,
      "logps/rejected": -743.1395874023438,
      "loss": 0.4957,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.6836191415786743,
      "rewards/margins": 1.473381519317627,
      "rewards/rejected": -3.157000780105591,
      "step": 650
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.920340499234116e-06,
      "logits/chosen": -0.24024248123168945,
      "logits/rejected": -0.06672336161136627,
      "logps/chosen": -472.2447814941406,
      "logps/rejected": -663.1290893554688,
      "loss": 0.4382,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.4003952741622925,
      "rewards/margins": 1.0303130149841309,
      "rewards/rejected": -2.430708408355713,
      "step": 660
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.914518682500995e-06,
      "logits/chosen": -0.8933698534965515,
      "logits/rejected": -0.014977499842643738,
      "logps/chosen": -509.6695251464844,
      "logps/rejected": -674.0376586914062,
      "loss": 0.4931,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4237163066864014,
      "rewards/margins": 1.024255394935608,
      "rewards/rejected": -2.4479715824127197,
      "step": 670
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9084952541527315e-06,
      "logits/chosen": -0.5897430181503296,
      "logits/rejected": 0.24839851260185242,
      "logps/chosen": -533.2153930664062,
      "logps/rejected": -633.3540649414062,
      "loss": 0.4995,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5195014476776123,
      "rewards/margins": 0.7836617827415466,
      "rewards/rejected": -2.3031630516052246,
      "step": 680
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.902270717143858e-06,
      "logits/chosen": -0.20460684597492218,
      "logits/rejected": 0.12876734137535095,
      "logps/chosen": -454.1747131347656,
      "logps/rejected": -706.5838623046875,
      "loss": 0.3716,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4049168825149536,
      "rewards/margins": 1.3669251203536987,
      "rewards/rejected": -2.7718420028686523,
      "step": 690
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.895845591221427e-06,
      "logits/chosen": -0.5709416270256042,
      "logits/rejected": 0.21547503769397736,
      "logps/chosen": -543.8369750976562,
      "logps/rejected": -720.9224853515625,
      "loss": 0.4566,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.6587600708007812,
      "rewards/margins": 1.1786607503890991,
      "rewards/rejected": -2.83742094039917,
      "step": 700
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8892204128816e-06,
      "logits/chosen": -0.27632415294647217,
      "logits/rejected": -0.12886568903923035,
      "logps/chosen": -468.2186584472656,
      "logps/rejected": -706.5438232421875,
      "loss": 0.4874,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.4443271160125732,
      "rewards/margins": 1.2774336338043213,
      "rewards/rejected": -2.7217605113983154,
      "step": 710
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882395735324864e-06,
      "logits/chosen": -0.5171593427658081,
      "logits/rejected": 0.06492243707180023,
      "logps/chosen": -399.1227111816406,
      "logps/rejected": -646.8035888671875,
      "loss": 0.4334,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.1794382333755493,
      "rewards/margins": 1.383467435836792,
      "rewards/rejected": -2.5629055500030518,
      "step": 720
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.87537212840983e-06,
      "logits/chosen": -0.38013777136802673,
      "logits/rejected": 0.17461785674095154,
      "logps/chosen": -431.56939697265625,
      "logps/rejected": -727.7451171875,
      "loss": 0.403,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.2866822481155396,
      "rewards/margins": 1.4361357688903809,
      "rewards/rejected": -2.722817897796631,
      "step": 730
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8681501786056545e-06,
      "logits/chosen": -0.4910075068473816,
      "logits/rejected": 0.1193656325340271,
      "logps/chosen": -524.5296630859375,
      "logps/rejected": -716.439697265625,
      "loss": 0.5003,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4974342584609985,
      "rewards/margins": 1.3101706504821777,
      "rewards/rejected": -2.8076047897338867,
      "step": 740
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.860730488943068e-06,
      "logits/chosen": -0.4912436902523041,
      "logits/rejected": 0.18202224373817444,
      "logps/chosen": -527.1237182617188,
      "logps/rejected": -659.1048583984375,
      "loss": 0.4865,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5281684398651123,
      "rewards/margins": 0.9601262211799622,
      "rewards/rejected": -2.4882943630218506,
      "step": 750
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.853113678964022e-06,
      "logits/chosen": -0.8603304624557495,
      "logits/rejected": -0.08993472903966904,
      "logps/chosen": -432.42279052734375,
      "logps/rejected": -642.409912109375,
      "loss": 0.3753,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.2163296937942505,
      "rewards/margins": 1.3121674060821533,
      "rewards/rejected": -2.5284969806671143,
      "step": 760
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.845300384669958e-06,
      "logits/chosen": 0.24725647270679474,
      "logits/rejected": 0.0013871907722204924,
      "logps/chosen": -527.3872680664062,
      "logps/rejected": -742.9652099609375,
      "loss": 0.4498,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.8072938919067383,
      "rewards/margins": 1.2479110956192017,
      "rewards/rejected": -3.0552048683166504,
      "step": 770
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.837291258468701e-06,
      "logits/chosen": -0.6004719734191895,
      "logits/rejected": 0.5962368249893188,
      "logps/chosen": -507.5689392089844,
      "logps/rejected": -673.4093627929688,
      "loss": 0.4537,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.6435458660125732,
      "rewards/margins": 1.2729610204696655,
      "rewards/rejected": -2.9165070056915283,
      "step": 780
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.829086969119984e-06,
      "logits/chosen": -0.4771009385585785,
      "logits/rejected": 0.03888826444745064,
      "logps/chosen": -551.6219482421875,
      "logps/rejected": -797.9405517578125,
      "loss": 0.439,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.8441495895385742,
      "rewards/margins": 1.5053937435150146,
      "rewards/rejected": -3.349543333053589,
      "step": 790
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.820688201679605e-06,
      "logits/chosen": -0.36556169390678406,
      "logits/rejected": 0.3322374224662781,
      "logps/chosen": -510.71661376953125,
      "logps/rejected": -704.6385498046875,
      "loss": 0.4447,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6285368204116821,
      "rewards/margins": 1.4855858087539673,
      "rewards/rejected": -3.1141226291656494,
      "step": 800
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.8120956574422315e-06,
      "logits/chosen": -0.3884666860103607,
      "logits/rejected": -0.13175992667675018,
      "logps/chosen": -518.641357421875,
      "logps/rejected": -642.6422119140625,
      "loss": 0.5053,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5760186910629272,
      "rewards/margins": 0.9459335207939148,
      "rewards/rejected": -2.5219521522521973,
      "step": 810
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.803310053882831e-06,
      "logits/chosen": -1.0650156736373901,
      "logits/rejected": 0.38095536828041077,
      "logps/chosen": -542.5806274414062,
      "logps/rejected": -690.2403564453125,
      "loss": 0.4374,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.6415122747421265,
      "rewards/margins": 1.045480728149414,
      "rewards/rejected": -2.686992883682251,
      "step": 820
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.794332124596775e-06,
      "logits/chosen": -0.6507046222686768,
      "logits/rejected": 0.048465847969055176,
      "logps/chosen": -479.3224182128906,
      "logps/rejected": -674.3448486328125,
      "loss": 0.401,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.3757288455963135,
      "rewards/margins": 1.2602720260620117,
      "rewards/rejected": -2.636000633239746,
      "step": 830
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.785162619238575e-06,
      "logits/chosen": -0.6413692235946655,
      "logits/rejected": -0.042513225227594376,
      "logps/chosen": -557.0155029296875,
      "logps/rejected": -792.69970703125,
      "loss": 0.4201,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.6387875080108643,
      "rewards/margins": 1.4760812520980835,
      "rewards/rejected": -3.1148688793182373,
      "step": 840
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.775802303459288e-06,
      "logits/chosen": -0.74284428358078,
      "logits/rejected": -0.1755208522081375,
      "logps/chosen": -588.0230712890625,
      "logps/rejected": -802.0186767578125,
      "loss": 0.4069,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.7381775379180908,
      "rewards/margins": 1.4629442691802979,
      "rewards/rejected": -3.2011218070983887,
      "step": 850
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.766251958842589e-06,
      "logits/chosen": -0.42949801683425903,
      "logits/rejected": -0.4735488295555115,
      "logps/chosen": -518.86572265625,
      "logps/rejected": -755.33154296875,
      "loss": 0.4045,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6570861339569092,
      "rewards/margins": 1.4493482112884521,
      "rewards/rejected": -3.1064348220825195,
      "step": 860
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7565123828395066e-06,
      "logits/chosen": -0.7870966196060181,
      "logits/rejected": 0.0325060598552227,
      "logps/chosen": -515.7349243164062,
      "logps/rejected": -726.00927734375,
      "loss": 0.5102,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6367769241333008,
      "rewards/margins": 1.335331678390503,
      "rewards/rejected": -2.9721086025238037,
      "step": 870
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.746584388701831e-06,
      "logits/chosen": -0.7208787202835083,
      "logits/rejected": -0.034571003168821335,
      "logps/chosen": -488.56341552734375,
      "logps/rejected": -712.2532958984375,
      "loss": 0.5118,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.7884547710418701,
      "rewards/margins": 1.3315757513046265,
      "rewards/rejected": -3.120030164718628,
      "step": 880
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.736468805414218e-06,
      "logits/chosen": -0.7053539752960205,
      "logits/rejected": -0.015096127986907959,
      "logps/chosen": -474.1915588378906,
      "logps/rejected": -634.8294677734375,
      "loss": 0.4931,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4765193462371826,
      "rewards/margins": 1.0172039270401,
      "rewards/rejected": -2.4937233924865723,
      "step": 890
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.7261664776249595e-06,
      "logits/chosen": -1.0484793186187744,
      "logits/rejected": -0.35419899225234985,
      "logps/chosen": -551.4525756835938,
      "logps/rejected": -721.5409545898438,
      "loss": 0.4519,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.49507737159729,
      "rewards/margins": 1.2039979696273804,
      "rewards/rejected": -2.699075222015381,
      "step": 900
    },
    {
      "epoch": 0.24,
      "eval_logits/chosen": 0.19788537919521332,
      "eval_logits/rejected": 1.0802806615829468,
      "eval_logps/chosen": -498.0536804199219,
      "eval_logps/rejected": -686.2431030273438,
      "eval_loss": 0.456636905670166,
      "eval_rewards/accuracies": 0.7730000019073486,
      "eval_rewards/chosen": -1.42392897605896,
      "eval_rewards/margins": 1.248477816581726,
      "eval_rewards/rejected": -2.6724064350128174,
      "eval_runtime": 1384.9339,
      "eval_samples_per_second": 1.444,
      "eval_steps_per_second": 0.361,
      "step": 900
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.715678265575463e-06,
      "logits/chosen": -0.890539824962616,
      "logits/rejected": 0.22412686049938202,
      "logps/chosen": -490.0135803222656,
      "logps/rejected": -712.4631958007812,
      "loss": 0.3771,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.2612826824188232,
      "rewards/margins": 1.602736234664917,
      "rewards/rejected": -2.8640189170837402,
      "step": 910
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.705005045028415e-06,
      "logits/chosen": -0.8860180974006653,
      "logits/rejected": 0.03514351695775986,
      "logps/chosen": -594.3389282226562,
      "logps/rejected": -782.8358154296875,
      "loss": 0.4195,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.7765058279037476,
      "rewards/margins": 1.2403103113174438,
      "rewards/rejected": -3.0168161392211914,
      "step": 920
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.694147707194659e-06,
      "logits/chosen": -0.5554194450378418,
      "logits/rejected": 0.016318077221512794,
      "logps/chosen": -585.2276000976562,
      "logps/rejected": -805.9215087890625,
      "loss": 0.3769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8951377868652344,
      "rewards/margins": 1.719668984413147,
      "rewards/rejected": -3.61480712890625,
      "step": 930
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.683107158658782e-06,
      "logits/chosen": -0.7583510875701904,
      "logits/rejected": 0.25492575764656067,
      "logps/chosen": -562.8099975585938,
      "logps/rejected": -745.3627319335938,
      "loss": 0.5012,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.83584725856781,
      "rewards/margins": 1.2255539894104004,
      "rewards/rejected": -3.061401128768921,
      "step": 940
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.671884321303407e-06,
      "logits/chosen": -1.1498582363128662,
      "logits/rejected": 0.7587814331054688,
      "logps/chosen": -591.7152099609375,
      "logps/rejected": -751.0265502929688,
      "loss": 0.4275,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.7667793035507202,
      "rewards/margins": 1.4644314050674438,
      "rewards/rejected": -3.231210231781006,
      "step": 950
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.660480132232224e-06,
      "logits/chosen": -0.7957364320755005,
      "logits/rejected": -0.17932990193367004,
      "logps/chosen": -428.00958251953125,
      "logps/rejected": -722.1585083007812,
      "loss": 0.3945,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4679275751113892,
      "rewards/margins": 1.536849021911621,
      "rewards/rejected": -3.0047767162323,
      "step": 960
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6488955436917414e-06,
      "logits/chosen": -0.7032185196876526,
      "logits/rejected": 0.244097039103508,
      "logps/chosen": -511.335693359375,
      "logps/rejected": -830.0217895507812,
      "loss": 0.4831,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.6884835958480835,
      "rewards/margins": 1.69620680809021,
      "rewards/rejected": -3.384690761566162,
      "step": 970
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6371315229917644e-06,
      "logits/chosen": -0.9565087556838989,
      "logits/rejected": -0.6389614343643188,
      "logps/chosen": -467.09814453125,
      "logps/rejected": -740.5447998046875,
      "loss": 0.4217,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.331072449684143,
      "rewards/margins": 1.5709624290466309,
      "rewards/rejected": -2.9020345211029053,
      "step": 980
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.625189052424638e-06,
      "logits/chosen": -0.9426406621932983,
      "logits/rejected": -0.6099969148635864,
      "logps/chosen": -464.8932189941406,
      "logps/rejected": -654.5234985351562,
      "loss": 0.457,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.2798701524734497,
      "rewards/margins": 1.0147043466567993,
      "rewards/rejected": -2.29457426071167,
      "step": 990
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.613069129183218e-06,
      "logits/chosen": -0.9859519004821777,
      "logits/rejected": -0.3881329894065857,
      "logps/chosen": -416.37750244140625,
      "logps/rejected": -661.3399658203125,
      "loss": 0.4259,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.1251696348190308,
      "rewards/margins": 1.4806818962097168,
      "rewards/rejected": -2.605851650238037,
      "step": 1000
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.600772765277607e-06,
      "logits/chosen": -0.8805161714553833,
      "logits/rejected": -0.5182097554206848,
      "logps/chosen": -533.8609008789062,
      "logps/rejected": -738.94921875,
      "loss": 0.4221,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4609010219573975,
      "rewards/margins": 1.319215178489685,
      "rewards/rejected": -2.780116319656372,
      "step": 1010
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.588300987450652e-06,
      "logits/chosen": -0.7461687922477722,
      "logits/rejected": -0.11924894899129868,
      "logps/chosen": -504.3556213378906,
      "logps/rejected": -709.5269165039062,
      "loss": 0.4198,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.595538854598999,
      "rewards/margins": 1.395408272743225,
      "rewards/rejected": -2.9909470081329346,
      "step": 1020
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5756548370922136e-06,
      "logits/chosen": -0.9196340441703796,
      "logits/rejected": 0.15583333373069763,
      "logps/chosen": -467.1847229003906,
      "logps/rejected": -698.4019165039062,
      "loss": 0.471,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.4349987506866455,
      "rewards/margins": 1.6199867725372314,
      "rewards/rejected": -3.054985523223877,
      "step": 1030
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.562835370152206e-06,
      "logits/chosen": -0.9880784153938293,
      "logits/rejected": -0.4833584427833557,
      "logps/chosen": -474.13946533203125,
      "logps/rejected": -677.9033813476562,
      "loss": 0.4454,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.532492756843567,
      "rewards/margins": 1.2476747035980225,
      "rewards/rejected": -2.7801673412323,
      "step": 1040
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.54984365705243e-06,
      "logits/chosen": -0.9044283628463745,
      "logits/rejected": 0.2612631916999817,
      "logps/chosen": -487.19573974609375,
      "logps/rejected": -666.8839721679688,
      "loss": 0.4112,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.5281569957733154,
      "rewards/margins": 1.2005985975265503,
      "rewards/rejected": -2.728755474090576,
      "step": 1050
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.536680782597191e-06,
      "logits/chosen": -1.1122404336929321,
      "logits/rejected": 0.34001001715660095,
      "logps/chosen": -424.924560546875,
      "logps/rejected": -637.4059448242188,
      "loss": 0.4054,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2176024913787842,
      "rewards/margins": 1.382258653640747,
      "rewards/rejected": -2.5998611450195312,
      "step": 1060
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.523347845882718e-06,
      "logits/chosen": -1.2397373914718628,
      "logits/rejected": -0.0026629925705492496,
      "logps/chosen": -473.2481994628906,
      "logps/rejected": -670.2132568359375,
      "loss": 0.472,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.3779747486114502,
      "rewards/margins": 1.2943679094314575,
      "rewards/rejected": -2.672342300415039,
      "step": 1070
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.50984596020539e-06,
      "logits/chosen": -1.0471652746200562,
      "logits/rejected": -0.250629723072052,
      "logps/chosen": -507.221435546875,
      "logps/rejected": -730.1007080078125,
      "loss": 0.4251,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.5079420804977417,
      "rewards/margins": 1.3903275728225708,
      "rewards/rejected": -2.8982696533203125,
      "step": 1080
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4961762529687745e-06,
      "logits/chosen": -0.47195902466773987,
      "logits/rejected": -0.18957489728927612,
      "logps/chosen": -446.6513671875,
      "logps/rejected": -749.2055053710938,
      "loss": 0.3931,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.421588659286499,
      "rewards/margins": 1.843101143836975,
      "rewards/rejected": -3.2646899223327637,
      "step": 1090
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.482339865589492e-06,
      "logits/chosen": -0.6734046936035156,
      "logits/rejected": 0.2455734759569168,
      "logps/chosen": -536.9996337890625,
      "logps/rejected": -779.7747802734375,
      "loss": 0.3976,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.8160717487335205,
      "rewards/margins": 1.6105674505233765,
      "rewards/rejected": -3.4266390800476074,
      "step": 1100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.468337953401909e-06,
      "logits/chosen": -0.7846983075141907,
      "logits/rejected": 0.3714667558670044,
      "logps/chosen": -643.3924560546875,
      "logps/rejected": -889.9669189453125,
      "loss": 0.4624,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.112175703048706,
      "rewards/margins": 1.6929798126220703,
      "rewards/rejected": -3.8051555156707764,
      "step": 1110
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.45417168556166e-06,
      "logits/chosen": -0.887243390083313,
      "logits/rejected": 0.19038431346416473,
      "logps/chosen": -501.67987060546875,
      "logps/rejected": -794.1953735351562,
      "loss": 0.4672,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.6879377365112305,
      "rewards/margins": 1.825568437576294,
      "rewards/rejected": -3.5135064125061035,
      "step": 1120
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.439842244948036e-06,
      "logits/chosen": -1.2177503108978271,
      "logits/rejected": -0.813552975654602,
      "logps/chosen": -524.879150390625,
      "logps/rejected": -667.7706298828125,
      "loss": 0.4243,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5180702209472656,
      "rewards/margins": 1.0134727954864502,
      "rewards/rejected": -2.531543016433716,
      "step": 1130
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.425350828065204e-06,
      "logits/chosen": -1.0455009937286377,
      "logits/rejected": 0.2684328556060791,
      "logps/chosen": -534.6132202148438,
      "logps/rejected": -721.2943115234375,
      "loss": 0.3871,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7661949396133423,
      "rewards/margins": 1.332626223564148,
      "rewards/rejected": -3.0988211631774902,
      "step": 1140
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.410698644942303e-06,
      "logits/chosen": -0.8786664009094238,
      "logits/rejected": 0.14112402498722076,
      "logps/chosen": -458.5977478027344,
      "logps/rejected": -710.25927734375,
      "loss": 0.3959,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.411984920501709,
      "rewards/margins": 1.6877896785736084,
      "rewards/rejected": -3.0997745990753174,
      "step": 1150
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.395886919032406e-06,
      "logits/chosen": -0.8496414422988892,
      "logits/rejected": -0.006237986497581005,
      "logps/chosen": -508.9517517089844,
      "logps/rejected": -735.8714599609375,
      "loss": 0.4109,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.5958458185195923,
      "rewards/margins": 1.4991271495819092,
      "rewards/rejected": -3.094973087310791,
      "step": 1160
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.380916887110366e-06,
      "logits/chosen": -0.7842418551445007,
      "logits/rejected": -0.08200596272945404,
      "logps/chosen": -496.2059020996094,
      "logps/rejected": -729.1738891601562,
      "loss": 0.3737,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.5399061441421509,
      "rewards/margins": 1.5425506830215454,
      "rewards/rejected": -3.0824568271636963,
      "step": 1170
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.365789799169539e-06,
      "logits/chosen": -1.2651920318603516,
      "logits/rejected": 0.12391755729913712,
      "logps/chosen": -512.1771850585938,
      "logps/rejected": -717.8344116210938,
      "loss": 0.4228,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.5166397094726562,
      "rewards/margins": 1.4622437953948975,
      "rewards/rejected": -2.9788835048675537,
      "step": 1180
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.350506918317416e-06,
      "logits/chosen": -0.9895895719528198,
      "logits/rejected": -0.03255582973361015,
      "logps/chosen": -519.2945556640625,
      "logps/rejected": -687.9844970703125,
      "loss": 0.4106,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.6044280529022217,
      "rewards/margins": 1.4460744857788086,
      "rewards/rejected": -3.050502300262451,
      "step": 1190
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.335069520670149e-06,
      "logits/chosen": -0.8492997884750366,
      "logits/rejected": -0.5858234167098999,
      "logps/chosen": -516.3770141601562,
      "logps/rejected": -707.2117309570312,
      "loss": 0.4034,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.7651269435882568,
      "rewards/margins": 1.3661835193634033,
      "rewards/rejected": -3.1313109397888184,
      "step": 1200
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": 0.7243556976318359,
      "eval_logits/rejected": 1.7155570983886719,
      "eval_logps/chosen": -545.945068359375,
      "eval_logps/rejected": -770.7060546875,
      "eval_loss": 0.44869744777679443,
      "eval_rewards/accuracies": 0.7870000004768372,
      "eval_rewards/chosen": -1.9028427600860596,
      "eval_rewards/margins": 1.614193320274353,
      "eval_rewards/rejected": -3.517036199569702,
      "eval_runtime": 1383.7156,
      "eval_samples_per_second": 1.445,
      "eval_steps_per_second": 0.361,
      "step": 1200
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.319478895246e-06,
      "logits/chosen": -0.6312126517295837,
      "logits/rejected": 0.010389542207121849,
      "logps/chosen": -485.5673828125,
      "logps/rejected": -724.3965454101562,
      "loss": 0.4596,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.7340030670166016,
      "rewards/margins": 1.5444698333740234,
      "rewards/rejected": -3.278472900390625,
      "step": 1210
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.303736343857704e-06,
      "logits/chosen": -0.5306238532066345,
      "logits/rejected": 0.09534727036952972,
      "logps/chosen": -501.3701171875,
      "logps/rejected": -773.635498046875,
      "loss": 0.4065,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.6401008367538452,
      "rewards/margins": 1.739175796508789,
      "rewards/rejected": -3.379276752471924,
      "step": 1220
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.287843181003772e-06,
      "logits/chosen": -0.7599018812179565,
      "logits/rejected": 0.544152557849884,
      "logps/chosen": -492.7386779785156,
      "logps/rejected": -739.1934814453125,
      "loss": 0.3467,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.591827392578125,
      "rewards/margins": 1.6447445154190063,
      "rewards/rejected": -3.236571788787842,
      "step": 1230
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.27180073375873e-06,
      "logits/chosen": -0.7321812510490417,
      "logits/rejected": -0.04632633179426193,
      "logps/chosen": -503.90655517578125,
      "logps/rejected": -751.9849243164062,
      "loss": 0.4352,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7118949890136719,
      "rewards/margins": 1.5225197076797485,
      "rewards/rejected": -3.234414577484131,
      "step": 1240
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.255610341662304e-06,
      "logits/chosen": -0.8730325698852539,
      "logits/rejected": -0.1791534423828125,
      "logps/chosen": -524.244140625,
      "logps/rejected": -752.1405029296875,
      "loss": 0.3827,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7566620111465454,
      "rewards/margins": 1.7185337543487549,
      "rewards/rejected": -3.4751956462860107,
      "step": 1250
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2392733566075764e-06,
      "logits/chosen": -0.8349526524543762,
      "logits/rejected": 0.03989090770483017,
      "logps/chosen": -555.2600708007812,
      "logps/rejected": -799.1808471679688,
      "loss": 0.4153,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8972892761230469,
      "rewards/margins": 1.5951424837112427,
      "rewards/rejected": -3.492431640625,
      "step": 1260
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2227911427280975e-06,
      "logits/chosen": -1.2052414417266846,
      "logits/rejected": 0.40989890694618225,
      "logps/chosen": -535.0908203125,
      "logps/rejected": -766.23974609375,
      "loss": 0.4851,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7357299327850342,
      "rewards/margins": 1.7371448278427124,
      "rewards/rejected": -3.472874879837036,
      "step": 1270
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.206165076283983e-06,
      "logits/chosen": -1.3161356449127197,
      "logits/rejected": -0.31805121898651123,
      "logps/chosen": -562.9700927734375,
      "logps/rejected": -699.4861450195312,
      "loss": 0.4345,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.6805912256240845,
      "rewards/margins": 1.0278584957122803,
      "rewards/rejected": -2.708449602127075,
      "step": 1280
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.189396545546995e-06,
      "logits/chosen": -1.2732408046722412,
      "logits/rejected": -0.36048611998558044,
      "logps/chosen": -463.95849609375,
      "logps/rejected": -677.6444091796875,
      "loss": 0.4661,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.385292649269104,
      "rewards/margins": 1.3424113988876343,
      "rewards/rejected": -2.7277040481567383,
      "step": 1290
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.172486950684627e-06,
      "logits/chosen": -1.3132286071777344,
      "logits/rejected": 0.031231578439474106,
      "logps/chosen": -535.4584350585938,
      "logps/rejected": -707.3728637695312,
      "loss": 0.439,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6812185049057007,
      "rewards/margins": 1.3230525255203247,
      "rewards/rejected": -3.0042712688446045,
      "step": 1300
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.155437703643182e-06,
      "logits/chosen": -1.2260167598724365,
      "logits/rejected": -0.5245649814605713,
      "logps/chosen": -462.94403076171875,
      "logps/rejected": -694.3932495117188,
      "loss": 0.4018,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4110779762268066,
      "rewards/margins": 1.4465053081512451,
      "rewards/rejected": -2.8575832843780518,
      "step": 1310
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.138250228029882e-06,
      "logits/chosen": -1.0029791593551636,
      "logits/rejected": -0.45876234769821167,
      "logps/chosen": -471.4588928222656,
      "logps/rejected": -711.7996215820312,
      "loss": 0.3972,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.6702163219451904,
      "rewards/margins": 1.6204955577850342,
      "rewards/rejected": -3.2907118797302246,
      "step": 1320
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.120925958993994e-06,
      "logits/chosen": -0.5831348299980164,
      "logits/rejected": -0.2067866027355194,
      "logps/chosen": -476.1014099121094,
      "logps/rejected": -722.8709106445312,
      "loss": 0.4493,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.6658756732940674,
      "rewards/margins": 1.6795238256454468,
      "rewards/rejected": -3.3453993797302246,
      "step": 1330
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.103466343106999e-06,
      "logits/chosen": -1.077728033065796,
      "logits/rejected": 0.1974649727344513,
      "logps/chosen": -636.1361083984375,
      "logps/rejected": -829.7213134765625,
      "loss": 0.5001,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.2163033485412598,
      "rewards/margins": 1.5475972890853882,
      "rewards/rejected": -3.7639007568359375,
      "step": 1340
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.085872838241797e-06,
      "logits/chosen": -0.8451377749443054,
      "logits/rejected": -0.4800887703895569,
      "logps/chosen": -523.9595947265625,
      "logps/rejected": -813.6463012695312,
      "loss": 0.4193,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.937745451927185,
      "rewards/margins": 1.8354793787002563,
      "rewards/rejected": -3.7732245922088623,
      "step": 1350
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.06814691345098e-06,
      "logits/chosen": -0.8073797225952148,
      "logits/rejected": -0.67634516954422,
      "logps/chosen": -499.55364990234375,
      "logps/rejected": -725.6375732421875,
      "loss": 0.5085,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8341518640518188,
      "rewards/margins": 1.3492587804794312,
      "rewards/rejected": -3.18341064453125,
      "step": 1360
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.050290048844171e-06,
      "logits/chosen": -1.3967281579971313,
      "logits/rejected": -0.1124715581536293,
      "logps/chosen": -586.6700439453125,
      "logps/rejected": -785.9407958984375,
      "loss": 0.4425,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.6672008037567139,
      "rewards/margins": 1.4039936065673828,
      "rewards/rejected": -3.0711944103240967,
      "step": 1370
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.032303735464422e-06,
      "logits/chosen": -1.3250014781951904,
      "logits/rejected": -0.28036853671073914,
      "logps/chosen": -523.9191284179688,
      "logps/rejected": -757.8228759765625,
      "loss": 0.3506,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.4709548950195312,
      "rewards/margins": 1.7138077020645142,
      "rewards/rejected": -3.184762477874756,
      "step": 1380
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.014189475163727e-06,
      "logits/chosen": -0.9671980142593384,
      "logits/rejected": -0.292216956615448,
      "logps/chosen": -559.5821533203125,
      "logps/rejected": -784.157958984375,
      "loss": 0.3887,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.7415062189102173,
      "rewards/margins": 1.629124641418457,
      "rewards/rejected": -3.370630979537964,
      "step": 1390
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.995948780477605e-06,
      "logits/chosen": -1.1137468814849854,
      "logits/rejected": -0.3016485273838043,
      "logps/chosen": -586.1868286132812,
      "logps/rejected": -790.1680908203125,
      "loss": 0.4715,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8678308725357056,
      "rewards/margins": 1.3062608242034912,
      "rewards/rejected": -3.1740918159484863,
      "step": 1400
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.977583174498816e-06,
      "logits/chosen": -0.9699400067329407,
      "logits/rejected": -0.7510320544242859,
      "logps/chosen": -545.1305541992188,
      "logps/rejected": -793.4750366210938,
      "loss": 0.4399,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.7863260507583618,
      "rewards/margins": 1.481147050857544,
      "rewards/rejected": -3.267472743988037,
      "step": 1410
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.959094190750172e-06,
      "logits/chosen": -1.3356729745864868,
      "logits/rejected": -0.6711053848266602,
      "logps/chosen": -574.8829956054688,
      "logps/rejected": -773.9825439453125,
      "loss": 0.3984,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.7231754064559937,
      "rewards/margins": 1.5199847221374512,
      "rewards/rejected": -3.2431602478027344,
      "step": 1420
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9404833730564975e-06,
      "logits/chosen": -1.2241183519363403,
      "logits/rejected": -0.7589890956878662,
      "logps/chosen": -524.2040405273438,
      "logps/rejected": -684.8192138671875,
      "loss": 0.4751,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.550222396850586,
      "rewards/margins": 1.2541849613189697,
      "rewards/rejected": -2.8044073581695557,
      "step": 1430
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.921752275415712e-06,
      "logits/chosen": -1.2616220712661743,
      "logits/rejected": -0.6698473691940308,
      "logps/chosen": -494.5272521972656,
      "logps/rejected": -719.5067749023438,
      "loss": 0.407,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5580322742462158,
      "rewards/margins": 1.4134986400604248,
      "rewards/rejected": -2.9715309143066406,
      "step": 1440
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.902902461869079e-06,
      "logits/chosen": -0.9971591234207153,
      "logits/rejected": -0.4130152761936188,
      "logps/chosen": -407.35736083984375,
      "logps/rejected": -699.933837890625,
      "loss": 0.3878,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3271167278289795,
      "rewards/margins": 1.8487539291381836,
      "rewards/rejected": -3.175870895385742,
      "step": 1450
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.883935506370605e-06,
      "logits/chosen": -1.3185368776321411,
      "logits/rejected": -0.5751763582229614,
      "logps/chosen": -538.9137573242188,
      "logps/rejected": -727.714599609375,
      "loss": 0.3787,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7377328872680664,
      "rewards/margins": 1.1966904401779175,
      "rewards/rejected": -2.9344232082366943,
      "step": 1460
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.864852992655617e-06,
      "logits/chosen": -1.2166504859924316,
      "logits/rejected": -0.45289698243141174,
      "logps/chosen": -516.9674682617188,
      "logps/rejected": -733.4182739257812,
      "loss": 0.3926,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6966655254364014,
      "rewards/margins": 1.5359889268875122,
      "rewards/rejected": -3.232654571533203,
      "step": 1470
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.845656514108516e-06,
      "logits/chosen": -1.2518984079360962,
      "logits/rejected": -0.23703515529632568,
      "logps/chosen": -571.0328979492188,
      "logps/rejected": -839.0745239257812,
      "loss": 0.409,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.9428062438964844,
      "rewards/margins": 1.905398964881897,
      "rewards/rejected": -3.84820556640625,
      "step": 1480
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.826347673629738e-06,
      "logits/chosen": -1.241381049156189,
      "logits/rejected": -0.424204021692276,
      "logps/chosen": -517.6029052734375,
      "logps/rejected": -811.5399169921875,
      "loss": 0.4235,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.9763988256454468,
      "rewards/margins": 1.8420568704605103,
      "rewards/rejected": -3.818455457687378,
      "step": 1490
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8069280835019062e-06,
      "logits/chosen": -1.3552948236465454,
      "logits/rejected": -0.24583733081817627,
      "logps/chosen": -539.5227661132812,
      "logps/rejected": -833.4625244140625,
      "loss": 0.4193,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.008434295654297,
      "rewards/margins": 2.0153846740722656,
      "rewards/rejected": -4.0238189697265625,
      "step": 1500
    },
    {
      "epoch": 0.39,
      "eval_logits/chosen": 0.0019010701216757298,
      "eval_logits/rejected": 0.9998253583908081,
      "eval_logps/chosen": -544.3021240234375,
      "eval_logps/rejected": -767.47119140625,
      "eval_loss": 0.44199585914611816,
      "eval_rewards/accuracies": 0.7839999794960022,
      "eval_rewards/chosen": -1.8864127397537231,
      "eval_rewards/margins": 1.5982747077941895,
      "eval_rewards/rejected": -3.484687566757202,
      "eval_runtime": 1381.698,
      "eval_samples_per_second": 1.447,
      "eval_steps_per_second": 0.362,
      "step": 1500
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7873993652552077e-06,
      "logits/chosen": -1.0648901462554932,
      "logits/rejected": -0.4144531786441803,
      "logps/chosen": -475.7496643066406,
      "logps/rejected": -861.4631958007812,
      "loss": 0.3316,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.5341570377349854,
      "rewards/margins": 2.4849419593811035,
      "rewards/rejected": -4.019099235534668,
      "step": 1510
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7677631495319953e-06,
      "logits/chosen": -1.0717235803604126,
      "logits/rejected": -0.44138726592063904,
      "logps/chosen": -545.8651733398438,
      "logps/rejected": -788.2182006835938,
      "loss": 0.4331,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6811964511871338,
      "rewards/margins": 1.6335046291351318,
      "rewards/rejected": -3.3147010803222656,
      "step": 1520
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.748021075950633e-06,
      "logits/chosen": -1.231894850730896,
      "logits/rejected": -0.649477481842041,
      "logps/chosen": -472.5684509277344,
      "logps/rejected": -648.9671630859375,
      "loss": 0.4403,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6376155614852905,
      "rewards/margins": 1.3435395956039429,
      "rewards/rejected": -2.9811549186706543,
      "step": 1530
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7281747929685824e-06,
      "logits/chosen": -0.8811131715774536,
      "logits/rejected": -0.19349880516529083,
      "logps/chosen": -556.5198364257812,
      "logps/rejected": -868.0969848632812,
      "loss": 0.3711,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.9225609302520752,
      "rewards/margins": 2.121070384979248,
      "rewards/rejected": -4.043631076812744,
      "step": 1540
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.7082259577447604e-06,
      "logits/chosen": -1.3660091161727905,
      "logits/rejected": -0.37334832549095154,
      "logps/chosen": -508.5003356933594,
      "logps/rejected": -700.1402587890625,
      "loss": 0.4505,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.6233818531036377,
      "rewards/margins": 1.5405502319335938,
      "rewards/rejected": -3.1639320850372314,
      "step": 1550
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6881762360011688e-06,
      "logits/chosen": -0.9465176463127136,
      "logits/rejected": -0.1583428531885147,
      "logps/chosen": -568.060302734375,
      "logps/rejected": -805.0638427734375,
      "loss": 0.3998,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.06211256980896,
      "rewards/margins": 1.7412744760513306,
      "rewards/rejected": -3.80338716506958,
      "step": 1560
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.668027301883802e-06,
      "logits/chosen": -0.6530407667160034,
      "logits/rejected": 0.17987249791622162,
      "logps/chosen": -569.39013671875,
      "logps/rejected": -819.4601440429688,
      "loss": 0.3991,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.138000726699829,
      "rewards/margins": 1.7371118068695068,
      "rewards/rejected": -3.8751120567321777,
      "step": 1570
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.64778083782286e-06,
      "logits/chosen": -1.1196366548538208,
      "logits/rejected": 0.07997065782546997,
      "logps/chosen": -588.8242797851562,
      "logps/rejected": -888.84619140625,
      "loss": 0.3689,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.009155750274658,
      "rewards/margins": 2.2112624645233154,
      "rewards/rejected": -4.220418453216553,
      "step": 1580
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.627438534392268e-06,
      "logits/chosen": -1.2351372241973877,
      "logits/rejected": -0.21363726258277893,
      "logps/chosen": -565.1137084960938,
      "logps/rejected": -799.2964477539062,
      "loss": 0.4302,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.9956506490707397,
      "rewards/margins": 1.7617241144180298,
      "rewards/rejected": -3.7573745250701904,
      "step": 1590
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.607002090168506e-06,
      "logits/chosen": -1.1077851057052612,
      "logits/rejected": -0.3146376609802246,
      "logps/chosen": -519.68505859375,
      "logps/rejected": -786.7335815429688,
      "loss": 0.3644,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6840426921844482,
      "rewards/margins": 1.7921777963638306,
      "rewards/rejected": -3.4762203693389893,
      "step": 1600
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.586473211588787e-06,
      "logits/chosen": -1.0645841360092163,
      "logits/rejected": 0.0885920599102974,
      "logps/chosen": -530.0689697265625,
      "logps/rejected": -746.6349487304688,
      "loss": 0.4293,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7966455221176147,
      "rewards/margins": 1.5383833646774292,
      "rewards/rejected": -3.335028886795044,
      "step": 1610
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5658536128085623e-06,
      "logits/chosen": -0.8512083292007446,
      "logits/rejected": -0.07845296710729599,
      "logps/chosen": -522.5213623046875,
      "logps/rejected": -783.8287353515625,
      "loss": 0.4465,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.6679248809814453,
      "rewards/margins": 1.7967729568481445,
      "rewards/rejected": -3.464698076248169,
      "step": 1620
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.545145015558399e-06,
      "logits/chosen": -1.1375799179077148,
      "logits/rejected": -0.7033450603485107,
      "logps/chosen": -488.27825927734375,
      "logps/rejected": -743.138427734375,
      "loss": 0.4717,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5062588453292847,
      "rewards/margins": 1.479614019393921,
      "rewards/rejected": -2.985872983932495,
      "step": 1630
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5243491490002056e-06,
      "logits/chosen": -1.122290849685669,
      "logits/rejected": -0.7316358685493469,
      "logps/chosen": -507.69036865234375,
      "logps/rejected": -690.9684448242188,
      "loss": 0.4651,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.5151287317276,
      "rewards/margins": 1.2003190517425537,
      "rewards/rejected": -2.7154476642608643,
      "step": 1640
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.503467749582857e-06,
      "logits/chosen": -1.7979711294174194,
      "logits/rejected": -0.23376531898975372,
      "logps/chosen": -447.5152282714844,
      "logps/rejected": -644.4968872070312,
      "loss": 0.4207,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.3395702838897705,
      "rewards/margins": 1.2873914241790771,
      "rewards/rejected": -2.6269614696502686,
      "step": 1650
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.4825025608971947e-06,
      "logits/chosen": -1.1949323415756226,
      "logits/rejected": -0.4591255784034729,
      "logps/chosen": -531.0167846679688,
      "logps/rejected": -708.4334716796875,
      "loss": 0.4157,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6559035778045654,
      "rewards/margins": 1.3914124965667725,
      "rewards/rejected": -3.047316074371338,
      "step": 1660
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4614553335304407e-06,
      "logits/chosen": -0.9112270474433899,
      "logits/rejected": -0.7079882025718689,
      "logps/chosen": -477.55645751953125,
      "logps/rejected": -665.2545166015625,
      "loss": 0.4351,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.638381004333496,
      "rewards/margins": 0.9713879823684692,
      "rewards/rejected": -2.609769105911255,
      "step": 1670
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4403278249200222e-06,
      "logits/chosen": -1.1047728061676025,
      "logits/rejected": -0.209273099899292,
      "logps/chosen": -513.626708984375,
      "logps/rejected": -751.2394409179688,
      "loss": 0.3962,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6597833633422852,
      "rewards/margins": 1.5487313270568848,
      "rewards/rejected": -3.20851469039917,
      "step": 1680
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4191217992068293e-06,
      "logits/chosen": -1.1829806566238403,
      "logits/rejected": 0.04266662523150444,
      "logps/chosen": -560.150634765625,
      "logps/rejected": -809.93896484375,
      "loss": 0.4279,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8385143280029297,
      "rewards/margins": 1.672101378440857,
      "rewards/rejected": -3.510615825653076,
      "step": 1690
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.3978390270879056e-06,
      "logits/chosen": -1.1551451683044434,
      "logits/rejected": -0.16915690898895264,
      "logps/chosen": -546.1964111328125,
      "logps/rejected": -733.3271484375,
      "loss": 0.4386,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.6938936710357666,
      "rewards/margins": 1.5033698081970215,
      "rewards/rejected": -3.197263240814209,
      "step": 1700
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3764812856685995e-06,
      "logits/chosen": -1.0993391275405884,
      "logits/rejected": -0.748005211353302,
      "logps/chosen": -503.6968688964844,
      "logps/rejected": -733.4053955078125,
      "loss": 0.4426,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.7742856740951538,
      "rewards/margins": 1.4196711778640747,
      "rewards/rejected": -3.1939570903778076,
      "step": 1710
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3550503583141726e-06,
      "logits/chosen": -1.1645628213882446,
      "logits/rejected": -0.29861804842948914,
      "logps/chosen": -516.9429931640625,
      "logps/rejected": -731.75634765625,
      "loss": 0.4486,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.7132374048233032,
      "rewards/margins": 1.4840338230133057,
      "rewards/rejected": -3.1972713470458984,
      "step": 1720
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3335480345008907e-06,
      "logits/chosen": -1.0770137310028076,
      "logits/rejected": -0.3164665400981903,
      "logps/chosen": -475.98345947265625,
      "logps/rejected": -717.9310302734375,
      "loss": 0.3996,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5537554025650024,
      "rewards/margins": 1.6571037769317627,
      "rewards/rejected": -3.2108588218688965,
      "step": 1730
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.3119761096666055e-06,
      "logits/chosen": -1.5323493480682373,
      "logits/rejected": -0.1781812459230423,
      "logps/chosen": -540.9231567382812,
      "logps/rejected": -732.0032958984375,
      "loss": 0.3972,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.7525060176849365,
      "rewards/margins": 1.5016968250274658,
      "rewards/rejected": -3.254202365875244,
      "step": 1740
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.290336385060832e-06,
      "logits/chosen": -1.1251745223999023,
      "logits/rejected": -0.11602558940649033,
      "logps/chosen": -634.4666748046875,
      "logps/rejected": -833.8592529296875,
      "loss": 0.3915,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.158566951751709,
      "rewards/margins": 1.6248620748519897,
      "rewards/rejected": -3.783428907394409,
      "step": 1750
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.268630667594348e-06,
      "logits/chosen": -0.6574426889419556,
      "logits/rejected": -0.43358176946640015,
      "logps/chosen": -525.9124755859375,
      "logps/rejected": -769.3137817382812,
      "loss": 0.4214,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.048187017440796,
      "rewards/margins": 1.6054658889770508,
      "rewards/rejected": -3.6536529064178467,
      "step": 1760
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2468607696883147e-06,
      "logits/chosen": -1.0191020965576172,
      "logits/rejected": -0.8884264230728149,
      "logps/chosen": -542.0821533203125,
      "logps/rejected": -717.0391845703125,
      "loss": 0.439,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9124071598052979,
      "rewards/margins": 1.0934618711471558,
      "rewards/rejected": -3.0058693885803223,
      "step": 1770
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.225028509122944e-06,
      "logits/chosen": -0.6006935834884644,
      "logits/rejected": -0.7735892534255981,
      "logps/chosen": -506.019287109375,
      "logps/rejected": -766.4857177734375,
      "loss": 0.4162,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.817831039428711,
      "rewards/margins": 1.61776602268219,
      "rewards/rejected": -3.4355969429016113,
      "step": 1780
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2031357088857083e-06,
      "logits/chosen": -1.139946699142456,
      "logits/rejected": -0.35632461309432983,
      "logps/chosen": -478.1253967285156,
      "logps/rejected": -795.1430053710938,
      "loss": 0.3478,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6329262256622314,
      "rewards/margins": 2.2241363525390625,
      "rewards/rejected": -3.857062816619873,
      "step": 1790
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.181184197019127e-06,
      "logits/chosen": -1.2290761470794678,
      "logits/rejected": -0.23889155685901642,
      "logps/chosen": -565.7088623046875,
      "logps/rejected": -842.8005981445312,
      "loss": 0.409,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.107991933822632,
      "rewards/margins": 2.049304485321045,
      "rewards/rejected": -4.157296180725098,
      "step": 1800
    },
    {
      "epoch": 0.47,
      "eval_logits/chosen": 0.5340744853019714,
      "eval_logits/rejected": 1.4875802993774414,
      "eval_logps/chosen": -561.5723266601562,
      "eval_logps/rejected": -791.2130126953125,
      "eval_loss": 0.43651697039604187,
      "eval_rewards/accuracies": 0.7919999957084656,
      "eval_rewards/chosen": -2.059115409851074,
      "eval_rewards/margins": 1.6629897356033325,
      "eval_rewards/rejected": -3.7221052646636963,
      "eval_runtime": 1381.695,
      "eval_samples_per_second": 1.447,
      "eval_steps_per_second": 0.362,
      "step": 1800
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.159175806468126e-06,
      "logits/chosen": -1.0473666191101074,
      "logits/rejected": -0.06902176141738892,
      "logps/chosen": -595.341552734375,
      "logps/rejected": -831.9974365234375,
      "loss": 0.4152,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.9551061391830444,
      "rewards/margins": 1.7293351888656616,
      "rewards/rejected": -3.684441328048706,
      "step": 1810
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1371123749269804e-06,
      "logits/chosen": -1.026745319366455,
      "logits/rejected": -0.4049917757511139,
      "logps/chosen": -565.2854614257812,
      "logps/rejected": -764.7696533203125,
      "loss": 0.4013,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.9236133098602295,
      "rewards/margins": 1.5329084396362305,
      "rewards/rejected": -3.456521511077881,
      "step": 1820
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.114995744685877e-06,
      "logits/chosen": -0.9977224469184875,
      "logits/rejected": -0.40141773223876953,
      "logps/chosen": -466.9925842285156,
      "logps/rejected": -759.6148071289062,
      "loss": 0.4686,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.7426468133926392,
      "rewards/margins": 1.7171356678009033,
      "rewards/rejected": -3.459782361984253,
      "step": 1830
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0928277624770743e-06,
      "logits/chosen": -1.512317180633545,
      "logits/rejected": -0.05047481134533882,
      "logps/chosen": -557.8958129882812,
      "logps/rejected": -753.0090942382812,
      "loss": 0.4698,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.98238205909729,
      "rewards/margins": 1.4256618022918701,
      "rewards/rejected": -3.408043622970581,
      "step": 1840
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.070610279320708e-06,
      "logits/chosen": -1.1659696102142334,
      "logits/rejected": -0.22102081775665283,
      "logps/chosen": -540.2110595703125,
      "logps/rejected": -764.9635620117188,
      "loss": 0.3463,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.8047651052474976,
      "rewards/margins": 1.6794044971466064,
      "rewards/rejected": -3.4841697216033936,
      "step": 1850
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0483451503702264e-06,
      "logits/chosen": -0.9429510831832886,
      "logits/rejected": -0.374999463558197,
      "logps/chosen": -633.7833862304688,
      "logps/rejected": -881.3572387695312,
      "loss": 0.4,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.2336530685424805,
      "rewards/margins": 1.6140598058700562,
      "rewards/rejected": -3.847712993621826,
      "step": 1860
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0260342347574916e-06,
      "logits/chosen": -0.9639546275138855,
      "logits/rejected": -0.8156697154045105,
      "logps/chosen": -493.8570861816406,
      "logps/rejected": -728.9312744140625,
      "loss": 0.3857,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.7304465770721436,
      "rewards/margins": 1.2347511053085327,
      "rewards/rejected": -2.965198040008545,
      "step": 1870
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0036793954375358e-06,
      "logits/chosen": -1.5436639785766602,
      "logits/rejected": 0.08726786822080612,
      "logps/chosen": -512.5186157226562,
      "logps/rejected": -664.79296875,
      "loss": 0.4216,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7313350439071655,
      "rewards/margins": 1.4134327173233032,
      "rewards/rejected": -3.144767999649048,
      "step": 1880
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.981282499033009e-06,
      "logits/chosen": -0.6750370264053345,
      "logits/rejected": -0.8198210597038269,
      "logps/chosen": -542.04052734375,
      "logps/rejected": -810.6235961914062,
      "loss": 0.4386,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.0354888439178467,
      "rewards/margins": 1.7842447757720947,
      "rewards/rejected": -3.8197338581085205,
      "step": 1890
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9588454156783163e-06,
      "logits/chosen": -1.0820724964141846,
      "logits/rejected": -0.7213941812515259,
      "logps/chosen": -541.3480224609375,
      "logps/rejected": -766.2906494140625,
      "loss": 0.4257,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.084608554840088,
      "rewards/margins": 1.3346532583236694,
      "rewards/rejected": -3.4192614555358887,
      "step": 1900
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9363700188634597e-06,
      "logits/chosen": -1.2453614473342896,
      "logits/rejected": -0.5085484385490417,
      "logps/chosen": -540.5175170898438,
      "logps/rejected": -765.7025756835938,
      "loss": 0.4321,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.895167589187622,
      "rewards/margins": 1.5056030750274658,
      "rewards/rejected": -3.4007697105407715,
      "step": 1910
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9138581852776053e-06,
      "logits/chosen": -1.1196304559707642,
      "logits/rejected": 0.026494156569242477,
      "logps/chosen": -559.1360473632812,
      "logps/rejected": -778.654296875,
      "loss": 0.4461,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9397118091583252,
      "rewards/margins": 1.5184619426727295,
      "rewards/rejected": -3.4581737518310547,
      "step": 1920
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8913117946523805e-06,
      "logits/chosen": -1.2101647853851318,
      "logits/rejected": 0.05119786784052849,
      "logps/chosen": -497.8490295410156,
      "logps/rejected": -711.6702270507812,
      "loss": 0.4391,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.8607673645019531,
      "rewards/margins": 1.6270701885223389,
      "rewards/rejected": -3.487837314605713,
      "step": 1930
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8687327296049126e-06,
      "logits/chosen": -0.9110754132270813,
      "logits/rejected": -0.3987257182598114,
      "logps/chosen": -488.52056884765625,
      "logps/rejected": -814.98388671875,
      "loss": 0.385,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.5353851318359375,
      "rewards/margins": 2.019871711730957,
      "rewards/rejected": -3.5552570819854736,
      "step": 1940
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8461228754806376e-06,
      "logits/chosen": -1.2645906209945679,
      "logits/rejected": 0.05890879034996033,
      "logps/chosen": -571.50146484375,
      "logps/rejected": -764.4017944335938,
      "loss": 0.3804,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.847496747970581,
      "rewards/margins": 1.488777995109558,
      "rewards/rejected": -3.3362746238708496,
      "step": 1950
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.823484120195865e-06,
      "logits/chosen": -1.035412311553955,
      "logits/rejected": 0.14751215279102325,
      "logps/chosen": -568.7293090820312,
      "logps/rejected": -764.1280517578125,
      "loss": 0.3716,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.9270780086517334,
      "rewards/margins": 1.58747398853302,
      "rewards/rejected": -3.514551877975464,
      "step": 1960
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.8008183540801486e-06,
      "logits/chosen": -1.152499794960022,
      "logits/rejected": -0.26493799686431885,
      "logps/chosen": -610.4190063476562,
      "logps/rejected": -777.2071533203125,
      "loss": 0.3893,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.9002234935760498,
      "rewards/margins": 1.6286661624908447,
      "rewards/rejected": -3.5288894176483154,
      "step": 1970
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7781274697184353e-06,
      "logits/chosen": -1.2689648866653442,
      "logits/rejected": 0.2965567111968994,
      "logps/chosen": -519.8406982421875,
      "logps/rejected": -776.2745971679688,
      "loss": 0.451,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.7539297342300415,
      "rewards/margins": 1.7792565822601318,
      "rewards/rejected": -3.533186435699463,
      "step": 1980
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7554133617930397e-06,
      "logits/chosen": -1.1502325534820557,
      "logits/rejected": -0.16516944766044617,
      "logps/chosen": -586.9942626953125,
      "logps/rejected": -799.5856323242188,
      "loss": 0.3986,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9755744934082031,
      "rewards/margins": 1.5421812534332275,
      "rewards/rejected": -3.5177555084228516,
      "step": 1990
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7326779269254363e-06,
      "logits/chosen": -1.2239593267440796,
      "logits/rejected": -0.29785847663879395,
      "logps/chosen": -577.9757080078125,
      "logps/rejected": -780.0157470703125,
      "loss": 0.4163,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8510382175445557,
      "rewards/margins": 1.496047019958496,
      "rewards/rejected": -3.3470852375030518,
      "step": 2000
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.7099230635178954e-06,
      "logits/chosen": -0.8359963297843933,
      "logits/rejected": -0.609653115272522,
      "logps/chosen": -522.3953857421875,
      "logps/rejected": -836.3997192382812,
      "loss": 0.3754,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.8302185535430908,
      "rewards/margins": 1.9239788055419922,
      "rewards/rejected": -3.754197359085083,
      "step": 2010
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6871506715949608e-06,
      "logits/chosen": -1.2713494300842285,
      "logits/rejected": 0.4311772286891937,
      "logps/chosen": -510.71942138671875,
      "logps/rejected": -779.8484497070312,
      "loss": 0.2961,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6959383487701416,
      "rewards/margins": 1.9984363317489624,
      "rewards/rejected": -3.6943740844726562,
      "step": 2020
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6643626526448063e-06,
      "logits/chosen": -0.745600700378418,
      "logits/rejected": -0.04406242445111275,
      "logps/chosen": -545.650390625,
      "logps/rejected": -726.6231689453125,
      "loss": 0.4021,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.117701768875122,
      "rewards/margins": 1.459877848625183,
      "rewards/rejected": -3.5775794982910156,
      "step": 2030
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6415609094604562e-06,
      "logits/chosen": -0.6423822641372681,
      "logits/rejected": -0.5382918119430542,
      "logps/chosen": -555.1259765625,
      "logps/rejected": -854.6931762695312,
      "loss": 0.4133,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.1485462188720703,
      "rewards/margins": 1.8231878280639648,
      "rewards/rejected": -3.9717342853546143,
      "step": 2040
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.618747345980904e-06,
      "logits/chosen": -0.942598819732666,
      "logits/rejected": -0.03942962735891342,
      "logps/chosen": -508.3501892089844,
      "logps/rejected": -809.349365234375,
      "loss": 0.3591,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.922590970993042,
      "rewards/margins": 2.0935001373291016,
      "rewards/rejected": -4.016091346740723,
      "step": 2050
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.595923867132136e-06,
      "logits/chosen": -0.9166833162307739,
      "logits/rejected": 0.08796543627977371,
      "logps/chosen": -561.3500366210938,
      "logps/rejected": -813.5736083984375,
      "loss": 0.3879,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.1458635330200195,
      "rewards/margins": 1.7413724660873413,
      "rewards/rejected": -3.887235641479492,
      "step": 2060
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5730923786680672e-06,
      "logits/chosen": -0.8908794522285461,
      "logits/rejected": 0.45012766122817993,
      "logps/chosen": -528.106689453125,
      "logps/rejected": -768.5990600585938,
      "loss": 0.3902,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.0598998069763184,
      "rewards/margins": 1.840790033340454,
      "rewards/rejected": -3.9006900787353516,
      "step": 2070
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5502547870114137e-06,
      "logits/chosen": -0.44030576944351196,
      "logits/rejected": 0.27115920186042786,
      "logps/chosen": -541.02392578125,
      "logps/rejected": -866.5523681640625,
      "loss": 0.4173,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.1856839656829834,
      "rewards/margins": 2.092510223388672,
      "rewards/rejected": -4.278193473815918,
      "step": 2080
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.527412999094507e-06,
      "logits/chosen": -0.960748553276062,
      "logits/rejected": 0.4776690602302551,
      "logps/chosen": -641.7643432617188,
      "logps/rejected": -842.5614013671875,
      "loss": 0.4317,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.455843448638916,
      "rewards/margins": 1.5613322257995605,
      "rewards/rejected": -4.017176151275635,
      "step": 2090
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.504568922200064e-06,
      "logits/chosen": -1.032212495803833,
      "logits/rejected": 0.38921135663986206,
      "logps/chosen": -526.1879272460938,
      "logps/rejected": -780.3800659179688,
      "loss": 0.4037,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.8752429485321045,
      "rewards/margins": 2.1657333374023438,
      "rewards/rejected": -4.040976524353027,
      "step": 2100
    },
    {
      "epoch": 0.55,
      "eval_logits/chosen": 0.9488687515258789,
      "eval_logits/rejected": 1.9484919309616089,
      "eval_logps/chosen": -568.4110107421875,
      "eval_logps/rejected": -807.3529052734375,
      "eval_loss": 0.43342798948287964,
      "eval_rewards/accuracies": 0.796999990940094,
      "eval_rewards/chosen": -2.127501964569092,
      "eval_rewards/margins": 1.7560021877288818,
      "eval_rewards/rejected": -3.8835039138793945,
      "eval_runtime": 1369.4586,
      "eval_samples_per_second": 1.46,
      "eval_steps_per_second": 0.365,
      "step": 2100
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4817244638019333e-06,
      "logits/chosen": -0.7853974103927612,
      "logits/rejected": 0.26276087760925293,
      "logps/chosen": -576.4677734375,
      "logps/rejected": -882.1696166992188,
      "loss": 0.4205,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.413973331451416,
      "rewards/margins": 1.9889461994171143,
      "rewards/rejected": -4.402919292449951,
      "step": 2110
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4588815314058155e-06,
      "logits/chosen": -0.7402883768081665,
      "logits/rejected": 0.13213138282299042,
      "logps/chosen": -515.807373046875,
      "logps/rejected": -840.2550048828125,
      "loss": 0.4364,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.9831836223602295,
      "rewards/margins": 2.3909099102020264,
      "rewards/rejected": -4.374093532562256,
      "step": 2120
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4360420323899922e-06,
      "logits/chosen": -0.9780701398849487,
      "logits/rejected": -0.47581759095191956,
      "logps/chosen": -536.0255126953125,
      "logps/rejected": -821.7635498046875,
      "loss": 0.4016,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8594684600830078,
      "rewards/margins": 1.7201998233795166,
      "rewards/rejected": -3.5796680450439453,
      "step": 2130
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4132078738460585e-06,
      "logits/chosen": -0.7851123809814453,
      "logits/rejected": -0.3619709610939026,
      "logps/chosen": -536.1060791015625,
      "logps/rejected": -812.8576049804688,
      "loss": 0.3879,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.774129867553711,
      "rewards/margins": 1.8565963506698608,
      "rewards/rejected": -3.6307265758514404,
      "step": 2140
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3903809624196826e-06,
      "logits/chosen": -1.0001862049102783,
      "logits/rejected": -0.3588159680366516,
      "logps/chosen": -524.4048461914062,
      "logps/rejected": -757.30224609375,
      "loss": 0.4045,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6777470111846924,
      "rewards/margins": 1.4243619441986084,
      "rewards/rejected": -3.10210919380188,
      "step": 2150
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3675632041513978e-06,
      "logits/chosen": -0.8742032051086426,
      "logits/rejected": 0.1250528246164322,
      "logps/chosen": -591.0662841796875,
      "logps/rejected": -873.0941162109375,
      "loss": 0.3503,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.1168835163116455,
      "rewards/margins": 1.8603336811065674,
      "rewards/rejected": -3.9772167205810547,
      "step": 2160
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3447565043174533e-06,
      "logits/chosen": -0.9236310124397278,
      "logits/rejected": -0.3211767077445984,
      "logps/chosen": -604.6006469726562,
      "logps/rejected": -848.4435424804688,
      "loss": 0.3555,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.398642063140869,
      "rewards/margins": 1.6593701839447021,
      "rewards/rejected": -4.058012962341309,
      "step": 2170
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.321962767270724e-06,
      "logits/chosen": -0.4545938968658447,
      "logits/rejected": -0.04614262655377388,
      "logps/chosen": -522.8587646484375,
      "logps/rejected": -840.7001953125,
      "loss": 0.424,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.117671012878418,
      "rewards/margins": 2.160691499710083,
      "rewards/rejected": -4.278363227844238,
      "step": 2180
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.299183896281692e-06,
      "logits/chosen": -0.8900741338729858,
      "logits/rejected": 0.11245179176330566,
      "logps/chosen": -621.9942626953125,
      "logps/rejected": -912.0670776367188,
      "loss": 0.3948,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.5180599689483643,
      "rewards/margins": 2.2509348392486572,
      "rewards/rejected": -4.768994331359863,
      "step": 2190
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2764217933795297e-06,
      "logits/chosen": -1.2973425388336182,
      "logits/rejected": 0.3502582907676697,
      "logps/chosen": -648.1968994140625,
      "logps/rejected": -849.2420043945312,
      "loss": 0.4569,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.5464279651641846,
      "rewards/margins": 1.8135731220245361,
      "rewards/rejected": -4.3600006103515625,
      "step": 2200
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2536783591932786e-06,
      "logits/chosen": -1.0717593431472778,
      "logits/rejected": -0.04763598367571831,
      "logps/chosen": -564.8670654296875,
      "logps/rejected": -862.4423828125,
      "loss": 0.383,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.1421093940734863,
      "rewards/margins": 2.115607976913452,
      "rewards/rejected": -4.257718086242676,
      "step": 2210
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.230955492793149e-06,
      "logits/chosen": -0.6327053308486938,
      "logits/rejected": -0.22368088364601135,
      "logps/chosen": -552.7775268554688,
      "logps/rejected": -753.1697998046875,
      "loss": 0.4299,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.939135193824768,
      "rewards/margins": 1.3090981245040894,
      "rewards/rejected": -3.2482333183288574,
      "step": 2220
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.208255091531947e-06,
      "logits/chosen": -0.9401483535766602,
      "logits/rejected": 0.13758346438407898,
      "logps/chosen": -531.6614990234375,
      "logps/rejected": -746.9278564453125,
      "loss": 0.4358,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.8913850784301758,
      "rewards/margins": 1.5273140668869019,
      "rewards/rejected": -3.418699264526367,
      "step": 2230
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1855790508866435e-06,
      "logits/chosen": -1.4784080982208252,
      "logits/rejected": 0.4357272982597351,
      "logps/chosen": -597.9056396484375,
      "logps/rejected": -766.9682006835938,
      "loss": 0.4395,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.0671706199645996,
      "rewards/margins": 1.5233346223831177,
      "rewards/rejected": -3.5905051231384277,
      "step": 2240
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.162929264300107e-06,
      "logits/chosen": -1.0196201801300049,
      "logits/rejected": 0.6838423013687134,
      "logps/chosen": -581.2928466796875,
      "logps/rejected": -801.315185546875,
      "loss": 0.4091,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.0828678607940674,
      "rewards/margins": 1.6539685726165771,
      "rewards/rejected": -3.7368361949920654,
      "step": 2250
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1403076230230006e-06,
      "logits/chosen": -0.6144061088562012,
      "logits/rejected": -0.3462333083152771,
      "logps/chosen": -555.2684326171875,
      "logps/rejected": -745.2354736328125,
      "loss": 0.4423,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8994195461273193,
      "rewards/margins": 1.2702502012252808,
      "rewards/rejected": -3.1696696281433105,
      "step": 2260
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.11771601595586e-06,
      "logits/chosen": -1.3248759508132935,
      "logits/rejected": -0.06512956321239471,
      "logps/chosen": -526.8836669921875,
      "logps/rejected": -727.4069213867188,
      "loss": 0.4316,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.7024120092391968,
      "rewards/margins": 1.4804304838180542,
      "rewards/rejected": -3.182842493057251,
      "step": 2270
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0951563294913737e-06,
      "logits/chosen": -0.8413470387458801,
      "logits/rejected": -0.06420852988958359,
      "logps/chosen": -514.109375,
      "logps/rejected": -770.6260986328125,
      "loss": 0.3656,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.8018481731414795,
      "rewards/margins": 1.71941339969635,
      "rewards/rejected": -3.5212619304656982,
      "step": 2280
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0726304473568693e-06,
      "logits/chosen": -1.5680863857269287,
      "logits/rejected": 0.14398300647735596,
      "logps/chosen": -554.5525512695312,
      "logps/rejected": -771.2307739257812,
      "loss": 0.4331,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.7925901412963867,
      "rewards/margins": 1.641396164894104,
      "rewards/rejected": -3.433986186981201,
      "step": 2290
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.050140250457023e-06,
      "logits/chosen": -1.147033452987671,
      "logits/rejected": -0.169643372297287,
      "logps/chosen": -544.5403442382812,
      "logps/rejected": -771.8599243164062,
      "loss": 0.411,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.702807068824768,
      "rewards/margins": 1.5390836000442505,
      "rewards/rejected": -3.2418906688690186,
      "step": 2300
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0276876167168042e-06,
      "logits/chosen": -1.0450040102005005,
      "logits/rejected": -0.5411997437477112,
      "logps/chosen": -464.33636474609375,
      "logps/rejected": -644.3059692382812,
      "loss": 0.4458,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.661592721939087,
      "rewards/margins": 1.0601894855499268,
      "rewards/rejected": -2.7217824459075928,
      "step": 2310
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.0052744209246682e-06,
      "logits/chosen": -0.9377092123031616,
      "logits/rejected": -0.40263956785202026,
      "logps/chosen": -498.0686950683594,
      "logps/rejected": -739.6976318359375,
      "loss": 0.4114,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.7086822986602783,
      "rewards/margins": 1.4424405097961426,
      "rewards/rejected": -3.151122570037842,
      "step": 2320
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9829025345760127e-06,
      "logits/chosen": -0.7755551338195801,
      "logits/rejected": -0.34596508741378784,
      "logps/chosen": -440.62872314453125,
      "logps/rejected": -650.15478515625,
      "loss": 0.3539,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.3974610567092896,
      "rewards/margins": 1.319804310798645,
      "rewards/rejected": -2.7172653675079346,
      "step": 2330
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9605738257169115e-06,
      "logits/chosen": -1.3644423484802246,
      "logits/rejected": -0.3873172700405121,
      "logps/chosen": -490.96466064453125,
      "logps/rejected": -707.909423828125,
      "loss": 0.3942,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.6896295547485352,
      "rewards/margins": 1.3346831798553467,
      "rewards/rejected": -3.0243124961853027,
      "step": 2340
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9382901587881275e-06,
      "logits/chosen": -1.3390328884124756,
      "logits/rejected": -0.3258362114429474,
      "logps/chosen": -560.3353271484375,
      "logps/rejected": -710.94091796875,
      "loss": 0.4555,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.788905143737793,
      "rewards/margins": 1.1810920238494873,
      "rewards/rejected": -2.9699971675872803,
      "step": 2350
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.916053394469437e-06,
      "logits/chosen": -1.2866407632827759,
      "logits/rejected": -0.09522955119609833,
      "logps/chosen": -534.5072021484375,
      "logps/rejected": -711.5755004882812,
      "loss": 0.3582,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.719448447227478,
      "rewards/margins": 1.4744646549224854,
      "rewards/rejected": -3.193912982940674,
      "step": 2360
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8938653895242604e-06,
      "logits/chosen": -1.3210828304290771,
      "logits/rejected": 0.18320707976818085,
      "logps/chosen": -570.3659057617188,
      "logps/rejected": -748.9262084960938,
      "loss": 0.38,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.9527915716171265,
      "rewards/margins": 1.4824120998382568,
      "rewards/rejected": -3.4352035522460938,
      "step": 2370
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8717279966446267e-06,
      "logits/chosen": -0.6077791452407837,
      "logits/rejected": -0.07136712223291397,
      "logps/chosen": -561.3985595703125,
      "logps/rejected": -833.6058349609375,
      "loss": 0.4602,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8687776327133179,
      "rewards/margins": 2.1088478565216064,
      "rewards/rejected": -3.9776253700256348,
      "step": 2380
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8496430642964698e-06,
      "logits/chosen": -0.6436141133308411,
      "logits/rejected": -0.7390815615653992,
      "logps/chosen": -535.98291015625,
      "logps/rejected": -785.1193237304688,
      "loss": 0.4384,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8680446147918701,
      "rewards/margins": 1.4708973169326782,
      "rewards/rejected": -3.338942050933838,
      "step": 2390
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.827612436565286e-06,
      "logits/chosen": -0.7472543716430664,
      "logits/rejected": -0.552331805229187,
      "logps/chosen": -512.9714965820312,
      "logps/rejected": -776.1787109375,
      "loss": 0.3829,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.701409101486206,
      "rewards/margins": 1.6592018604278564,
      "rewards/rejected": -3.3606104850769043,
      "step": 2400
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": 0.5047381520271301,
      "eval_logits/rejected": 1.5421233177185059,
      "eval_logps/chosen": -543.5669555664062,
      "eval_logps/rejected": -768.0193481445312,
      "eval_loss": 0.4248420000076294,
      "eval_rewards/accuracies": 0.8009999990463257,
      "eval_rewards/chosen": -1.879061222076416,
      "eval_rewards/margins": 1.611107587814331,
      "eval_rewards/rejected": -3.490169048309326,
      "eval_runtime": 1378.4799,
      "eval_samples_per_second": 1.451,
      "eval_steps_per_second": 0.363,
      "step": 2400
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8056379530021492e-06,
      "logits/chosen": -1.644568681716919,
      "logits/rejected": 0.2553193271160126,
      "logps/chosen": -535.88134765625,
      "logps/rejected": -778.2227783203125,
      "loss": 0.3769,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.729379415512085,
      "rewards/margins": 1.8933357000350952,
      "rewards/rejected": -3.6227145195007324,
      "step": 2410
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7837214484701154e-06,
      "logits/chosen": -1.0312998294830322,
      "logits/rejected": -0.9269927144050598,
      "logps/chosen": -492.1142578125,
      "logps/rejected": -759.744140625,
      "loss": 0.3937,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.6029274463653564,
      "rewards/margins": 1.807284951210022,
      "rewards/rejected": -3.410212278366089,
      "step": 2420
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7618647529910043e-06,
      "logits/chosen": -1.2176530361175537,
      "logits/rejected": 0.223758265376091,
      "logps/chosen": -529.0576171875,
      "logps/rejected": -756.2348022460938,
      "loss": 0.4664,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.851143479347229,
      "rewards/margins": 1.5376180410385132,
      "rewards/rejected": -3.388761520385742,
      "step": 2430
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7400696915925996e-06,
      "logits/chosen": -1.2333937883377075,
      "logits/rejected": -0.22815477848052979,
      "logps/chosen": -530.9912719726562,
      "logps/rejected": -742.2661743164062,
      "loss": 0.396,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.743680715560913,
      "rewards/margins": 1.500349521636963,
      "rewards/rejected": -3.244030475616455,
      "step": 2440
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.718338084156254e-06,
      "logits/chosen": -1.519798994064331,
      "logits/rejected": -0.11792447417974472,
      "logps/chosen": -558.024658203125,
      "logps/rejected": -786.2930908203125,
      "loss": 0.4233,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.6490389108657837,
      "rewards/margins": 1.6921495199203491,
      "rewards/rejected": -3.341188430786133,
      "step": 2450
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.6966717452649372e-06,
      "logits/chosen": -1.2660577297210693,
      "logits/rejected": -0.0037010847590863705,
      "logps/chosen": -472.52471923828125,
      "logps/rejected": -651.4722900390625,
      "loss": 0.4358,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.6838678121566772,
      "rewards/margins": 1.2879998683929443,
      "rewards/rejected": -2.971867561340332,
      "step": 2460
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6750724840517103e-06,
      "logits/chosen": -1.2322640419006348,
      "logits/rejected": -0.4316504895687103,
      "logps/chosen": -495.26544189453125,
      "logps/rejected": -720.3032836914062,
      "loss": 0.4283,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.5014667510986328,
      "rewards/margins": 1.5642062425613403,
      "rewards/rejected": -3.0656726360321045,
      "step": 2470
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6535421040486686e-06,
      "logits/chosen": -0.9225546717643738,
      "logits/rejected": 0.307370662689209,
      "logps/chosen": -537.8445434570312,
      "logps/rejected": -764.1954345703125,
      "loss": 0.4295,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8402236700057983,
      "rewards/margins": 1.5403947830200195,
      "rewards/rejected": -3.3806185722351074,
      "step": 2480
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6320824030363458e-06,
      "logits/chosen": -0.8234320878982544,
      "logits/rejected": -0.4902985095977783,
      "logps/chosen": -521.9378662109375,
      "logps/rejected": -800.2440185546875,
      "loss": 0.3775,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.6927446126937866,
      "rewards/margins": 1.8545596599578857,
      "rewards/rejected": -3.547304630279541,
      "step": 2490
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6106951728936028e-06,
      "logits/chosen": -0.9694339036941528,
      "logits/rejected": -0.04356659576296806,
      "logps/chosen": -550.1474609375,
      "logps/rejected": -714.2723999023438,
      "loss": 0.3934,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.6208375692367554,
      "rewards/margins": 1.5635387897491455,
      "rewards/rejected": -3.1843764781951904,
      "step": 2500
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5893821994479996e-06,
      "logits/chosen": -1.3845258951187134,
      "logits/rejected": 0.13586857914924622,
      "logps/chosen": -514.47509765625,
      "logps/rejected": -733.2557373046875,
      "loss": 0.4431,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.7695598602294922,
      "rewards/margins": 1.6184390783309937,
      "rewards/rejected": -3.3879990577697754,
      "step": 2510
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5681452623266868e-06,
      "logits/chosen": -0.6911166310310364,
      "logits/rejected": -0.19654271006584167,
      "logps/chosen": -473.919189453125,
      "logps/rejected": -720.5162963867188,
      "loss": 0.4099,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.6180083751678467,
      "rewards/margins": 1.4277281761169434,
      "rewards/rejected": -3.04573655128479,
      "step": 2520
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5469861348078014e-06,
      "logits/chosen": -1.3716986179351807,
      "logits/rejected": -0.06388586759567261,
      "logps/chosen": -564.1226196289062,
      "logps/rejected": -798.0968017578125,
      "loss": 0.4469,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9020678997039795,
      "rewards/margins": 1.5944623947143555,
      "rewards/rejected": -3.496530532836914,
      "step": 2530
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5259065836724035e-06,
      "logits/chosen": -0.8546286821365356,
      "logits/rejected": -0.3107864260673523,
      "logps/chosen": -550.43017578125,
      "logps/rejected": -754.4808349609375,
      "loss": 0.3968,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.006619453430176,
      "rewards/margins": 1.325626015663147,
      "rewards/rejected": -3.3322455883026123,
      "step": 2540
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5049083690569456e-06,
      "logits/chosen": -1.1054461002349854,
      "logits/rejected": -0.2836257517337799,
      "logps/chosen": -498.7848205566406,
      "logps/rejected": -744.7251586914062,
      "loss": 0.4576,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7602741718292236,
      "rewards/margins": 1.6243937015533447,
      "rewards/rejected": -3.3846676349639893,
      "step": 2550
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4839932443063057e-06,
      "logits/chosen": -1.052145004272461,
      "logits/rejected": -0.4817884564399719,
      "logps/chosen": -500.67022705078125,
      "logps/rejected": -772.7791748046875,
      "loss": 0.4111,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7564786672592163,
      "rewards/margins": 1.9239161014556885,
      "rewards/rejected": -3.6803946495056152,
      "step": 2560
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4631629558273803e-06,
      "logits/chosen": -1.1972812414169312,
      "logits/rejected": -0.44412803649902344,
      "logps/chosen": -612.2957153320312,
      "logps/rejected": -818.8905029296875,
      "loss": 0.4121,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.9536006450653076,
      "rewards/margins": 1.6227566003799438,
      "rewards/rejected": -3.576357364654541,
      "step": 2570
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4424192429432657e-06,
      "logits/chosen": -1.1940516233444214,
      "logits/rejected": -0.6190989017486572,
      "logps/chosen": -545.1439208984375,
      "logps/rejected": -734.8392333984375,
      "loss": 0.5085,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8257439136505127,
      "rewards/margins": 1.1313966512680054,
      "rewards/rejected": -2.9571404457092285,
      "step": 2580
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.421763837748016e-06,
      "logits/chosen": -0.8670721054077148,
      "logits/rejected": 0.19919352233409882,
      "logps/chosen": -536.2199096679688,
      "logps/rejected": -726.8831787109375,
      "loss": 0.3427,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.5812265872955322,
      "rewards/margins": 1.5793380737304688,
      "rewards/rejected": -3.160564422607422,
      "step": 2590
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.401198464962021e-06,
      "logits/chosen": -0.7592190504074097,
      "logits/rejected": -0.6002682447433472,
      "logps/chosen": -536.698974609375,
      "logps/rejected": -750.5850219726562,
      "loss": 0.4207,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.7439777851104736,
      "rewards/margins": 1.444570779800415,
      "rewards/rejected": -3.1885488033294678,
      "step": 2600
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3807248417879896e-06,
      "logits/chosen": -0.9805696606636047,
      "logits/rejected": -0.1842125803232193,
      "logps/chosen": -496.516845703125,
      "logps/rejected": -714.7666015625,
      "loss": 0.4135,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.775261640548706,
      "rewards/margins": 1.5725538730621338,
      "rewards/rejected": -3.347815752029419,
      "step": 2610
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3603446777675665e-06,
      "logits/chosen": -1.047123670578003,
      "logits/rejected": -0.30356377363204956,
      "logps/chosen": -510.2223205566406,
      "logps/rejected": -704.94091796875,
      "loss": 0.4062,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.7939850091934204,
      "rewards/margins": 1.3114802837371826,
      "rewards/rejected": -3.1054649353027344,
      "step": 2620
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3400596746385817e-06,
      "logits/chosen": -1.089388370513916,
      "logits/rejected": -0.30947428941726685,
      "logps/chosen": -532.441650390625,
      "logps/rejected": -787.63720703125,
      "loss": 0.3845,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.7372934818267822,
      "rewards/margins": 1.8088710308074951,
      "rewards/rejected": -3.5461642742156982,
      "step": 2630
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3198715261929587e-06,
      "logits/chosen": -0.878930389881134,
      "logits/rejected": -0.4860079884529114,
      "logps/chosen": -504.37701416015625,
      "logps/rejected": -719.3287963867188,
      "loss": 0.4429,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.7471263408660889,
      "rewards/margins": 1.4954102039337158,
      "rewards/rejected": -3.2425365447998047,
      "step": 2640
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.2997819181352823e-06,
      "logits/chosen": -1.1908671855926514,
      "logits/rejected": 0.06708762049674988,
      "logps/chosen": -486.3038024902344,
      "logps/rejected": -774.8641357421875,
      "loss": 0.3094,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.6818081140518188,
      "rewards/margins": 2.000152587890625,
      "rewards/rejected": -3.6819605827331543,
      "step": 2650
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2797925279420454e-06,
      "logits/chosen": -0.9907468557357788,
      "logits/rejected": -0.3420366942882538,
      "logps/chosen": -546.3676147460938,
      "logps/rejected": -735.4510498046875,
      "loss": 0.3966,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.9783893823623657,
      "rewards/margins": 1.4355647563934326,
      "rewards/rejected": -3.413954257965088,
      "step": 2660
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2599050247215764e-06,
      "logits/chosen": -1.4948866367340088,
      "logits/rejected": 0.49219974875450134,
      "logps/chosen": -573.19140625,
      "logps/rejected": -743.2158203125,
      "loss": 0.4396,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.086617946624756,
      "rewards/margins": 1.3520699739456177,
      "rewards/rejected": -3.438687801361084,
      "step": 2670
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2401210690746705e-06,
      "logits/chosen": -0.823261559009552,
      "logits/rejected": -0.15638458728790283,
      "logps/chosen": -535.7666625976562,
      "logps/rejected": -808.8283081054688,
      "loss": 0.4014,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.9818382263183594,
      "rewards/margins": 1.6475715637207031,
      "rewards/rejected": -3.629409074783325,
      "step": 2680
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2204423129559306e-06,
      "logits/chosen": -0.9111806154251099,
      "logits/rejected": -0.8399251699447632,
      "logps/chosen": -543.661865234375,
      "logps/rejected": -744.8538818359375,
      "loss": 0.4439,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.9010549783706665,
      "rewards/margins": 1.2256118059158325,
      "rewards/rejected": -3.126666784286499,
      "step": 2690
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.20087039953583e-06,
      "logits/chosen": -1.051735281944275,
      "logits/rejected": -0.22251495718955994,
      "logps/chosen": -584.2539672851562,
      "logps/rejected": -825.45703125,
      "loss": 0.47,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.9151395559310913,
      "rewards/margins": 1.5970687866210938,
      "rewards/rejected": -3.5122084617614746,
      "step": 2700
    },
    {
      "epoch": 0.71,
      "eval_logits/chosen": 0.5343354940414429,
      "eval_logits/rejected": 1.5151687860488892,
      "eval_logps/chosen": -541.308837890625,
      "eval_logps/rejected": -759.2698974609375,
      "eval_loss": 0.4210600256919861,
      "eval_rewards/accuracies": 0.8029999732971191,
      "eval_rewards/chosen": -1.8564802408218384,
      "eval_rewards/margins": 1.5461931228637695,
      "eval_rewards/rejected": -3.4026734828948975,
      "eval_runtime": 1376.5386,
      "eval_samples_per_second": 1.453,
      "eval_steps_per_second": 0.363,
      "step": 2700
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.181406963063507e-06,
      "logits/chosen": -1.0936121940612793,
      "logits/rejected": -0.3421854078769684,
      "logps/chosen": -551.6010131835938,
      "logps/rejected": -727.1361083984375,
      "loss": 0.523,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.9586423635482788,
      "rewards/margins": 1.3370798826217651,
      "rewards/rejected": -3.295722484588623,
      "step": 2710
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1620536287303052e-06,
      "logits/chosen": -1.2988653182983398,
      "logits/rejected": -0.4047287404537201,
      "logps/chosen": -515.3692626953125,
      "logps/rejected": -714.9964599609375,
      "loss": 0.4243,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6312452554702759,
      "rewards/margins": 1.321842074394226,
      "rewards/rejected": -2.953087091445923,
      "step": 2720
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1428120125340717e-06,
      "logits/chosen": -1.1262309551239014,
      "logits/rejected": 0.1220148354768753,
      "logps/chosen": -560.1183471679688,
      "logps/rejected": -755.31103515625,
      "loss": 0.4397,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7473366260528564,
      "rewards/margins": 1.4563024044036865,
      "rewards/rejected": -3.203639268875122,
      "step": 2730
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.123683721144223e-06,
      "logits/chosen": -1.1425328254699707,
      "logits/rejected": -0.3113950788974762,
      "logps/chosen": -507.19598388671875,
      "logps/rejected": -742.7738037109375,
      "loss": 0.3137,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6102272272109985,
      "rewards/margins": 1.5865800380706787,
      "rewards/rejected": -3.196807384490967,
      "step": 2740
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1046703517675848e-06,
      "logits/chosen": -1.2373132705688477,
      "logits/rejected": -0.09322497248649597,
      "logps/chosen": -504.95501708984375,
      "logps/rejected": -680.431396484375,
      "loss": 0.4333,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.792452096939087,
      "rewards/margins": 1.1824333667755127,
      "rewards/rejected": -2.9748852252960205,
      "step": 2750
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.085773492015028e-06,
      "logits/chosen": -0.8464914560317993,
      "logits/rejected": -0.11784086376428604,
      "logps/chosen": -549.912841796875,
      "logps/rejected": -770.5538330078125,
      "loss": 0.4082,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.764224648475647,
      "rewards/margins": 1.4827383756637573,
      "rewards/rejected": -3.2469630241394043,
      "step": 2760
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0669947197689034e-06,
      "logits/chosen": -0.7767351269721985,
      "logits/rejected": 0.08518421649932861,
      "logps/chosen": -487.90399169921875,
      "logps/rejected": -754.19287109375,
      "loss": 0.3626,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7886440753936768,
      "rewards/margins": 1.6782163381576538,
      "rewards/rejected": -3.466860294342041,
      "step": 2770
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.048335603051291e-06,
      "logits/chosen": -0.9023948907852173,
      "logits/rejected": 0.5675928592681885,
      "logps/chosen": -573.4138793945312,
      "logps/rejected": -760.1019897460938,
      "loss": 0.3304,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.9305530786514282,
      "rewards/margins": 1.7111543416976929,
      "rewards/rejected": -3.641707181930542,
      "step": 2780
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0297976998930665e-06,
      "logits/chosen": -1.0089476108551025,
      "logits/rejected": 0.039520103484392166,
      "logps/chosen": -544.458251953125,
      "logps/rejected": -781.134521484375,
      "loss": 0.4407,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9217383861541748,
      "rewards/margins": 1.6432183980941772,
      "rewards/rejected": -3.5649566650390625,
      "step": 2790
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0113825582038078e-06,
      "logits/chosen": -0.9111200571060181,
      "logits/rejected": 0.09320324659347534,
      "logps/chosen": -485.19464111328125,
      "logps/rejected": -772.6851806640625,
      "loss": 0.3662,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6882908344268799,
      "rewards/margins": 1.9764955043792725,
      "rewards/rejected": -3.6647861003875732,
      "step": 2800
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.930917156425477e-07,
      "logits/chosen": -1.3273289203643799,
      "logits/rejected": 0.12840789556503296,
      "logps/chosen": -539.8751220703125,
      "logps/rejected": -770.5255126953125,
      "loss": 0.4417,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.8574135303497314,
      "rewards/margins": 1.6678911447525024,
      "rewards/rejected": -3.525304079055786,
      "step": 2810
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.749266994893756e-07,
      "logits/chosen": -1.2743171453475952,
      "logits/rejected": -0.29028937220573425,
      "logps/chosen": -505.72515869140625,
      "logps/rejected": -708.1859130859375,
      "loss": 0.3927,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.6994062662124634,
      "rewards/margins": 1.5197365283966064,
      "rewards/rejected": -3.2191429138183594,
      "step": 2820
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.56889026517913e-07,
      "logits/chosen": -1.2260441780090332,
      "logits/rejected": 0.2879168689250946,
      "logps/chosen": -580.123291015625,
      "logps/rejected": -747.0691528320312,
      "loss": 0.3703,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.0501606464385986,
      "rewards/margins": 1.359299659729004,
      "rewards/rejected": -3.4094605445861816,
      "step": 2830
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.389802028686617e-07,
      "logits/chosen": -1.1472103595733643,
      "logits/rejected": 0.685859203338623,
      "logps/chosen": -564.8762817382812,
      "logps/rejected": -833.5897216796875,
      "loss": 0.3696,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.9314100742340088,
      "rewards/margins": 2.114525318145752,
      "rewards/rejected": -4.04593563079834,
      "step": 2840
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.212017239232427e-07,
      "logits/chosen": -1.1055911779403687,
      "logits/rejected": 0.17128732800483704,
      "logps/chosen": -461.57757568359375,
      "logps/rejected": -762.3133544921875,
      "loss": 0.3902,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.596985101699829,
      "rewards/margins": 1.919716477394104,
      "rewards/rejected": -3.5167019367218018,
      "step": 2850
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.03555074179533e-07,
      "logits/chosen": -0.5496357679367065,
      "logits/rejected": -0.5428146123886108,
      "logps/chosen": -548.8982543945312,
      "logps/rejected": -794.3575439453125,
      "loss": 0.4303,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.8840789794921875,
      "rewards/margins": 1.6221778392791748,
      "rewards/rejected": -3.506256580352783,
      "step": 2860
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.860417271277067e-07,
      "logits/chosen": -1.2351821660995483,
      "logits/rejected": -0.3069414794445038,
      "logps/chosen": -512.6089477539062,
      "logps/rejected": -765.6232299804688,
      "loss": 0.3669,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.8200585842132568,
      "rewards/margins": 1.858338713645935,
      "rewards/rejected": -3.6783974170684814,
      "step": 2870
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.686631451272029e-07,
      "logits/chosen": -0.9176104664802551,
      "logits/rejected": 0.11124134063720703,
      "logps/chosen": -530.0169067382812,
      "logps/rejected": -772.3027954101562,
      "loss": 0.3666,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.805767297744751,
      "rewards/margins": 1.8668705224990845,
      "rewards/rejected": -3.672637462615967,
      "step": 2880
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.514207792846168e-07,
      "logits/chosen": -0.8543869853019714,
      "logits/rejected": -0.33468276262283325,
      "logps/chosen": -563.6397705078125,
      "logps/rejected": -726.9634399414062,
      "loss": 0.3891,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.042240619659424,
      "rewards/margins": 1.2924143075942993,
      "rewards/rejected": -3.3346545696258545,
      "step": 2890
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.343160693325356e-07,
      "logits/chosen": -0.5689171552658081,
      "logits/rejected": -0.3701861500740051,
      "logps/chosen": -522.55078125,
      "logps/rejected": -799.8737182617188,
      "loss": 0.3666,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9046812057495117,
      "rewards/margins": 1.847630262374878,
      "rewards/rejected": -3.7523112297058105,
      "step": 2900
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.173504435093174e-07,
      "logits/chosen": -1.1127324104309082,
      "logits/rejected": 0.49559181928634644,
      "logps/chosen": -580.339599609375,
      "logps/rejected": -790.05419921875,
      "loss": 0.4036,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.1893162727355957,
      "rewards/margins": 1.816868543624878,
      "rewards/rejected": -4.0061845779418945,
      "step": 2910
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.00525318439836e-07,
      "logits/chosen": -1.1705108880996704,
      "logits/rejected": 0.1877966821193695,
      "logps/chosen": -577.6532592773438,
      "logps/rejected": -775.518310546875,
      "loss": 0.5292,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.123340129852295,
      "rewards/margins": 1.601309061050415,
      "rewards/rejected": -3.724648952484131,
      "step": 2920
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.838420990171927e-07,
      "logits/chosen": -0.675153374671936,
      "logits/rejected": -0.15449100732803345,
      "logps/chosen": -529.3169555664062,
      "logps/rejected": -849.1887817382812,
      "loss": 0.3346,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.992500901222229,
      "rewards/margins": 2.1089320182800293,
      "rewards/rejected": -4.101432800292969,
      "step": 2930
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.673021782854084e-07,
      "logits/chosen": -1.0276451110839844,
      "logits/rejected": 0.12752141058444977,
      "logps/chosen": -569.2623901367188,
      "logps/rejected": -797.4970703125,
      "loss": 0.4235,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.9917293787002563,
      "rewards/margins": 1.7630789279937744,
      "rewards/rejected": -3.7548089027404785,
      "step": 2940
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.509069373231039e-07,
      "logits/chosen": -0.532359778881073,
      "logits/rejected": -0.37800487875938416,
      "logps/chosen": -572.4589233398438,
      "logps/rejected": -849.5089111328125,
      "loss": 0.4203,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.19490385055542,
      "rewards/margins": 1.7656660079956055,
      "rewards/rejected": -3.9605698585510254,
      "step": 2950
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.346577451281822e-07,
      "logits/chosen": -0.9249979853630066,
      "logits/rejected": 0.5974918603897095,
      "logps/chosen": -583.9189453125,
      "logps/rejected": -770.7642822265625,
      "loss": 0.4452,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.059628486633301,
      "rewards/margins": 1.6830648183822632,
      "rewards/rejected": -3.7426934242248535,
      "step": 2960
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.185559585035138e-07,
      "logits/chosen": -0.7646081447601318,
      "logits/rejected": -0.08741030842065811,
      "logps/chosen": -494.5181579589844,
      "logps/rejected": -819.3825073242188,
      "loss": 0.4152,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.7815096378326416,
      "rewards/margins": 2.074185848236084,
      "rewards/rejected": -3.8556952476501465,
      "step": 2970
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.026029219436504e-07,
      "logits/chosen": -1.0040438175201416,
      "logits/rejected": -0.2370959222316742,
      "logps/chosen": -547.5843505859375,
      "logps/rejected": -775.7852783203125,
      "loss": 0.3377,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8116861581802368,
      "rewards/margins": 1.7394940853118896,
      "rewards/rejected": -3.551180362701416,
      "step": 2980
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.867999675225523e-07,
      "logits/chosen": -0.8310391306877136,
      "logits/rejected": -0.02760641649365425,
      "logps/chosen": -590.3953857421875,
      "logps/rejected": -830.1414184570312,
      "loss": 0.3744,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.0531251430511475,
      "rewards/margins": 1.790724515914917,
      "rewards/rejected": -3.843848705291748,
      "step": 2990
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.711484147823663e-07,
      "logits/chosen": -1.0045768022537231,
      "logits/rejected": -0.32932716608047485,
      "logps/chosen": -519.6864013671875,
      "logps/rejected": -840.6529541015625,
      "loss": 0.3769,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.8191581964492798,
      "rewards/margins": 1.9201765060424805,
      "rewards/rejected": -3.7393341064453125,
      "step": 3000
    },
    {
      "epoch": 0.79,
      "eval_logits/chosen": 0.532638430595398,
      "eval_logits/rejected": 1.514159917831421,
      "eval_logps/chosen": -547.6463012695312,
      "eval_logps/rejected": -772.1762084960938,
      "eval_loss": 0.4205494523048401,
      "eval_rewards/accuracies": 0.8009999990463257,
      "eval_rewards/chosen": -1.919854760169983,
      "eval_rewards/margins": 1.6118818521499634,
      "eval_rewards/rejected": -3.5317368507385254,
      "eval_runtime": 1383.4868,
      "eval_samples_per_second": 1.446,
      "eval_steps_per_second": 0.361,
      "step": 3000
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.556495706232413e-07,
      "logits/chosen": -1.1887633800506592,
      "logits/rejected": 0.10280628502368927,
      "logps/chosen": -592.3970947265625,
      "logps/rejected": -802.2684326171875,
      "loss": 0.4594,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.981632947921753,
      "rewards/margins": 1.690386414527893,
      "rewards/rejected": -3.6720194816589355,
      "step": 3010
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.403047291942057e-07,
      "logits/chosen": -1.0731422901153564,
      "logits/rejected": -0.11721036583185196,
      "logps/chosen": -500.95733642578125,
      "logps/rejected": -810.4306030273438,
      "loss": 0.3202,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.6846420764923096,
      "rewards/margins": 2.057126522064209,
      "rewards/rejected": -3.7417690753936768,
      "step": 3020
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.251151717851023e-07,
      "logits/chosen": -0.5943460464477539,
      "logits/rejected": -0.3044959008693695,
      "logps/chosen": -552.7174682617188,
      "logps/rejected": -838.6080322265625,
      "loss": 0.3992,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.064213275909424,
      "rewards/margins": 1.780678153038025,
      "rewards/rejected": -3.844891309738159,
      "step": 3030
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.100821667196041e-07,
      "logits/chosen": -0.9400644302368164,
      "logits/rejected": -0.20678548514842987,
      "logps/chosen": -539.8009033203125,
      "logps/rejected": -764.1534423828125,
      "loss": 0.3418,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.653412103652954,
      "rewards/margins": 1.8391025066375732,
      "rewards/rejected": -3.4925143718719482,
      "step": 3040
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.952069692493062e-07,
      "logits/chosen": -0.7938990592956543,
      "logits/rejected": -0.40649691224098206,
      "logps/chosen": -596.44873046875,
      "logps/rejected": -842.193359375,
      "loss": 0.4439,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.0670700073242188,
      "rewards/margins": 1.7952619791030884,
      "rewards/rejected": -3.8623321056365967,
      "step": 3050
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.80490821448918e-07,
      "logits/chosen": -0.9244669079780579,
      "logits/rejected": -0.5335060358047485,
      "logps/chosen": -596.2068481445312,
      "logps/rejected": -874.0540161132812,
      "loss": 0.391,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.2196078300476074,
      "rewards/margins": 1.8115314245224,
      "rewards/rejected": -4.031139373779297,
      "step": 3060
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.659349521125459e-07,
      "logits/chosen": -0.6347008347511292,
      "logits/rejected": -0.5196251273155212,
      "logps/chosen": -535.7493286132812,
      "logps/rejected": -856.0029296875,
      "loss": 0.4562,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.9471817016601562,
      "rewards/margins": 1.7889111042022705,
      "rewards/rejected": -3.7360928058624268,
      "step": 3070
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.5154057665109e-07,
      "logits/chosen": -1.1485843658447266,
      "logits/rejected": 0.004442277364432812,
      "logps/chosen": -522.5572509765625,
      "logps/rejected": -784.026123046875,
      "loss": 0.3419,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.899583101272583,
      "rewards/margins": 1.6547266244888306,
      "rewards/rejected": -3.554309844970703,
      "step": 3080
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.373088969907586e-07,
      "logits/chosen": -0.7392430901527405,
      "logits/rejected": -0.4351174235343933,
      "logps/chosen": -475.52294921875,
      "logps/rejected": -764.8416748046875,
      "loss": 0.3467,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.6972386837005615,
      "rewards/margins": 1.8677946329116821,
      "rewards/rejected": -3.565033435821533,
      "step": 3090
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.23241101472709e-07,
      "logits/chosen": -0.841932475566864,
      "logits/rejected": -0.11554646492004395,
      "logps/chosen": -619.42724609375,
      "logps/rejected": -848.4366455078125,
      "loss": 0.3589,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.211543560028076,
      "rewards/margins": 1.8801252841949463,
      "rewards/rejected": -4.091668605804443,
      "step": 3100
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.09338364753818e-07,
      "logits/chosen": -1.0390411615371704,
      "logits/rejected": 0.6153230667114258,
      "logps/chosen": -606.2908325195312,
      "logps/rejected": -770.7852783203125,
      "loss": 0.5322,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.2032546997070312,
      "rewards/margins": 1.4495493173599243,
      "rewards/rejected": -3.652804136276245,
      "step": 3110
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.956018477086005e-07,
      "logits/chosen": -0.8461894989013672,
      "logits/rejected": -0.3413206934928894,
      "logps/chosen": -537.2196655273438,
      "logps/rejected": -825.6803588867188,
      "loss": 0.3784,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.8306541442871094,
      "rewards/margins": 1.9641211032867432,
      "rewards/rejected": -3.7947754859924316,
      "step": 3120
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.820326973322764e-07,
      "logits/chosen": -0.8142082095146179,
      "logits/rejected": -0.10756425559520721,
      "logps/chosen": -565.7686157226562,
      "logps/rejected": -817.3389892578125,
      "loss": 0.4628,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.146563768386841,
      "rewards/margins": 1.795789122581482,
      "rewards/rejected": -3.9423530101776123,
      "step": 3130
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.686320466449981e-07,
      "logits/chosen": -0.9902013540267944,
      "logits/rejected": -0.15123017132282257,
      "logps/chosen": -511.6470642089844,
      "logps/rejected": -815.7200317382812,
      "loss": 0.3672,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.8265291452407837,
      "rewards/margins": 2.014096975326538,
      "rewards/rejected": -3.8406262397766113,
      "step": 3140
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.554010145972418e-07,
      "logits/chosen": -0.8389061689376831,
      "logits/rejected": -0.14444035291671753,
      "logps/chosen": -551.7734375,
      "logps/rejected": -798.772705078125,
      "loss": 0.4095,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.063991069793701,
      "rewards/margins": 1.5325976610183716,
      "rewards/rejected": -3.596588611602783,
      "step": 3150
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.4234070597637455e-07,
      "logits/chosen": -0.8169253468513489,
      "logits/rejected": 0.03178207948803902,
      "logps/chosen": -529.4456787109375,
      "logps/rejected": -754.3330078125,
      "loss": 0.3978,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.110259771347046,
      "rewards/margins": 1.4216582775115967,
      "rewards/rejected": -3.5319180488586426,
      "step": 3160
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.2945221131440783e-07,
      "logits/chosen": -0.7767388224601746,
      "logits/rejected": -0.2466239631175995,
      "logps/chosen": -586.1234130859375,
      "logps/rejected": -792.2913818359375,
      "loss": 0.4286,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.040156602859497,
      "rewards/margins": 1.661505937576294,
      "rewards/rejected": -3.701662063598633,
      "step": 3170
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.167366067969381e-07,
      "logits/chosen": -0.8122004270553589,
      "logits/rejected": -0.4158555567264557,
      "logps/chosen": -579.0358276367188,
      "logps/rejected": -846.17626953125,
      "loss": 0.3993,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.079592227935791,
      "rewards/margins": 1.6793367862701416,
      "rewards/rejected": -3.758929491043091,
      "step": 3180
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.041949541732826e-07,
      "logits/chosen": -1.0485798120498657,
      "logits/rejected": -0.36888834834098816,
      "logps/chosen": -630.3906860351562,
      "logps/rejected": -814.126953125,
      "loss": 0.4027,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.191230535507202,
      "rewards/margins": 1.4397324323654175,
      "rewards/rejected": -3.63096284866333,
      "step": 3190
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9182830066782614e-07,
      "logits/chosen": -0.8572107553482056,
      "logits/rejected": -0.33156412839889526,
      "logps/chosen": -506.94781494140625,
      "logps/rejected": -776.4071044921875,
      "loss": 0.348,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.8441927433013916,
      "rewards/margins": 1.8142344951629639,
      "rewards/rejected": -3.6584274768829346,
      "step": 3200
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.796376788925771e-07,
      "logits/chosen": -1.5727512836456299,
      "logits/rejected": 0.38068026304244995,
      "logps/chosen": -552.1102905273438,
      "logps/rejected": -777.140869140625,
      "loss": 0.4251,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.8679554462432861,
      "rewards/margins": 1.6639436483383179,
      "rewards/rejected": -3.5318992137908936,
      "step": 3210
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.676241067609465e-07,
      "logits/chosen": -0.8122023344039917,
      "logits/rejected": 0.24542848765850067,
      "logps/chosen": -533.0785522460938,
      "logps/rejected": -790.8440551757812,
      "loss": 0.3919,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8818871974945068,
      "rewards/margins": 1.9045559167861938,
      "rewards/rejected": -3.7864432334899902,
      "step": 3220
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.5578858740274976e-07,
      "logits/chosen": -1.0608148574829102,
      "logits/rejected": -0.1883460134267807,
      "logps/chosen": -546.2855224609375,
      "logps/rejected": -784.9697265625,
      "loss": 0.425,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.9488483667373657,
      "rewards/margins": 1.45414400100708,
      "rewards/rejected": -3.4029927253723145,
      "step": 3230
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.44132109080447e-07,
      "logits/chosen": -1.0673831701278687,
      "logits/rejected": -0.0653887614607811,
      "logps/chosen": -512.1033935546875,
      "logps/rejected": -733.269287109375,
      "loss": 0.4068,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.7813838720321655,
      "rewards/margins": 1.5279889106750488,
      "rewards/rejected": -3.309372663497925,
      "step": 3240
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.3265564510662344e-07,
      "logits/chosen": -1.1314384937286377,
      "logits/rejected": -0.5283291935920715,
      "logps/chosen": -552.1278076171875,
      "logps/rejected": -818.47509765625,
      "loss": 0.4451,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8823707103729248,
      "rewards/margins": 1.676279067993164,
      "rewards/rejected": -3.558649778366089,
      "step": 3250
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.213601537627195e-07,
      "logits/chosen": -0.9113371968269348,
      "logits/rejected": -0.332157701253891,
      "logps/chosen": -565.0186767578125,
      "logps/rejected": -805.5143432617188,
      "loss": 0.4066,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.2068862915039062,
      "rewards/margins": 1.6060466766357422,
      "rewards/rejected": -3.8129334449768066,
      "step": 3260
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.1024657821901063e-07,
      "logits/chosen": -0.7972087264060974,
      "logits/rejected": -0.6192452311515808,
      "logps/chosen": -460.00244140625,
      "logps/rejected": -661.6898193359375,
      "loss": 0.4204,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.722604513168335,
      "rewards/margins": 1.273694396018982,
      "rewards/rejected": -2.9962992668151855,
      "step": 3270
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.9931584645585654e-07,
      "logits/chosen": -1.2535960674285889,
      "logits/rejected": 0.02778279222548008,
      "logps/chosen": -465.5445861816406,
      "logps/rejected": -718.2966918945312,
      "loss": 0.3941,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.620823860168457,
      "rewards/margins": 1.7875875234603882,
      "rewards/rejected": -3.4084110260009766,
      "step": 3280
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.885688711862136e-07,
      "logits/chosen": -0.6595426201820374,
      "logits/rejected": -0.6082831621170044,
      "logps/chosen": -518.3923950195312,
      "logps/rejected": -844.9713745117188,
      "loss": 0.3628,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.020922899246216,
      "rewards/margins": 2.008241653442383,
      "rewards/rejected": -4.0291643142700195,
      "step": 3290
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.7800654977942486e-07,
      "logits/chosen": -1.1551368236541748,
      "logits/rejected": 0.04624384641647339,
      "logps/chosen": -494.8173828125,
      "logps/rejected": -759.0722045898438,
      "loss": 0.3921,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.8496835231781006,
      "rewards/margins": 1.606042504310608,
      "rewards/rejected": -3.455725908279419,
      "step": 3300
    },
    {
      "epoch": 0.86,
      "eval_logits/chosen": 0.5531209707260132,
      "eval_logits/rejected": 1.5286740064620972,
      "eval_logps/chosen": -559.9616088867188,
      "eval_logps/rejected": -791.3992309570312,
      "eval_loss": 0.4215858280658722,
      "eval_rewards/accuracies": 0.8050000071525574,
      "eval_rewards/chosen": -2.043008327484131,
      "eval_rewards/margins": 1.680959939956665,
      "eval_rewards/rejected": -3.723968267440796,
      "eval_runtime": 1375.3072,
      "eval_samples_per_second": 1.454,
      "eval_steps_per_second": 0.364,
      "step": 3300
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.6762976418628797e-07,
      "logits/chosen": -0.6151852011680603,
      "logits/rejected": -0.4355131983757019,
      "logps/chosen": -569.3573608398438,
      "logps/rejected": -845.0255737304688,
      "loss": 0.3994,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.9695743322372437,
      "rewards/margins": 1.774019479751587,
      "rewards/rejected": -3.743593692779541,
      "step": 3310
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5743938086541354e-07,
      "logits/chosen": -1.1306208372116089,
      "logits/rejected": -0.369152694940567,
      "logps/chosen": -528.9949951171875,
      "logps/rejected": -786.5325927734375,
      "loss": 0.4035,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8521554470062256,
      "rewards/margins": 2.019233226776123,
      "rewards/rejected": -3.8713886737823486,
      "step": 3320
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.4743625071087574e-07,
      "logits/chosen": -0.9898750185966492,
      "logits/rejected": -0.0026702166069298983,
      "logps/chosen": -589.635498046875,
      "logps/rejected": -785.4796752929688,
      "loss": 0.481,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.26185941696167,
      "rewards/margins": 1.3372209072113037,
      "rewards/rejected": -3.5990803241729736,
      "step": 3330
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3762120898116498e-07,
      "logits/chosen": -1.0256075859069824,
      "logits/rejected": -0.4684371054172516,
      "logps/chosen": -513.4962158203125,
      "logps/rejected": -773.9256591796875,
      "loss": 0.489,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8556444644927979,
      "rewards/margins": 1.662695288658142,
      "rewards/rejected": -3.5183398723602295,
      "step": 3340
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2799507522944048e-07,
      "logits/chosen": -1.2172832489013672,
      "logits/rejected": -0.7886725068092346,
      "logps/chosen": -603.7682495117188,
      "logps/rejected": -839.2596435546875,
      "loss": 0.3825,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.133934259414673,
      "rewards/margins": 1.56403386592865,
      "rewards/rejected": -3.6979682445526123,
      "step": 3350
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1855865323510056e-07,
      "logits/chosen": -0.9375013113021851,
      "logits/rejected": -0.4790850281715393,
      "logps/chosen": -529.7852783203125,
      "logps/rejected": -789.4864501953125,
      "loss": 0.3798,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.7933248281478882,
      "rewards/margins": 1.7892353534698486,
      "rewards/rejected": -3.5825603008270264,
      "step": 3360
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0931273093666575e-07,
      "logits/chosen": -1.0546633005142212,
      "logits/rejected": 0.11733438819646835,
      "logps/chosen": -566.7720947265625,
      "logps/rejected": -769.4909057617188,
      "loss": 0.4561,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.1270699501037598,
      "rewards/margins": 1.5025631189346313,
      "rewards/rejected": -3.6296334266662598,
      "step": 3370
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.002580803659873e-07,
      "logits/chosen": -1.0718940496444702,
      "logits/rejected": -0.24651813507080078,
      "logps/chosen": -563.5213012695312,
      "logps/rejected": -746.7576904296875,
      "loss": 0.4189,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.0106587409973145,
      "rewards/margins": 1.5113112926483154,
      "rewards/rejected": -3.52197003364563,
      "step": 3380
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.913954575837826e-07,
      "logits/chosen": -1.211354374885559,
      "logits/rejected": -0.7178353071212769,
      "logps/chosen": -547.396728515625,
      "logps/rejected": -785.0447998046875,
      "loss": 0.3956,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.859985113143921,
      "rewards/margins": 1.6019798517227173,
      "rewards/rejected": -3.4619648456573486,
      "step": 3390
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.827256026165028e-07,
      "logits/chosen": -1.3360965251922607,
      "logits/rejected": -0.5832281112670898,
      "logps/chosen": -572.5958251953125,
      "logps/rejected": -827.7864379882812,
      "loss": 0.4079,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.8811748027801514,
      "rewards/margins": 1.826063871383667,
      "rewards/rejected": -3.7072386741638184,
      "step": 3400
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7424923939454274e-07,
      "logits/chosen": -1.0015101432800293,
      "logits/rejected": -0.4501993656158447,
      "logps/chosen": -516.7034912109375,
      "logps/rejected": -737.7506713867188,
      "loss": 0.3917,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.871683120727539,
      "rewards/margins": 1.2368314266204834,
      "rewards/rejected": -3.1085145473480225,
      "step": 3410
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6596707569179304e-07,
      "logits/chosen": -1.220655918121338,
      "logits/rejected": -0.36421042680740356,
      "logps/chosen": -664.6968994140625,
      "logps/rejected": -811.2049560546875,
      "loss": 0.5067,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.4624738693237305,
      "rewards/margins": 1.2412056922912598,
      "rewards/rejected": -3.7036795616149902,
      "step": 3420
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.578798030665385e-07,
      "logits/chosen": -0.7742003202438354,
      "logits/rejected": -0.1663471907377243,
      "logps/chosen": -584.2454833984375,
      "logps/rejected": -789.2788696289062,
      "loss": 0.4325,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.159472703933716,
      "rewards/margins": 1.4603602886199951,
      "rewards/rejected": -3.619832992553711,
      "step": 3430
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.499880968037165e-07,
      "logits/chosen": -0.9450448155403137,
      "logits/rejected": -0.24974000453948975,
      "logps/chosen": -543.8734130859375,
      "logps/rejected": -776.0538330078125,
      "loss": 0.3894,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.0840260982513428,
      "rewards/margins": 1.4753036499023438,
      "rewards/rejected": -3.5593299865722656,
      "step": 3440
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4229261585852805e-07,
      "logits/chosen": -1.1183446645736694,
      "logits/rejected": -0.8927680253982544,
      "logps/chosen": -516.3631591796875,
      "logps/rejected": -750.56884765625,
      "loss": 0.4428,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.7409288883209229,
      "rewards/margins": 1.517566442489624,
      "rewards/rejected": -3.258495330810547,
      "step": 3450
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.3479400280141886e-07,
      "logits/chosen": -1.0318033695220947,
      "logits/rejected": -0.8086441159248352,
      "logps/chosen": -516.8532104492188,
      "logps/rejected": -741.1300659179688,
      "loss": 0.3884,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.8979904651641846,
      "rewards/margins": 1.477073311805725,
      "rewards/rejected": -3.37506365776062,
      "step": 3460
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2749288376442044e-07,
      "logits/chosen": -0.9179407358169556,
      "logits/rejected": -0.33625102043151855,
      "logps/chosen": -536.0686645507812,
      "logps/rejected": -847.4317626953125,
      "loss": 0.3721,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.9626020193099976,
      "rewards/margins": 1.956992745399475,
      "rewards/rejected": -3.9195950031280518,
      "step": 3470
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.203898683888713e-07,
      "logits/chosen": -1.2370408773422241,
      "logits/rejected": -0.602063775062561,
      "logps/chosen": -547.3104858398438,
      "logps/rejected": -757.8153076171875,
      "loss": 0.4135,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.9618017673492432,
      "rewards/margins": 1.5197532176971436,
      "rewards/rejected": -3.4815547466278076,
      "step": 3480
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1348554977451132e-07,
      "logits/chosen": -0.8971865773200989,
      "logits/rejected": -0.45237869024276733,
      "logps/chosen": -549.7821044921875,
      "logps/rejected": -819.00439453125,
      "loss": 0.422,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.9521732330322266,
      "rewards/margins": 1.8261715173721313,
      "rewards/rejected": -3.7783446311950684,
      "step": 3490
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0678050442995802e-07,
      "logits/chosen": -0.5732991099357605,
      "logits/rejected": -0.1439836025238037,
      "logps/chosen": -517.4790649414062,
      "logps/rejected": -790.7830810546875,
      "loss": 0.3695,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.7863785028457642,
      "rewards/margins": 1.7641347646713257,
      "rewards/rejected": -3.550513505935669,
      "step": 3500
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0027529222456755e-07,
      "logits/chosen": -0.8079764246940613,
      "logits/rejected": -0.43154460191726685,
      "logps/chosen": -505.2015075683594,
      "logps/rejected": -735.8001708984375,
      "loss": 0.4543,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.861318588256836,
      "rewards/margins": 1.397134780883789,
      "rewards/rejected": -3.258453369140625,
      "step": 3510
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.397045634168766e-08,
      "logits/chosen": -1.5205557346343994,
      "logits/rejected": -0.21061238646507263,
      "logps/chosen": -541.4683837890625,
      "logps/rejected": -778.14453125,
      "loss": 0.3821,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8572975397109985,
      "rewards/margins": 1.6669906377792358,
      "rewards/rejected": -3.5242881774902344,
      "step": 3520
    },
    {
      "epoch": 0.92,
      "learning_rate": 8.78665232332998e-08,
      "logits/chosen": -1.0609691143035889,
      "logits/rejected": -0.4593663811683655,
      "logps/chosen": -509.010986328125,
      "logps/rejected": -796.3077392578125,
      "loss": 0.363,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.9296966791152954,
      "rewards/margins": 1.6614776849746704,
      "rewards/rejected": -3.591174364089966,
      "step": 3530
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.196400257606208e-08,
      "logits/chosen": -0.8713966608047485,
      "logits/rejected": -0.9145506024360657,
      "logps/chosen": -527.7672729492188,
      "logps/rejected": -777.5648193359375,
      "loss": 0.4802,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.896409273147583,
      "rewards/margins": 1.6549888849258423,
      "rewards/rejected": -3.551398515701294,
      "step": 3540
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.626338722875076e-08,
      "logits/chosen": -1.1116408109664917,
      "logits/rejected": -0.13830144703388214,
      "logps/chosen": -569.3795166015625,
      "logps/rejected": -845.25830078125,
      "loss": 0.3774,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.9394264221191406,
      "rewards/margins": 2.049816131591797,
      "rewards/rejected": -3.9892425537109375,
      "step": 3550
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.076515319110688e-08,
      "logits/chosen": -0.9675251841545105,
      "logits/rejected": -0.38351327180862427,
      "logps/chosen": -519.7999267578125,
      "logps/rejected": -768.5065307617188,
      "loss": 0.3933,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.9502222537994385,
      "rewards/margins": 1.693610429763794,
      "rewards/rejected": -3.6438324451446533,
      "step": 3560
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.54697595640899e-08,
      "logits/chosen": -0.8377977609634399,
      "logits/rejected": -0.5262192487716675,
      "logps/chosen": -530.46728515625,
      "logps/rejected": -769.6790771484375,
      "loss": 0.4231,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.032832622528076,
      "rewards/margins": 1.6006113290786743,
      "rewards/rejected": -3.633444309234619,
      "step": 3570
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.037764851154426e-08,
      "logits/chosen": -0.985633373260498,
      "logits/rejected": 0.16631217300891876,
      "logps/chosen": -595.2657470703125,
      "logps/rejected": -853.1846923828125,
      "loss": 0.3474,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.9297056198120117,
      "rewards/margins": 1.8593957424163818,
      "rewards/rejected": -3.7891018390655518,
      "step": 3580
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.548924522327748e-08,
      "logits/chosen": -1.016361951828003,
      "logits/rejected": -0.7056697607040405,
      "logps/chosen": -516.3272705078125,
      "logps/rejected": -732.7681274414062,
      "loss": 0.4643,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8100614547729492,
      "rewards/margins": 1.4326629638671875,
      "rewards/rejected": -3.242724657058716,
      "step": 3590
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.0804957879556915e-08,
      "logits/chosen": -1.4801714420318604,
      "logits/rejected": -0.21010088920593262,
      "logps/chosen": -538.2376708984375,
      "logps/rejected": -716.570556640625,
      "loss": 0.4249,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.8412452936172485,
      "rewards/margins": 1.4520995616912842,
      "rewards/rejected": -3.2933449745178223,
      "step": 3600
    },
    {
      "epoch": 0.94,
      "eval_logits/chosen": 0.3916805386543274,
      "eval_logits/rejected": 1.3532880544662476,
      "eval_logps/chosen": -551.5703735351562,
      "eval_logps/rejected": -777.8283081054688,
      "eval_loss": 0.4203905165195465,
      "eval_rewards/accuracies": 0.800000011920929,
      "eval_rewards/chosen": -1.9590957164764404,
      "eval_rewards/margins": 1.6291632652282715,
      "eval_rewards/rejected": -3.588258981704712,
      "eval_runtime": 1380.3242,
      "eval_samples_per_second": 1.449,
      "eval_steps_per_second": 0.362,
      "step": 3600
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.632517761702815e-08,
      "logits/chosen": -1.3528892993927002,
      "logits/rejected": 0.41183796525001526,
      "logps/chosen": -596.6740112304688,
      "logps/rejected": -815.2229614257812,
      "loss": 0.3384,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.033761739730835,
      "rewards/margins": 1.7981764078140259,
      "rewards/rejected": -3.8319382667541504,
      "step": 3610
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.205027849605359e-08,
      "logits/chosen": -0.6885486245155334,
      "logits/rejected": -0.6011554002761841,
      "logps/chosen": -501.90008544921875,
      "logps/rejected": -728.01416015625,
      "loss": 0.3797,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8948495388031006,
      "rewards/margins": 1.430154800415039,
      "rewards/rejected": -3.3250045776367188,
      "step": 3620
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.798061746947995e-08,
      "logits/chosen": -1.1346218585968018,
      "logits/rejected": -0.2788551449775696,
      "logps/chosen": -572.7951049804688,
      "logps/rejected": -818.1143798828125,
      "loss": 0.3816,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.859471321105957,
      "rewards/margins": 1.9316068887710571,
      "rewards/rejected": -3.7910780906677246,
      "step": 3630
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.411653435283158e-08,
      "logits/chosen": -1.2903286218643188,
      "logits/rejected": -0.4007846713066101,
      "logps/chosen": -612.4331665039062,
      "logps/rejected": -790.9478759765625,
      "loss": 0.4783,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.9661962985992432,
      "rewards/margins": 1.4086124897003174,
      "rewards/rejected": -3.3748087882995605,
      "step": 3640
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.04583517959367e-08,
      "logits/chosen": -1.4373764991760254,
      "logits/rejected": -0.18271857500076294,
      "logps/chosen": -567.571533203125,
      "logps/rejected": -814.9035034179688,
      "loss": 0.372,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.9404630661010742,
      "rewards/margins": 1.8180534839630127,
      "rewards/rejected": -3.758516788482666,
      "step": 3650
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.7006375255985984e-08,
      "logits/chosen": -0.8399646878242493,
      "logits/rejected": -0.41886812448501587,
      "logps/chosen": -487.2755432128906,
      "logps/rejected": -792.0075073242188,
      "loss": 0.3617,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.7799431085586548,
      "rewards/margins": 2.1151885986328125,
      "rewards/rejected": -3.895132064819336,
      "step": 3660
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3760892972027328e-08,
      "logits/chosen": -0.7552706003189087,
      "logits/rejected": -0.37330105900764465,
      "logps/chosen": -550.1450805664062,
      "logps/rejected": -821.5558471679688,
      "loss": 0.4513,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.0513758659362793,
      "rewards/margins": 1.6616461277008057,
      "rewards/rejected": -3.713021755218506,
      "step": 3670
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.072217594089765e-08,
      "logits/chosen": -1.048200249671936,
      "logits/rejected": -0.7420127987861633,
      "logps/chosen": -548.27587890625,
      "logps/rejected": -818.7357788085938,
      "loss": 0.4605,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.8455013036727905,
      "rewards/margins": 1.6794688701629639,
      "rewards/rejected": -3.524970293045044,
      "step": 3680
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.789047789459375e-08,
      "logits/chosen": -0.8459684252738953,
      "logits/rejected": -0.16287431120872498,
      "logps/chosen": -508.4056091308594,
      "logps/rejected": -766.1015625,
      "loss": 0.3671,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6832281351089478,
      "rewards/margins": 1.7482646703720093,
      "rewards/rejected": -3.431492567062378,
      "step": 3690
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.5266035279088708e-08,
      "logits/chosen": -0.6956599354743958,
      "logits/rejected": -0.6482317447662354,
      "logps/chosen": -574.1497802734375,
      "logps/rejected": -854.5062255859375,
      "loss": 0.3785,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.2222962379455566,
      "rewards/margins": 1.6956876516342163,
      "rewards/rejected": -3.9179844856262207,
      "step": 3700
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.2849067234584623e-08,
      "logits/chosen": -0.6731444001197815,
      "logits/rejected": -0.2936319410800934,
      "logps/chosen": -508.5874938964844,
      "logps/rejected": -779.5755004882812,
      "loss": 0.3793,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.7908990383148193,
      "rewards/margins": 1.8119395971298218,
      "rewards/rejected": -3.6028385162353516,
      "step": 3710
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.0639775577218625e-08,
      "logits/chosen": -1.1039248704910278,
      "logits/rejected": -0.3895350396633148,
      "logps/chosen": -532.2550048828125,
      "logps/rejected": -799.4228515625,
      "loss": 0.3993,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.914385437965393,
      "rewards/margins": 1.754346489906311,
      "rewards/rejected": -3.668731689453125,
      "step": 3720
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.638344782207486e-09,
      "logits/chosen": -1.1005773544311523,
      "logits/rejected": -0.48734521865844727,
      "logps/chosen": -638.8570556640625,
      "logps/rejected": -876.8406372070312,
      "loss": 0.441,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.3087477684020996,
      "rewards/margins": 1.5587693452835083,
      "rewards/rejected": -3.8675169944763184,
      "step": 3730
    },
    {
      "epoch": 0.98,
      "learning_rate": 6.84494196844715e-09,
      "logits/chosen": -1.2625192403793335,
      "logits/rejected": -0.4201609194278717,
      "logps/chosen": -579.9595336914062,
      "logps/rejected": -743.4432373046875,
      "loss": 0.4261,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.0454494953155518,
      "rewards/margins": 1.329736351966858,
      "rewards/rejected": -3.37518572807312,
      "step": 3740
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.259716884556121e-09,
      "logits/chosen": -1.0971883535385132,
      "logits/rejected": 0.13197267055511475,
      "logps/chosen": -497.7351989746094,
      "logps/rejected": -844.2490234375,
      "loss": 0.3651,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.8625205755233765,
      "rewards/margins": 2.320369243621826,
      "rewards/rejected": -4.182889461517334,
      "step": 3750
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.882801896372967e-09,
      "logits/chosen": -1.2590482234954834,
      "logits/rejected": 0.06436818838119507,
      "logps/chosen": -604.3903198242188,
      "logps/rejected": -838.1032104492188,
      "loss": 0.3897,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.1124839782714844,
      "rewards/margins": 1.6700093746185303,
      "rewards/rejected": -3.7824935913085938,
      "step": 3760
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.7143119759026614e-09,
      "logits/chosen": -0.8534577488899231,
      "logits/rejected": -0.31645748019218445,
      "logps/chosen": -536.9246215820312,
      "logps/rejected": -888.1343994140625,
      "loss": 0.2884,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.8014726638793945,
      "rewards/margins": 2.4648687839508057,
      "rewards/rejected": -4.266341209411621,
      "step": 3770
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.754344691717591e-09,
      "logits/chosen": -1.0792902708053589,
      "logits/rejected": -0.5682160258293152,
      "logps/chosen": -508.158447265625,
      "logps/rejected": -773.3580932617188,
      "loss": 0.3699,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.8245254755020142,
      "rewards/margins": 1.8749605417251587,
      "rewards/rejected": -3.6994857788085938,
      "step": 3780
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.0029802008096335e-09,
      "logits/chosen": -0.6716892719268799,
      "logits/rejected": -0.40372976660728455,
      "logps/chosen": -489.3829040527344,
      "logps/rejected": -723.220703125,
      "loss": 0.4356,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.7527339458465576,
      "rewards/margins": 1.6192394495010376,
      "rewards/rejected": -3.371973752975464,
      "step": 3790
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.602812418974534e-10,
      "logits/chosen": -0.8607047200202942,
      "logits/rejected": -0.9630060195922852,
      "logps/chosen": -502.1089782714844,
      "logps/rejected": -731.4326171875,
      "loss": 0.4389,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.8323230743408203,
      "rewards/margins": 1.3677552938461304,
      "rewards/rejected": -3.200078248977661,
      "step": 3800
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.2629313018819312e-10,
      "logits/chosen": -0.9664648175239563,
      "logits/rejected": 0.03846656158566475,
      "logps/chosen": -550.194091796875,
      "logps/rejected": -774.918212890625,
      "loss": 0.385,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.9849185943603516,
      "rewards/margins": 1.5859264135360718,
      "rewards/rejected": -3.5708446502685547,
      "step": 3810
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.0437535929996855e-12,
      "logits/chosen": -1.1103519201278687,
      "logits/rejected": -0.32411596179008484,
      "logps/chosen": -598.4796752929688,
      "logps/rejected": -901.5104370117188,
      "loss": 0.4032,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.1410813331604004,
      "rewards/margins": 1.9627602100372314,
      "rewards/rejected": -4.103841304779053,
      "step": 3820
    },
    {
      "epoch": 1.0,
      "step": 3821,
      "total_flos": 0.0,
      "train_loss": 0.43769783746998087,
      "train_runtime": 91189.3347,
      "train_samples_per_second": 0.67,
      "train_steps_per_second": 0.042
    }
  ],
  "logging_steps": 10,
  "max_steps": 3821,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 400,
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}