diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -3,18 +3,18 @@
   "best_model_checkpoint": null,
   "epoch": 2.9992254066615027,
   "eval_steps": 100,
-  "global_step": 2904,
+  "global_step": 726,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "learning_rate": 1.7182130584192438e-09,
-      "logits/chosen": -2.293962240219116,
-      "logits/rejected": -2.235898017883301,
-      "logps/chosen": -280.7555847167969,
-      "logps/rejected": -204.8462371826172,
+      "learning_rate": 6.84931506849315e-09,
+      "logits/chosen": -2.348097085952759,
+      "logits/rejected": -2.4175631999969482,
+      "logps/chosen": -271.4220275878906,
+      "logps/rejected": -208.93710327148438,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -23,4125 +23,1073 @@
       "step": 1
     },
     {
-      "epoch": 0.01,
-      "learning_rate": 1.718213058419244e-08,
-      "logits/chosen": -2.4112908840179443,
-      "logits/rejected": -2.3392038345336914,
-      "logps/chosen": -294.24664306640625,
-      "logps/rejected": -213.90679931640625,
-      "loss": 0.6947,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.002664534840732813,
-      "rewards/margins": -9.001558646559715e-05,
-      "rewards/rejected": 0.00275455042719841,
+      "epoch": 0.04,
+      "learning_rate": 6.84931506849315e-08,
+      "logits/chosen": -2.4228014945983887,
+      "logits/rejected": -2.356138229370117,
+      "logps/chosen": -293.40594482421875,
+      "logps/rejected": -226.2930450439453,
+      "loss": 0.6938,
+      "rewards/accuracies": 0.4652777910232544,
+      "rewards/chosen": 0.00204316433519125,
+      "rewards/margins": 0.002578532788902521,
+      "rewards/rejected": -0.0005353688611648977,
       "step": 10
     },
     {
-      "epoch": 0.02,
-      "learning_rate": 3.436426116838488e-08,
-      "logits/chosen": -2.4150779247283936,
-      "logits/rejected": -2.3804850578308105,
-      "logps/chosen": -279.4464416503906,
-      "logps/rejected": -237.6862335205078,
-      "loss": 0.6923,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": 0.003086227923631668,
-      "rewards/margins": 0.009707429446280003,
-      "rewards/rejected": -0.006621202919632196,
+      "epoch": 0.08,
+      "learning_rate": 1.36986301369863e-07,
+      "logits/chosen": -2.431565046310425,
+      "logits/rejected": -2.40484881401062,
+      "logps/chosen": -278.5029296875,
+      "logps/rejected": -216.76760864257812,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.534375011920929,
+      "rewards/chosen": 0.003985759802162647,
+      "rewards/margins": 0.004827108699828386,
+      "rewards/rejected": -0.0008413494797423482,
       "step": 20
     },
     {
-      "epoch": 0.03,
-      "learning_rate": 5.154639175257731e-08,
-      "logits/chosen": -2.4616353511810303,
-      "logits/rejected": -2.3937487602233887,
-      "logps/chosen": -301.0641174316406,
-      "logps/rejected": -215.75418090820312,
-      "loss": 0.6935,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.0004951163427904248,
-      "rewards/margins": 0.000455385452369228,
-      "rewards/rejected": 3.9730781281832606e-05,
+      "epoch": 0.12,
+      "learning_rate": 2.054794520547945e-07,
+      "logits/chosen": -2.38895583152771,
+      "logits/rejected": -2.348848819732666,
+      "logps/chosen": -253.019775390625,
+      "logps/rejected": -207.16421508789062,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4828124940395355,
+      "rewards/chosen": -0.0008784265955910087,
+      "rewards/margins": 0.0001159904059022665,
+      "rewards/rejected": -0.0009944172343239188,
       "step": 30
     },
     {
-      "epoch": 0.04,
-      "learning_rate": 6.872852233676976e-08,
-      "logits/chosen": -2.3844287395477295,
-      "logits/rejected": -2.344656467437744,
-      "logps/chosen": -291.42413330078125,
-      "logps/rejected": -231.827880859375,
-      "loss": 0.6935,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": 0.0011853076284751296,
-      "rewards/margins": 0.002922216197475791,
-      "rewards/rejected": -0.0017369084525853395,
+      "epoch": 0.17,
+      "learning_rate": 2.73972602739726e-07,
+      "logits/chosen": -2.4601290225982666,
+      "logits/rejected": -2.4147398471832275,
+      "logps/chosen": -283.7552795410156,
+      "logps/rejected": -216.47750854492188,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.520312488079071,
+      "rewards/chosen": 0.004305425100028515,
+      "rewards/margins": 0.006620796862989664,
+      "rewards/rejected": -0.0023153722286224365,
       "step": 40
     },
     {
-      "epoch": 0.05,
-      "learning_rate": 8.59106529209622e-08,
-      "logits/chosen": -2.4430339336395264,
-      "logits/rejected": -2.382826566696167,
-      "logps/chosen": -299.23663330078125,
-      "logps/rejected": -220.186767578125,
-      "loss": 0.6932,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": 0.0030969330109655857,
-      "rewards/margins": 0.004286954645067453,
-      "rewards/rejected": -0.0011900208191946149,
+      "epoch": 0.21,
+      "learning_rate": 3.424657534246575e-07,
+      "logits/chosen": -2.4362146854400635,
+      "logits/rejected": -2.3963656425476074,
+      "logps/chosen": -267.26031494140625,
+      "logps/rejected": -223.655029296875,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.005432569421827793,
+      "rewards/margins": 0.007884417660534382,
+      "rewards/rejected": -0.002451848704367876,
       "step": 50
     },
     {
-      "epoch": 0.06,
-      "learning_rate": 1.0309278350515462e-07,
-      "logits/chosen": -2.4357097148895264,
-      "logits/rejected": -2.4250669479370117,
-      "logps/chosen": -272.5534973144531,
-      "logps/rejected": -227.50332641601562,
-      "loss": 0.6936,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.00050017872126773,
-      "rewards/margins": 0.0015349002787843347,
-      "rewards/rejected": -0.0020350790582597256,
+      "epoch": 0.25,
+      "learning_rate": 4.10958904109589e-07,
+      "logits/chosen": -2.412108898162842,
+      "logits/rejected": -2.3888187408447266,
+      "logps/chosen": -266.83953857421875,
+      "logps/rejected": -214.74533081054688,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.008083345368504524,
+      "rewards/margins": 0.013038607314229012,
+      "rewards/rejected": -0.0049552614800632,
       "step": 60
     },
     {
-      "epoch": 0.07,
-      "learning_rate": 1.202749140893471e-07,
-      "logits/chosen": -2.465843677520752,
-      "logits/rejected": -2.4208121299743652,
-      "logps/chosen": -292.08990478515625,
-      "logps/rejected": -206.94558715820312,
-      "loss": 0.6919,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0012726224958896637,
-      "rewards/margins": 0.003322723787277937,
-      "rewards/rejected": -0.0020501010585576296,
+      "epoch": 0.29,
+      "learning_rate": 4.794520547945205e-07,
+      "logits/chosen": -2.390169858932495,
+      "logits/rejected": -2.3971309661865234,
+      "logps/chosen": -254.08779907226562,
+      "logps/rejected": -214.64498901367188,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.007395301945507526,
+      "rewards/margins": 0.017588596791028976,
+      "rewards/rejected": -0.010193293914198875,
       "step": 70
     },
     {
-      "epoch": 0.08,
-      "learning_rate": 1.3745704467353952e-07,
-      "logits/chosen": -2.3812198638916016,
-      "logits/rejected": -2.390000104904175,
-      "logps/chosen": -250.2107391357422,
-      "logps/rejected": -212.4706268310547,
-      "loss": 0.6901,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.004158427938818932,
-      "rewards/margins": 0.005836354102939367,
-      "rewards/rejected": -0.0016779262805357575,
+      "epoch": 0.33,
+      "learning_rate": 4.946401225114854e-07,
+      "logits/chosen": -2.4293293952941895,
+      "logits/rejected": -2.3780810832977295,
+      "logps/chosen": -265.03875732421875,
+      "logps/rejected": -218.77554321289062,
+      "loss": 0.679,
+      "rewards/accuracies": 0.6421874761581421,
+      "rewards/chosen": 0.016448482871055603,
+      "rewards/margins": 0.03233351930975914,
+      "rewards/rejected": -0.015885034576058388,
       "step": 80
     },
     {
-      "epoch": 0.09,
-      "learning_rate": 1.5463917525773197e-07,
-      "logits/chosen": -2.3327183723449707,
-      "logits/rejected": -2.254629611968994,
-      "logps/chosen": -241.6092529296875,
-      "logps/rejected": -185.40869140625,
-      "loss": 0.6951,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.005251345224678516,
-      "rewards/margins": -0.004248884506523609,
-      "rewards/rejected": -0.0010024613002315164,
+      "epoch": 0.37,
+      "learning_rate": 4.869831546707504e-07,
+      "logits/chosen": -2.478893756866455,
+      "logits/rejected": -2.422461986541748,
+      "logps/chosen": -271.6805114746094,
+      "logps/rejected": -224.47640991210938,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.026180211454629898,
+      "rewards/margins": 0.05046360567212105,
+      "rewards/rejected": -0.0242833960801363,
       "step": 90
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 1.718213058419244e-07,
-      "logits/chosen": -2.401676654815674,
-      "logits/rejected": -2.3915371894836426,
-      "logps/chosen": -259.74755859375,
-      "logps/rejected": -221.87295532226562,
-      "loss": 0.6923,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.005552566610276699,
-      "rewards/margins": -0.0006124695646576583,
-      "rewards/rejected": 0.006165036000311375,
+      "epoch": 0.41,
+      "learning_rate": 4.793261868300153e-07,
+      "logits/chosen": -2.417175054550171,
+      "logits/rejected": -2.4038565158843994,
+      "logps/chosen": -273.87823486328125,
+      "logps/rejected": -227.7811279296875,
+      "loss": 0.6687,
+      "rewards/accuracies": 0.6734374761581421,
+      "rewards/chosen": 0.026967059820890427,
+      "rewards/margins": 0.05768311023712158,
+      "rewards/rejected": -0.030716046690940857,
       "step": 100
     },
     {
-      "epoch": 0.11,
-      "learning_rate": 1.8900343642611682e-07,
-      "logits/chosen": -2.441145181655884,
-      "logits/rejected": -2.311302423477173,
-      "logps/chosen": -252.06753540039062,
-      "logps/rejected": -210.41696166992188,
-      "loss": 0.6889,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.0075301057659089565,
-      "rewards/margins": 0.013402941636741161,
-      "rewards/rejected": -0.0058728354051709175,
+      "epoch": 0.45,
+      "learning_rate": 4.7166921898928023e-07,
+      "logits/chosen": -2.450390338897705,
+      "logits/rejected": -2.39005708694458,
+      "logps/chosen": -252.6284942626953,
+      "logps/rejected": -222.63601684570312,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": 0.022277798503637314,
+      "rewards/margins": 0.060135841369628906,
+      "rewards/rejected": -0.03785804286599159,
       "step": 110
     },
     {
-      "epoch": 0.12,
-      "learning_rate": 2.0618556701030925e-07,
-      "logits/chosen": -2.3808252811431885,
-      "logits/rejected": -2.4377057552337646,
-      "logps/chosen": -258.4937438964844,
-      "logps/rejected": -210.9638671875,
-      "loss": 0.6905,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.0047661615535616875,
-      "rewards/margins": 0.008594167418777943,
-      "rewards/rejected": -0.003828004002571106,
+      "epoch": 0.5,
+      "learning_rate": 4.640122511485451e-07,
+      "logits/chosen": -2.412520170211792,
+      "logits/rejected": -2.401998281478882,
+      "logps/chosen": -256.9021301269531,
+      "logps/rejected": -219.8153076171875,
+      "loss": 0.6579,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.027152011170983315,
+      "rewards/margins": 0.07799454033374786,
+      "rewards/rejected": -0.0508425310254097,
       "step": 120
     },
     {
-      "epoch": 0.13,
-      "learning_rate": 2.2336769759450173e-07,
-      "logits/chosen": -2.447084903717041,
-      "logits/rejected": -2.4316954612731934,
-      "logps/chosen": -279.2828674316406,
-      "logps/rejected": -213.9290313720703,
-      "loss": 0.6873,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.015633000060915947,
-      "rewards/margins": 0.019236568361520767,
-      "rewards/rejected": -0.0036035701632499695,
+      "epoch": 0.54,
+      "learning_rate": 4.563552833078101e-07,
+      "logits/chosen": -2.456604480743408,
+      "logits/rejected": -2.3952419757843018,
+      "logps/chosen": -265.9836730957031,
+      "logps/rejected": -225.9434356689453,
+      "loss": 0.6504,
+      "rewards/accuracies": 0.667187511920929,
+      "rewards/chosen": 0.02718115784227848,
+      "rewards/margins": 0.09283201396465302,
+      "rewards/rejected": -0.06565085798501968,
       "step": 130
     },
     {
-      "epoch": 0.14,
-      "learning_rate": 2.405498281786942e-07,
-      "logits/chosen": -2.574856996536255,
-      "logits/rejected": -2.4002137184143066,
-      "logps/chosen": -267.64520263671875,
-      "logps/rejected": -203.2295684814453,
-      "loss": 0.6892,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.008941258303821087,
-      "rewards/margins": 0.014786717481911182,
-      "rewards/rejected": -0.00584546010941267,
+      "epoch": 0.58,
+      "learning_rate": 4.4869831546707505e-07,
+      "logits/chosen": -2.4601752758026123,
+      "logits/rejected": -2.4006924629211426,
+      "logps/chosen": -270.29620361328125,
+      "logps/rejected": -230.3610076904297,
+      "loss": 0.65,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": 0.03528743237257004,
+      "rewards/margins": 0.1161808967590332,
+      "rewards/rejected": -0.08089347183704376,
       "step": 140
     },
     {
-      "epoch": 0.15,
-      "learning_rate": 2.5773195876288655e-07,
-      "logits/chosen": -2.3573081493377686,
-      "logits/rejected": -2.390941619873047,
-      "logps/chosen": -280.3595886230469,
-      "logps/rejected": -214.0395965576172,
-      "loss": 0.6876,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": 0.0026760019827634096,
-      "rewards/margins": 0.010370850563049316,
-      "rewards/rejected": -0.0076948488131165504,
+      "epoch": 0.62,
+      "learning_rate": 4.4104134762633994e-07,
+      "logits/chosen": -2.4845683574676514,
+      "logits/rejected": -2.4314005374908447,
+      "logps/chosen": -265.96466064453125,
+      "logps/rejected": -226.5783233642578,
+      "loss": 0.643,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.03652546927332878,
+      "rewards/margins": 0.12623493373394012,
+      "rewards/rejected": -0.08970947563648224,
       "step": 150
     },
     {
-      "epoch": 0.17,
-      "learning_rate": 2.7491408934707903e-07,
-      "logits/chosen": -2.460977077484131,
-      "logits/rejected": -2.4357001781463623,
-      "logps/chosen": -307.5310974121094,
-      "logps/rejected": -234.86721801757812,
-      "loss": 0.6857,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.010203736834228039,
-      "rewards/margins": 0.017851073294878006,
-      "rewards/rejected": -0.0076473369263112545,
+      "epoch": 0.66,
+      "learning_rate": 4.333843797856049e-07,
+      "logits/chosen": -2.4465065002441406,
+      "logits/rejected": -2.402820348739624,
+      "logps/chosen": -279.31195068359375,
+      "logps/rejected": -238.9833221435547,
+      "loss": 0.6353,
+      "rewards/accuracies": 0.6781250238418579,
+      "rewards/chosen": 0.042833976447582245,
+      "rewards/margins": 0.14586737751960754,
+      "rewards/rejected": -0.10303342342376709,
       "step": 160
     },
     {
-      "epoch": 0.18,
-      "learning_rate": 2.9209621993127146e-07,
-      "logits/chosen": -2.4167332649230957,
-      "logits/rejected": -2.4196879863739014,
-      "logps/chosen": -284.23834228515625,
-      "logps/rejected": -220.6202392578125,
-      "loss": 0.6861,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": 0.006566309370100498,
-      "rewards/margins": 0.00825690571218729,
-      "rewards/rejected": -0.001690595643594861,
+      "epoch": 0.7,
+      "learning_rate": 4.257274119448698e-07,
+      "logits/chosen": -2.479480028152466,
+      "logits/rejected": -2.4522595405578613,
+      "logps/chosen": -270.7332458496094,
+      "logps/rejected": -239.1272735595703,
+      "loss": 0.6338,
+      "rewards/accuracies": 0.667187511920929,
+      "rewards/chosen": 0.022215088829398155,
+      "rewards/margins": 0.1372935026884079,
+      "rewards/rejected": -0.11507842689752579,
       "step": 170
     },
     {
-      "epoch": 0.19,
-      "learning_rate": 3.0927835051546394e-07,
-      "logits/chosen": -2.426633358001709,
-      "logits/rejected": -2.426823139190674,
-      "logps/chosen": -261.1161193847656,
-      "logps/rejected": -236.690673828125,
-      "loss": 0.682,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.007787441369146109,
-      "rewards/margins": 0.021113665774464607,
-      "rewards/rejected": -0.013326220214366913,
+      "epoch": 0.74,
+      "learning_rate": 4.180704441041347e-07,
+      "logits/chosen": -2.4355900287628174,
+      "logits/rejected": -2.3883790969848633,
+      "logps/chosen": -256.80621337890625,
+      "logps/rejected": -226.12857055664062,
+      "loss": 0.6362,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.01779203675687313,
+      "rewards/margins": 0.14599665999412537,
+      "rewards/rejected": -0.1282046139240265,
       "step": 180
     },
     {
-      "epoch": 0.2,
-      "learning_rate": 3.2646048109965636e-07,
-      "logits/chosen": -2.47978138923645,
-      "logits/rejected": -2.393406391143799,
-      "logps/chosen": -261.36138916015625,
-      "logps/rejected": -213.78268432617188,
-      "loss": 0.6807,
+      "epoch": 0.78,
+      "learning_rate": 4.1041347626339966e-07,
+      "logits/chosen": -2.44791316986084,
+      "logits/rejected": -2.406308174133301,
+      "logps/chosen": -261.647705078125,
+      "logps/rejected": -213.19442749023438,
+      "loss": 0.6272,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.010398685932159424,
-      "rewards/margins": 0.022776171565055847,
-      "rewards/rejected": -0.012377489358186722,
+      "rewards/chosen": 0.019534433260560036,
+      "rewards/margins": 0.17334969341754913,
+      "rewards/rejected": -0.15381526947021484,
       "step": 190
     },
     {
-      "epoch": 0.21,
-      "learning_rate": 3.436426116838488e-07,
-      "logits/chosen": -2.421168565750122,
-      "logits/rejected": -2.3462955951690674,
-      "logps/chosen": -262.1727294921875,
-      "logps/rejected": -223.8699188232422,
-      "loss": 0.6806,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.012244177982211113,
-      "rewards/margins": 0.029001509770751,
-      "rewards/rejected": -0.016757333651185036,
+      "epoch": 0.83,
+      "learning_rate": 4.027565084226646e-07,
+      "logits/chosen": -2.4711079597473145,
+      "logits/rejected": -2.414386510848999,
+      "logps/chosen": -262.3142395019531,
+      "logps/rejected": -217.9895477294922,
+      "loss": 0.6187,
+      "rewards/accuracies": 0.692187488079071,
+      "rewards/chosen": 0.02562810108065605,
+      "rewards/margins": 0.19041840732097626,
+      "rewards/rejected": -0.1647903025150299,
       "step": 200
     },
     {
-      "epoch": 0.22,
-      "learning_rate": 3.608247422680412e-07,
-      "logits/chosen": -2.446596145629883,
-      "logits/rejected": -2.367396593093872,
-      "logps/chosen": -276.6649475097656,
-      "logps/rejected": -203.23178100585938,
-      "loss": 0.6758,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.012773854658007622,
-      "rewards/margins": 0.03576343134045601,
-      "rewards/rejected": -0.022989574819803238,
+      "epoch": 0.87,
+      "learning_rate": 3.9509954058192954e-07,
+      "logits/chosen": -2.474663019180298,
+      "logits/rejected": -2.4352869987487793,
+      "logps/chosen": -283.8729248046875,
+      "logps/rejected": -231.70376586914062,
+      "loss": 0.6162,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": 0.029517430812120438,
+      "rewards/margins": 0.2187691181898117,
+      "rewards/rejected": -0.18925169110298157,
       "step": 210
     },
     {
-      "epoch": 0.23,
-      "learning_rate": 3.7800687285223364e-07,
-      "logits/chosen": -2.4427998065948486,
-      "logits/rejected": -2.400299549102783,
-      "logps/chosen": -248.6843719482422,
-      "logps/rejected": -211.16293334960938,
-      "loss": 0.674,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.01836214028298855,
-      "rewards/margins": 0.0383928082883358,
-      "rewards/rejected": -0.020030666142702103,
+      "epoch": 0.91,
+      "learning_rate": 3.874425727411945e-07,
+      "logits/chosen": -2.4217212200164795,
+      "logits/rejected": -2.4082586765289307,
+      "logps/chosen": -276.77996826171875,
+      "logps/rejected": -229.3105010986328,
+      "loss": 0.6138,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.017411770299077034,
+      "rewards/margins": 0.2216881960630417,
+      "rewards/rejected": -0.204276442527771,
       "step": 220
     },
     {
-      "epoch": 0.24,
-      "learning_rate": 3.9518900343642607e-07,
-      "logits/chosen": -2.4508414268493652,
-      "logits/rejected": -2.444462299346924,
-      "logps/chosen": -261.3500061035156,
-      "logps/rejected": -210.68807983398438,
-      "loss": 0.6755,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.02510516345500946,
-      "rewards/margins": 0.04327309876680374,
-      "rewards/rejected": -0.018167927861213684,
+      "epoch": 0.95,
+      "learning_rate": 3.797856049004594e-07,
+      "logits/chosen": -2.4441373348236084,
+      "logits/rejected": -2.3886184692382812,
+      "logps/chosen": -264.1249084472656,
+      "logps/rejected": -228.3013458251953,
+      "loss": 0.6119,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": 0.02687259018421173,
+      "rewards/margins": 0.21526794135570526,
+      "rewards/rejected": -0.18839535117149353,
       "step": 230
     },
     {
-      "epoch": 0.25,
-      "learning_rate": 4.123711340206185e-07,
-      "logits/chosen": -2.3069140911102295,
-      "logits/rejected": -2.3430962562561035,
-      "logps/chosen": -280.17633056640625,
-      "logps/rejected": -234.605224609375,
-      "loss": 0.6696,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 0.024332646280527115,
-      "rewards/margins": 0.05363120511174202,
-      "rewards/rejected": -0.029298555105924606,
+      "epoch": 0.99,
+      "learning_rate": 3.7212863705972436e-07,
+      "logits/chosen": -2.462822198867798,
+      "logits/rejected": -2.418255567550659,
+      "logps/chosen": -271.4612731933594,
+      "logps/rejected": -222.49185180664062,
+      "loss": 0.6115,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": 0.025034094229340553,
+      "rewards/margins": 0.21872933208942413,
+      "rewards/rejected": -0.19369521737098694,
       "step": 240
     },
     {
-      "epoch": 0.26,
-      "learning_rate": 4.2955326460481097e-07,
-      "logits/chosen": -2.408478021621704,
-      "logits/rejected": -2.38106369972229,
-      "logps/chosen": -267.2620849609375,
-      "logps/rejected": -210.53933715820312,
-      "loss": 0.669,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.022226443514227867,
-      "rewards/margins": 0.06285347044467926,
-      "rewards/rejected": -0.040627025067806244,
+      "epoch": 1.0,
+      "eval_logits/chosen": -2.1432242393493652,
+      "eval_logits/rejected": -2.0230488777160645,
+      "eval_logps/chosen": -264.6322937011719,
+      "eval_logps/rejected": -221.69525146484375,
+      "eval_loss": 0.6071053147315979,
+      "eval_rewards/accuracies": 0.6700000166893005,
+      "eval_rewards/chosen": 0.0028544252272695303,
+      "eval_rewards/margins": 0.24394646286964417,
+      "eval_rewards/rejected": -0.2410920411348343,
+      "eval_runtime": 443.3534,
+      "eval_samples_per_second": 4.511,
+      "eval_steps_per_second": 0.282,
+      "step": 242
+    },
+    {
+      "epoch": 1.03,
+      "learning_rate": 3.6447166921898925e-07,
+      "logits/chosen": -2.3999691009521484,
+      "logits/rejected": -2.347167730331421,
+      "logps/chosen": -257.44598388671875,
+      "logps/rejected": -210.4270782470703,
+      "loss": 0.617,
+      "rewards/accuracies": 0.6703125238418579,
+      "rewards/chosen": 0.0046371398493647575,
+      "rewards/margins": 0.2242714911699295,
+      "rewards/rejected": -0.21963433921337128,
       "step": 250
     },
     {
-      "epoch": 0.27,
-      "learning_rate": 4.4673539518900345e-07,
-      "logits/chosen": -2.3904354572296143,
-      "logits/rejected": -2.4175901412963867,
-      "logps/chosen": -261.8872375488281,
-      "logps/rejected": -215.2822723388672,
-      "loss": 0.6659,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.012217490933835506,
-      "rewards/margins": 0.041740067303180695,
-      "rewards/rejected": -0.029522571712732315,
+      "epoch": 1.07,
+      "learning_rate": 3.568147013782542e-07,
+      "logits/chosen": -2.4152634143829346,
+      "logits/rejected": -2.37550687789917,
+      "logps/chosen": -261.9479675292969,
+      "logps/rejected": -226.1567840576172,
+      "loss": 0.6004,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.021805018186569214,
+      "rewards/margins": 0.26413100957870483,
+      "rewards/rejected": -0.2423260509967804,
       "step": 260
     },
     {
-      "epoch": 0.28,
-      "learning_rate": 4.639175257731959e-07,
-      "logits/chosen": -2.3512496948242188,
-      "logits/rejected": -2.374730110168457,
-      "logps/chosen": -221.1552276611328,
-      "logps/rejected": -214.0994110107422,
-      "loss": 0.6617,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.02718324586749077,
-      "rewards/margins": 0.07020439207553864,
-      "rewards/rejected": -0.04302113875746727,
+      "epoch": 1.12,
+      "learning_rate": 3.4915773353751913e-07,
+      "logits/chosen": -2.42952299118042,
+      "logits/rejected": -2.365931272506714,
+      "logps/chosen": -278.4363708496094,
+      "logps/rejected": -236.10116577148438,
+      "loss": 0.5919,
+      "rewards/accuracies": 0.745312511920929,
+      "rewards/chosen": 0.026421889662742615,
+      "rewards/margins": 0.33800598978996277,
+      "rewards/rejected": -0.31158408522605896,
       "step": 270
     },
     {
-      "epoch": 0.29,
-      "learning_rate": 4.810996563573884e-07,
-      "logits/chosen": -2.4109158515930176,
-      "logits/rejected": -2.416260242462158,
-      "logps/chosen": -265.5227355957031,
-      "logps/rejected": -219.8955841064453,
-      "loss": 0.6591,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.02033821865916252,
-      "rewards/margins": 0.0716070905327797,
-      "rewards/rejected": -0.051268868148326874,
+      "epoch": 1.16,
+      "learning_rate": 3.41500765696784e-07,
+      "logits/chosen": -2.4357521533966064,
+      "logits/rejected": -2.396134614944458,
+      "logps/chosen": -251.1164093017578,
+      "logps/rejected": -225.797119140625,
+      "loss": 0.6028,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": 0.009803700260818005,
+      "rewards/margins": 0.2455863505601883,
+      "rewards/rejected": -0.2357826679944992,
       "step": 280
     },
     {
-      "epoch": 0.3,
-      "learning_rate": 4.982817869415807e-07,
-      "logits/chosen": -2.4649157524108887,
-      "logits/rejected": -2.336876153945923,
-      "logps/chosen": -299.04083251953125,
-      "logps/rejected": -233.513671875,
-      "loss": 0.6561,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.04183030128479004,
-      "rewards/margins": 0.08989714086055756,
-      "rewards/rejected": -0.04806683957576752,
+      "epoch": 1.2,
+      "learning_rate": 3.33843797856049e-07,
+      "logits/chosen": -2.4084014892578125,
+      "logits/rejected": -2.394484043121338,
+      "logps/chosen": -283.18255615234375,
+      "logps/rejected": -228.3118438720703,
+      "loss": 0.5931,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": 0.00656654080376029,
+      "rewards/margins": 0.30469295382499695,
+      "rewards/rejected": -0.2981263995170593,
       "step": 290
     },
     {
-      "epoch": 0.31,
-      "learning_rate": 4.982778415614236e-07,
-      "logits/chosen": -2.3887062072753906,
-      "logits/rejected": -2.34873628616333,
-      "logps/chosen": -236.2488250732422,
-      "logps/rejected": -220.21450805664062,
-      "loss": 0.6523,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.022816497832536697,
-      "rewards/margins": 0.09650030732154846,
-      "rewards/rejected": -0.07368380576372147,
+      "epoch": 1.24,
+      "learning_rate": 3.2618683001531396e-07,
+      "logits/chosen": -2.408291816711426,
+      "logits/rejected": -2.337432861328125,
+      "logps/chosen": -261.3971252441406,
+      "logps/rejected": -227.752685546875,
+      "loss": 0.5892,
+      "rewards/accuracies": 0.707812488079071,
+      "rewards/chosen": 0.009347354993224144,
+      "rewards/margins": 0.2984946370124817,
+      "rewards/rejected": -0.289147287607193,
       "step": 300
     },
     {
-      "epoch": 0.32,
-      "learning_rate": 4.963643321852277e-07,
-      "logits/chosen": -2.425257444381714,
-      "logits/rejected": -2.330592393875122,
-      "logps/chosen": -285.37896728515625,
-      "logps/rejected": -231.2700653076172,
-      "loss": 0.6515,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.010598802007734776,
-      "rewards/margins": 0.0920100286602974,
-      "rewards/rejected": -0.0814112201333046,
+      "epoch": 1.28,
+      "learning_rate": 3.1852986217457885e-07,
+      "logits/chosen": -2.45731782913208,
+      "logits/rejected": -2.3738834857940674,
+      "logps/chosen": -261.27099609375,
+      "logps/rejected": -228.57211303710938,
+      "loss": 0.5974,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": 0.005537848919630051,
+      "rewards/margins": 0.2985115647315979,
+      "rewards/rejected": -0.29297369718551636,
       "step": 310
     },
     {
-      "epoch": 0.33,
-      "learning_rate": 4.944508228090318e-07,
-      "logits/chosen": -2.4376401901245117,
-      "logits/rejected": -2.4971015453338623,
-      "logps/chosen": -238.9815216064453,
-      "logps/rejected": -192.4442138671875,
-      "loss": 0.6369,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.04103684425354004,
-      "rewards/margins": 0.13544507324695587,
-      "rewards/rejected": -0.09440822899341583,
+      "epoch": 1.32,
+      "learning_rate": 3.108728943338438e-07,
+      "logits/chosen": -2.4441769123077393,
+      "logits/rejected": -2.4352951049804688,
+      "logps/chosen": -252.8352813720703,
+      "logps/rejected": -237.8563995361328,
+      "loss": 0.5959,
+      "rewards/accuracies": 0.667187511920929,
+      "rewards/chosen": -0.016652364283800125,
+      "rewards/margins": 0.24610964953899384,
+      "rewards/rejected": -0.26276201009750366,
       "step": 320
     },
     {
-      "epoch": 0.34,
-      "learning_rate": 4.925373134328357e-07,
-      "logits/chosen": -2.4623780250549316,
-      "logits/rejected": -2.439487934112549,
-      "logps/chosen": -252.5996551513672,
-      "logps/rejected": -222.22421264648438,
-      "loss": 0.6422,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 0.04304979741573334,
-      "rewards/margins": 0.10223046690225601,
-      "rewards/rejected": -0.05918065458536148,
+      "epoch": 1.36,
+      "learning_rate": 3.0321592649310873e-07,
+      "logits/chosen": -2.424257516860962,
+      "logits/rejected": -2.357093572616577,
+      "logps/chosen": -253.7244873046875,
+      "logps/rejected": -224.3267822265625,
+      "loss": 0.5865,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.007790223695337772,
+      "rewards/margins": 0.30015501379966736,
+      "rewards/rejected": -0.30794528126716614,
       "step": 330
     },
     {
-      "epoch": 0.35,
-      "learning_rate": 4.906238040566398e-07,
-      "logits/chosen": -2.4748172760009766,
-      "logits/rejected": -2.356078863143921,
-      "logps/chosen": -271.64324951171875,
-      "logps/rejected": -231.6174774169922,
-      "loss": 0.6425,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.041741594672203064,
-      "rewards/margins": 0.15150585770606995,
-      "rewards/rejected": -0.10976427793502808,
+      "epoch": 1.4,
+      "learning_rate": 2.955589586523736e-07,
+      "logits/chosen": -2.442631244659424,
+      "logits/rejected": -2.3830604553222656,
+      "logps/chosen": -265.7073974609375,
+      "logps/rejected": -226.4299774169922,
+      "loss": 0.5916,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.016206540167331696,
+      "rewards/margins": 0.32533493638038635,
+      "rewards/rejected": -0.34154146909713745,
       "step": 340
     },
     {
-      "epoch": 0.36,
-      "learning_rate": 4.887102946804438e-07,
-      "logits/chosen": -2.5339407920837402,
-      "logits/rejected": -2.4247193336486816,
-      "logps/chosen": -289.107177734375,
-      "logps/rejected": -223.7015838623047,
-      "loss": 0.6301,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.05352502316236496,
-      "rewards/margins": 0.20187970995903015,
-      "rewards/rejected": -0.1483546942472458,
+      "epoch": 1.45,
+      "learning_rate": 2.8790199081163856e-07,
+      "logits/chosen": -2.4397027492523193,
+      "logits/rejected": -2.369683265686035,
+      "logps/chosen": -266.0716857910156,
+      "logps/rejected": -225.29916381835938,
+      "loss": 0.5941,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.029331281781196594,
+      "rewards/margins": 0.2816322445869446,
+      "rewards/rejected": -0.31096351146698,
       "step": 350
     },
     {
-      "epoch": 0.37,
-      "learning_rate": 4.867967853042479e-07,
-      "logits/chosen": -2.44026255607605,
-      "logits/rejected": -2.4695184230804443,
-      "logps/chosen": -272.41827392578125,
-      "logps/rejected": -223.8868408203125,
-      "loss": 0.6206,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.06177166849374771,
-      "rewards/margins": 0.19406349956989288,
-      "rewards/rejected": -0.13229182362556458,
+      "epoch": 1.49,
+      "learning_rate": 2.802450229709035e-07,
+      "logits/chosen": -2.3996329307556152,
+      "logits/rejected": -2.348875045776367,
+      "logps/chosen": -254.8851776123047,
+      "logps/rejected": -213.3196563720703,
+      "loss": 0.5843,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.037624605000019073,
+      "rewards/margins": 0.30041417479515076,
+      "rewards/rejected": -0.3380388021469116,
       "step": 360
     },
     {
-      "epoch": 0.38,
-      "learning_rate": 4.84883275928052e-07,
-      "logits/chosen": -2.4581284523010254,
-      "logits/rejected": -2.422722339630127,
-      "logps/chosen": -274.32763671875,
-      "logps/rejected": -228.18148803710938,
-      "loss": 0.6222,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.04478592425584793,
-      "rewards/margins": 0.19903826713562012,
-      "rewards/rejected": -0.1542523354291916,
+      "epoch": 1.53,
+      "learning_rate": 2.725880551301684e-07,
+      "logits/chosen": -2.4621551036834717,
+      "logits/rejected": -2.4065911769866943,
+      "logps/chosen": -274.72161865234375,
+      "logps/rejected": -232.85791015625,
+      "loss": 0.5928,
+      "rewards/accuracies": 0.6734374761581421,
+      "rewards/chosen": -0.0233127661049366,
+      "rewards/margins": 0.30943089723587036,
+      "rewards/rejected": -0.33274370431900024,
       "step": 370
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 4.82969766551856e-07,
-      "logits/chosen": -2.365147829055786,
-      "logits/rejected": -2.376405954360962,
-      "logps/chosen": -258.0191345214844,
-      "logps/rejected": -228.0791015625,
-      "loss": 0.6345,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.04648314788937569,
-      "rewards/margins": 0.18209731578826904,
-      "rewards/rejected": -0.13561417162418365,
+      "epoch": 1.57,
+      "learning_rate": 2.649310872894334e-07,
+      "logits/chosen": -2.448286771774292,
+      "logits/rejected": -2.4156296253204346,
+      "logps/chosen": -274.979248046875,
+      "logps/rejected": -223.11279296875,
+      "loss": 0.5817,
+      "rewards/accuracies": 0.7093750238418579,
+      "rewards/chosen": -0.01715572364628315,
+      "rewards/margins": 0.3696514070034027,
+      "rewards/rejected": -0.3868071436882019,
       "step": 380
     },
     {
-      "epoch": 0.4,
-      "learning_rate": 4.810562571756601e-07,
-      "logits/chosen": -2.418147325515747,
-      "logits/rejected": -2.405946969985962,
-      "logps/chosen": -255.6215362548828,
-      "logps/rejected": -219.8120880126953,
-      "loss": 0.6262,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.01959371007978916,
-      "rewards/margins": 0.18485160171985626,
-      "rewards/rejected": -0.16525788605213165,
+      "epoch": 1.61,
+      "learning_rate": 2.572741194486983e-07,
+      "logits/chosen": -2.4490604400634766,
+      "logits/rejected": -2.3936374187469482,
+      "logps/chosen": -273.819580078125,
+      "logps/rejected": -208.39321899414062,
+      "loss": 0.5798,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.02806365117430687,
+      "rewards/margins": 0.3674148619174957,
+      "rewards/rejected": -0.3954785466194153,
       "step": 390
     },
     {
-      "epoch": 0.41,
-      "learning_rate": 4.791427477994642e-07,
-      "logits/chosen": -2.419095516204834,
-      "logits/rejected": -2.4036061763763428,
-      "logps/chosen": -306.9060974121094,
-      "logps/rejected": -239.7941436767578,
-      "loss": 0.6195,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.06084824725985527,
-      "rewards/margins": 0.20281776785850525,
-      "rewards/rejected": -0.14196953177452087,
+      "epoch": 1.65,
+      "learning_rate": 2.496171516079632e-07,
+      "logits/chosen": -2.4658896923065186,
+      "logits/rejected": -2.3998141288757324,
+      "logps/chosen": -293.2339172363281,
+      "logps/rejected": -239.4620819091797,
+      "loss": 0.5813,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": 0.006555554457008839,
+      "rewards/margins": 0.39713528752326965,
+      "rewards/rejected": -0.3905797302722931,
       "step": 400
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 4.772292384232682e-07,
-      "logits/chosen": -2.417729377746582,
-      "logits/rejected": -2.3787436485290527,
-      "logps/chosen": -246.24478149414062,
-      "logps/rejected": -245.55166625976562,
-      "loss": 0.6208,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.024662364274263382,
-      "rewards/margins": 0.1620568484067917,
-      "rewards/rejected": -0.18671919405460358,
+      "epoch": 1.69,
+      "learning_rate": 2.4196018376722816e-07,
+      "logits/chosen": -2.4295265674591064,
+      "logits/rejected": -2.4006526470184326,
+      "logps/chosen": -278.5876159667969,
+      "logps/rejected": -228.4311065673828,
+      "loss": 0.5869,
+      "rewards/accuracies": 0.6890624761581421,
+      "rewards/chosen": -0.02325974591076374,
+      "rewards/margins": 0.365833044052124,
+      "rewards/rejected": -0.3890928328037262,
       "step": 410
     },
     {
-      "epoch": 0.43,
-      "learning_rate": 4.753157290470723e-07,
-      "logits/chosen": -2.4877963066101074,
-      "logits/rejected": -2.4664740562438965,
-      "logps/chosen": -256.0787048339844,
-      "logps/rejected": -225.0019989013672,
-      "loss": 0.6281,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.006953413598239422,
-      "rewards/margins": 0.161538228392601,
-      "rewards/rejected": -0.15458481013774872,
+      "epoch": 1.74,
+      "learning_rate": 2.343032159264931e-07,
+      "logits/chosen": -2.403923511505127,
+      "logits/rejected": -2.3335375785827637,
+      "logps/chosen": -268.8808898925781,
+      "logps/rejected": -224.3599395751953,
+      "loss": 0.5782,
+      "rewards/accuracies": 0.7203124761581421,
+      "rewards/chosen": -0.01940683089196682,
+      "rewards/margins": 0.4325495660305023,
+      "rewards/rejected": -0.4519564211368561,
       "step": 420
     },
     {
-      "epoch": 0.44,
-      "learning_rate": 4.7340221967087635e-07,
-      "logits/chosen": -2.412123918533325,
-      "logits/rejected": -2.3679380416870117,
-      "logps/chosen": -251.4219970703125,
-      "logps/rejected": -224.938720703125,
-      "loss": 0.6154,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.0361722894012928,
-      "rewards/margins": 0.22137188911437988,
-      "rewards/rejected": -0.1851995885372162,
+      "epoch": 1.78,
+      "learning_rate": 2.26646248085758e-07,
+      "logits/chosen": -2.3836731910705566,
+      "logits/rejected": -2.3358378410339355,
+      "logps/chosen": -259.7103576660156,
+      "logps/rejected": -217.7801055908203,
+      "loss": 0.5726,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": -0.03996918350458145,
+      "rewards/margins": 0.39050012826919556,
+      "rewards/rejected": -0.4304693341255188,
       "step": 430
     },
     {
-      "epoch": 0.45,
-      "learning_rate": 4.714887102946804e-07,
-      "logits/chosen": -2.470468521118164,
-      "logits/rejected": -2.3362622261047363,
-      "logps/chosen": -257.301513671875,
-      "logps/rejected": -200.90008544921875,
-      "loss": 0.6021,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.01734147220849991,
-      "rewards/margins": 0.22712071239948273,
-      "rewards/rejected": -0.20977921783924103,
+      "epoch": 1.82,
+      "learning_rate": 2.1898928024502298e-07,
+      "logits/chosen": -2.4447569847106934,
+      "logits/rejected": -2.387596845626831,
+      "logps/chosen": -263.4599609375,
+      "logps/rejected": -221.48947143554688,
+      "loss": 0.5757,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.018027612939476967,
+      "rewards/margins": 0.3998379111289978,
+      "rewards/rejected": -0.417865514755249,
       "step": 440
     },
     {
-      "epoch": 0.46,
-      "learning_rate": 4.6957520091848447e-07,
-      "logits/chosen": -2.4263899326324463,
-      "logits/rejected": -2.3918890953063965,
-      "logps/chosen": -228.27206420898438,
-      "logps/rejected": -204.10134887695312,
-      "loss": 0.6338,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.007703957613557577,
-      "rewards/margins": 0.17124322056770325,
-      "rewards/rejected": -0.16353924572467804,
+      "epoch": 1.86,
+      "learning_rate": 2.113323124042879e-07,
+      "logits/chosen": -2.4276745319366455,
+      "logits/rejected": -2.390892744064331,
+      "logps/chosen": -271.26995849609375,
+      "logps/rejected": -231.4360809326172,
+      "loss": 0.5756,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": -0.021533582359552383,
+      "rewards/margins": 0.40987348556518555,
+      "rewards/rejected": -0.43140706419944763,
       "step": 450
     },
     {
-      "epoch": 0.48,
-      "learning_rate": 4.6766169154228853e-07,
-      "logits/chosen": -2.3560242652893066,
-      "logits/rejected": -2.3139007091522217,
-      "logps/chosen": -268.39398193359375,
-      "logps/rejected": -239.73159790039062,
-      "loss": 0.5952,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.06542753428220749,
-      "rewards/margins": 0.2891160547733307,
-      "rewards/rejected": -0.2236885279417038,
+      "epoch": 1.9,
+      "learning_rate": 2.036753445635528e-07,
+      "logits/chosen": -2.4362778663635254,
+      "logits/rejected": -2.4150547981262207,
+      "logps/chosen": -284.03668212890625,
+      "logps/rejected": -238.2490234375,
+      "loss": 0.5694,
+      "rewards/accuracies": 0.7171875238418579,
+      "rewards/chosen": -0.020668620243668556,
+      "rewards/margins": 0.45047345757484436,
+      "rewards/rejected": -0.47114211320877075,
       "step": 460
     },
     {
-      "epoch": 0.49,
-      "learning_rate": 4.657481821660926e-07,
-      "logits/chosen": -2.452782154083252,
-      "logits/rejected": -2.4332146644592285,
-      "logps/chosen": -283.4723205566406,
-      "logps/rejected": -211.5760498046875,
-      "loss": 0.5978,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.004657271318137646,
-      "rewards/margins": 0.2682662606239319,
-      "rewards/rejected": -0.2729235589504242,
+      "epoch": 1.94,
+      "learning_rate": 1.9601837672281775e-07,
+      "logits/chosen": -2.418320894241333,
+      "logits/rejected": -2.385996103286743,
+      "logps/chosen": -270.16778564453125,
+      "logps/rejected": -236.37612915039062,
+      "loss": 0.5749,
+      "rewards/accuracies": 0.7015625238418579,
+      "rewards/chosen": -0.05912749841809273,
+      "rewards/margins": 0.3770337700843811,
+      "rewards/rejected": -0.4361612796783447,
       "step": 470
     },
     {
-      "epoch": 0.5,
-      "learning_rate": 4.6383467278989666e-07,
-      "logits/chosen": -2.3910794258117676,
-      "logits/rejected": -2.4533417224884033,
-      "logps/chosen": -248.0291748046875,
-      "logps/rejected": -230.6380157470703,
-      "loss": 0.6143,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.015768401324748993,
-      "rewards/margins": 0.2060297429561615,
-      "rewards/rejected": -0.22179818153381348,
+      "epoch": 1.98,
+      "learning_rate": 1.883614088820827e-07,
+      "logits/chosen": -2.438436269760132,
+      "logits/rejected": -2.3922858238220215,
+      "logps/chosen": -268.78424072265625,
+      "logps/rejected": -224.87625122070312,
+      "loss": 0.5855,
+      "rewards/accuracies": 0.7203124761581421,
+      "rewards/chosen": -0.05140022188425064,
+      "rewards/margins": 0.37707191705703735,
+      "rewards/rejected": -0.4284721314907074,
       "step": 480
     },
     {
-      "epoch": 0.51,
-      "learning_rate": 4.6192116341370067e-07,
-      "logits/chosen": -2.5222439765930176,
-      "logits/rejected": -2.4398193359375,
-      "logps/chosen": -277.2117614746094,
-      "logps/rejected": -230.4596710205078,
-      "loss": 0.5947,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.04367998614907265,
-      "rewards/margins": 0.33074626326560974,
-      "rewards/rejected": -0.2870662808418274,
+      "epoch": 2.0,
+      "eval_logits/chosen": -2.1276535987854004,
+      "eval_logits/rejected": -2.0070486068725586,
+      "eval_logps/chosen": -265.26239013671875,
+      "eval_logps/rejected": -223.9853057861328,
+      "eval_loss": 0.5732065439224243,
+      "eval_rewards/accuracies": 0.6919999718666077,
+      "eval_rewards/chosen": -0.06015579402446747,
+      "eval_rewards/margins": 0.4099440574645996,
+      "eval_rewards/rejected": -0.47009986639022827,
+      "eval_runtime": 443.3179,
+      "eval_samples_per_second": 4.511,
+      "eval_steps_per_second": 0.282,
+      "step": 484
+    },
+    {
+      "epoch": 2.02,
+      "learning_rate": 1.807044410413476e-07,
+      "logits/chosen": -2.4100403785705566,
+      "logits/rejected": -2.3768649101257324,
+      "logps/chosen": -257.94842529296875,
+      "logps/rejected": -236.3602294921875,
+      "loss": 0.5773,
+      "rewards/accuracies": 0.707812488079071,
+      "rewards/chosen": -0.05285615846514702,
+      "rewards/margins": 0.3980172574520111,
+      "rewards/rejected": -0.4508734345436096,
       "step": 490
     },
     {
-      "epoch": 0.52,
-      "learning_rate": 4.6000765403750473e-07,
-      "logits/chosen": -2.423351764678955,
-      "logits/rejected": -2.3357410430908203,
-      "logps/chosen": -255.89846801757812,
-      "logps/rejected": -211.130126953125,
-      "loss": 0.5989,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 0.019995370879769325,
-      "rewards/margins": 0.26641523838043213,
-      "rewards/rejected": -0.24641986191272736,
+      "epoch": 2.07,
+      "learning_rate": 1.7304747320061255e-07,
+      "logits/chosen": -2.4070606231689453,
+      "logits/rejected": -2.403454542160034,
+      "logps/chosen": -263.56475830078125,
+      "logps/rejected": -230.64697265625,
+      "loss": 0.577,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.01720670983195305,
+      "rewards/margins": 0.4001820683479309,
+      "rewards/rejected": -0.41738876700401306,
       "step": 500
     },
     {
-      "epoch": 0.53,
-      "learning_rate": 4.580941446613088e-07,
-      "logits/chosen": -2.4628067016601562,
-      "logits/rejected": -2.4185214042663574,
-      "logps/chosen": -257.75567626953125,
-      "logps/rejected": -238.31454467773438,
-      "loss": 0.5993,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.023913182318210602,
-      "rewards/margins": 0.23212233185768127,
-      "rewards/rejected": -0.2560355067253113,
+      "epoch": 2.11,
+      "learning_rate": 1.6539050535987747e-07,
+      "logits/chosen": -2.479860782623291,
+      "logits/rejected": -2.3707234859466553,
+      "logps/chosen": -270.1263122558594,
+      "logps/rejected": -225.0696563720703,
+      "loss": 0.5704,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.03719691187143326,
+      "rewards/margins": 0.4389099180698395,
+      "rewards/rejected": -0.47610679268836975,
       "step": 510
     },
     {
-      "epoch": 0.54,
-      "learning_rate": 4.5618063528511285e-07,
-      "logits/chosen": -2.3876211643218994,
-      "logits/rejected": -2.357927083969116,
-      "logps/chosen": -274.25555419921875,
-      "logps/rejected": -232.0738525390625,
-      "loss": 0.5993,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.04971958324313164,
-      "rewards/margins": 0.2438163459300995,
-      "rewards/rejected": -0.29353591799736023,
+      "epoch": 2.15,
+      "learning_rate": 1.5773353751914243e-07,
+      "logits/chosen": -2.486112356185913,
+      "logits/rejected": -2.425720691680908,
+      "logps/chosen": -284.87408447265625,
+      "logps/rejected": -229.99740600585938,
+      "loss": 0.5715,
+      "rewards/accuracies": 0.739062488079071,
+      "rewards/chosen": -0.023222165182232857,
+      "rewards/margins": 0.4711039662361145,
+      "rewards/rejected": -0.4943261742591858,
       "step": 520
     },
     {
-      "epoch": 0.55,
-      "learning_rate": 4.542671259089169e-07,
-      "logits/chosen": -2.4641833305358887,
-      "logits/rejected": -2.40731143951416,
-      "logps/chosen": -273.7682189941406,
-      "logps/rejected": -233.98583984375,
-      "loss": 0.6062,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.0051215579733252525,
-      "rewards/margins": 0.301483690738678,
-      "rewards/rejected": -0.3066052496433258,
+      "epoch": 2.19,
+      "learning_rate": 1.5007656967840735e-07,
+      "logits/chosen": -2.3832104206085205,
+      "logits/rejected": -2.342264175415039,
+      "logps/chosen": -254.04464721679688,
+      "logps/rejected": -230.5897216796875,
+      "loss": 0.5673,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.0390273854136467,
+      "rewards/margins": 0.44446659088134766,
+      "rewards/rejected": -0.4834940433502197,
       "step": 530
     },
     {
-      "epoch": 0.56,
-      "learning_rate": 4.52353616532721e-07,
-      "logits/chosen": -2.4376657009124756,
-      "logits/rejected": -2.4150702953338623,
-      "logps/chosen": -266.14349365234375,
-      "logps/rejected": -235.13882446289062,
-      "loss": 0.6018,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.014219239354133606,
-      "rewards/margins": 0.26282960176467896,
-      "rewards/rejected": -0.27704882621765137,
+      "epoch": 2.23,
+      "learning_rate": 1.4241960183767226e-07,
+      "logits/chosen": -2.429234266281128,
+      "logits/rejected": -2.3727104663848877,
+      "logps/chosen": -282.85357666015625,
+      "logps/rejected": -235.9081573486328,
+      "loss": 0.5731,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.05646444484591484,
+      "rewards/margins": 0.42204102873802185,
+      "rewards/rejected": -0.47850552201271057,
       "step": 540
     },
     {
-      "epoch": 0.57,
-      "learning_rate": 4.5044010715652504e-07,
-      "logits/chosen": -2.4388842582702637,
-      "logits/rejected": -2.390779495239258,
-      "logps/chosen": -249.3515167236328,
-      "logps/rejected": -228.97885131835938,
-      "loss": 0.6,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.028151463717222214,
-      "rewards/margins": 0.2760944068431854,
-      "rewards/rejected": -0.30424588918685913,
+      "epoch": 2.27,
+      "learning_rate": 1.347626339969372e-07,
+      "logits/chosen": -2.4232146739959717,
+      "logits/rejected": -2.387540578842163,
+      "logps/chosen": -270.8162841796875,
+      "logps/rejected": -242.1083984375,
+      "loss": 0.5748,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.012197254225611687,
+      "rewards/margins": 0.41364020109176636,
+      "rewards/rejected": -0.4258374571800232,
       "step": 550
     },
     {
-      "epoch": 0.58,
-      "learning_rate": 4.485265977803291e-07,
-      "logits/chosen": -2.460728406906128,
-      "logits/rejected": -2.3551411628723145,
-      "logps/chosen": -293.16400146484375,
-      "logps/rejected": -232.632080078125,
-      "loss": 0.5871,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.06439458578824997,
-      "rewards/margins": 0.4292263984680176,
-      "rewards/rejected": -0.3648317754268646,
+      "epoch": 2.31,
+      "learning_rate": 1.2710566615620215e-07,
+      "logits/chosen": -2.3734302520751953,
+      "logits/rejected": -2.3281941413879395,
+      "logps/chosen": -274.3441467285156,
+      "logps/rejected": -225.1654052734375,
+      "loss": 0.5604,
+      "rewards/accuracies": 0.7046874761581421,
+      "rewards/chosen": -0.03567253798246384,
+      "rewards/margins": 0.44296032190322876,
+      "rewards/rejected": -0.4786328375339508,
       "step": 560
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 4.4661308840413316e-07,
-      "logits/chosen": -2.509218215942383,
-      "logits/rejected": -2.375960111618042,
-      "logps/chosen": -285.7981262207031,
-      "logps/rejected": -236.2060546875,
-      "loss": 0.6024,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.004936461336910725,
-      "rewards/margins": 0.36126184463500977,
-      "rewards/rejected": -0.3563253879547119,
+      "epoch": 2.35,
+      "learning_rate": 1.1944869831546706e-07,
+      "logits/chosen": -2.3997061252593994,
+      "logits/rejected": -2.3794922828674316,
+      "logps/chosen": -267.03338623046875,
+      "logps/rejected": -238.7495880126953,
+      "loss": 0.573,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.07530860602855682,
+      "rewards/margins": 0.4091506898403168,
+      "rewards/rejected": -0.4844593107700348,
       "step": 570
     },
     {
-      "epoch": 0.6,
-      "learning_rate": 4.446995790279372e-07,
-      "logits/chosen": -2.455012321472168,
-      "logits/rejected": -2.436042547225952,
-      "logps/chosen": -283.45404052734375,
-      "logps/rejected": -230.90603637695312,
-      "loss": 0.586,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.004780585877597332,
-      "rewards/margins": 0.32074013352394104,
-      "rewards/rejected": -0.31595954298973083,
+      "epoch": 2.4,
+      "learning_rate": 1.11791730474732e-07,
+      "logits/chosen": -2.413222551345825,
+      "logits/rejected": -2.3744866847991943,
+      "logps/chosen": -262.73187255859375,
+      "logps/rejected": -226.4947967529297,
+      "loss": 0.5655,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.0698440670967102,
+      "rewards/margins": 0.4843732416629791,
+      "rewards/rejected": -0.5542173385620117,
       "step": 580
     },
     {
-      "epoch": 0.61,
-      "learning_rate": 4.4278606965174123e-07,
-      "logits/chosen": -2.4725747108459473,
-      "logits/rejected": -2.470695734024048,
-      "logps/chosen": -262.63873291015625,
-      "logps/rejected": -241.70535278320312,
-      "loss": 0.5869,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.011715828441083431,
-      "rewards/margins": 0.34255507588386536,
-      "rewards/rejected": -0.35427090525627136,
+      "epoch": 2.44,
+      "learning_rate": 1.0413476263399694e-07,
+      "logits/chosen": -2.4516360759735107,
+      "logits/rejected": -2.4073665142059326,
+      "logps/chosen": -269.501220703125,
+      "logps/rejected": -219.1105499267578,
+      "loss": 0.567,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.07218033820390701,
+      "rewards/margins": 0.43574219942092896,
+      "rewards/rejected": -0.5079224705696106,
       "step": 590
     },
     {
-      "epoch": 0.62,
-      "learning_rate": 4.408725602755453e-07,
-      "logits/chosen": -2.460839033126831,
-      "logits/rejected": -2.407060146331787,
-      "logps/chosen": -233.80014038085938,
-      "logps/rejected": -207.22445678710938,
-      "loss": 0.5908,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.03514878451824188,
-      "rewards/margins": 0.27000272274017334,
-      "rewards/rejected": -0.3051515221595764,
+      "epoch": 2.48,
+      "learning_rate": 9.647779479326186e-08,
+      "logits/chosen": -2.3931024074554443,
+      "logits/rejected": -2.3882880210876465,
+      "logps/chosen": -251.0735321044922,
+      "logps/rejected": -224.48800659179688,
+      "loss": 0.5767,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.06539195775985718,
+      "rewards/margins": 0.3568459153175354,
+      "rewards/rejected": -0.4222378134727478,
       "step": 600
     },
     {
-      "epoch": 0.63,
-      "learning_rate": 4.3895905089934936e-07,
-      "logits/chosen": -2.4760079383850098,
-      "logits/rejected": -2.3460214138031006,
-      "logps/chosen": -276.608154296875,
-      "logps/rejected": -231.1323699951172,
-      "loss": 0.59,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.0008779823547229171,
-      "rewards/margins": 0.3627086579799652,
-      "rewards/rejected": -0.3618307411670685,
+      "epoch": 2.52,
+      "learning_rate": 8.88208269525268e-08,
+      "logits/chosen": -2.3815078735351562,
+      "logits/rejected": -2.3914084434509277,
+      "logps/chosen": -260.74053955078125,
+      "logps/rejected": -223.4197540283203,
+      "loss": 0.5663,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.07167039811611176,
+      "rewards/margins": 0.41475382447242737,
+      "rewards/rejected": -0.4864242672920227,
       "step": 610
     },
     {
-      "epoch": 0.64,
-      "learning_rate": 4.370455415231534e-07,
-      "logits/chosen": -2.4187588691711426,
-      "logits/rejected": -2.426133632659912,
-      "logps/chosen": -276.6905212402344,
-      "logps/rejected": -260.84893798828125,
-      "loss": 0.5706,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.05571802332997322,
-      "rewards/margins": 0.310151606798172,
-      "rewards/rejected": -0.3658696413040161,
+      "epoch": 2.56,
+      "learning_rate": 8.116385911179173e-08,
+      "logits/chosen": -2.3715600967407227,
+      "logits/rejected": -2.362975597381592,
+      "logps/chosen": -279.5382080078125,
+      "logps/rejected": -219.50137329101562,
+      "loss": 0.5684,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": -0.020305603742599487,
+      "rewards/margins": 0.4676498472690582,
+      "rewards/rejected": -0.4879554212093353,
       "step": 620
     },
     {
-      "epoch": 0.65,
-      "learning_rate": 4.351320321469575e-07,
-      "logits/chosen": -2.4383225440979004,
-      "logits/rejected": -2.401784658432007,
-      "logps/chosen": -298.56390380859375,
-      "logps/rejected": -235.006591796875,
-      "loss": 0.5838,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.035277482122182846,
-      "rewards/margins": 0.44551342725753784,
-      "rewards/rejected": -0.4102359712123871,
+      "epoch": 2.6,
+      "learning_rate": 7.350689127105667e-08,
+      "logits/chosen": -2.453105926513672,
+      "logits/rejected": -2.378638744354248,
+      "logps/chosen": -272.66357421875,
+      "logps/rejected": -232.73440551757812,
+      "loss": 0.5605,
+      "rewards/accuracies": 0.7046874761581421,
+      "rewards/chosen": -0.04887257143855095,
+      "rewards/margins": 0.47081702947616577,
+      "rewards/rejected": -0.519689679145813,
       "step": 630
     },
     {
-      "epoch": 0.66,
-      "learning_rate": 4.3321852277076154e-07,
-      "logits/chosen": -2.4030394554138184,
-      "logits/rejected": -2.390498638153076,
-      "logps/chosen": -267.2048645019531,
-      "logps/rejected": -240.0560760498047,
-      "loss": 0.5693,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.008936256170272827,
-      "rewards/margins": 0.3941975235939026,
-      "rewards/rejected": -0.3852612376213074,
+      "epoch": 2.64,
+      "learning_rate": 6.584992343032159e-08,
+      "logits/chosen": -2.355799436569214,
+      "logits/rejected": -2.3423495292663574,
+      "logps/chosen": -263.2275390625,
+      "logps/rejected": -229.0945587158203,
+      "loss": 0.5757,
+      "rewards/accuracies": 0.7015625238418579,
+      "rewards/chosen": -0.05284266546368599,
+      "rewards/margins": 0.4212065637111664,
+      "rewards/rejected": -0.47404924035072327,
       "step": 640
     },
     {
-      "epoch": 0.67,
-      "learning_rate": 4.313050133945656e-07,
-      "logits/chosen": -2.508209705352783,
-      "logits/rejected": -2.45387601852417,
-      "logps/chosen": -291.5556640625,
-      "logps/rejected": -241.94131469726562,
-      "loss": 0.5804,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.0002497173845767975,
-      "rewards/margins": 0.4533967971801758,
-      "rewards/rejected": -0.4536465108394623,
+      "epoch": 2.69,
+      "learning_rate": 5.819295558958652e-08,
+      "logits/chosen": -2.399625301361084,
+      "logits/rejected": -2.349210262298584,
+      "logps/chosen": -288.31817626953125,
+      "logps/rejected": -222.42892456054688,
+      "loss": 0.5645,
+      "rewards/accuracies": 0.7484375238418579,
+      "rewards/chosen": -0.02390669845044613,
+      "rewards/margins": 0.5248440504074097,
+      "rewards/rejected": -0.5487507581710815,
       "step": 650
     },
     {
-      "epoch": 0.68,
-      "learning_rate": 4.2939150401836967e-07,
-      "logits/chosen": -2.4588632583618164,
-      "logits/rejected": -2.461416482925415,
-      "logps/chosen": -245.1895751953125,
-      "logps/rejected": -246.0583038330078,
-      "loss": 0.5693,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.06281181424856186,
-      "rewards/margins": 0.3227921426296234,
-      "rewards/rejected": -0.38560396432876587,
+      "epoch": 2.73,
+      "learning_rate": 5.0535987748851455e-08,
+      "logits/chosen": -2.4646735191345215,
+      "logits/rejected": -2.4114043712615967,
+      "logps/chosen": -275.809326171875,
+      "logps/rejected": -231.97802734375,
+      "loss": 0.5615,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.07590371370315552,
+      "rewards/margins": 0.411339670419693,
+      "rewards/rejected": -0.4872434139251709,
       "step": 660
     },
     {
-      "epoch": 0.69,
-      "learning_rate": 4.2747799464217373e-07,
-      "logits/chosen": -2.397804021835327,
-      "logits/rejected": -2.3986358642578125,
-      "logps/chosen": -262.5384521484375,
-      "logps/rejected": -231.0225830078125,
-      "loss": 0.5746,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.10527386516332626,
-      "rewards/margins": 0.3180972933769226,
-      "rewards/rejected": -0.42337116599082947,
+      "epoch": 2.77,
+      "learning_rate": 4.287901990811638e-08,
+      "logits/chosen": -2.4426465034484863,
+      "logits/rejected": -2.3569984436035156,
+      "logps/chosen": -268.51104736328125,
+      "logps/rejected": -244.9394989013672,
+      "loss": 0.5639,
+      "rewards/accuracies": 0.7015625238418579,
+      "rewards/chosen": -0.04740380123257637,
+      "rewards/margins": 0.4665365219116211,
+      "rewards/rejected": -0.5139402747154236,
       "step": 670
     },
     {
-      "epoch": 0.7,
-      "learning_rate": 4.255644852659778e-07,
-      "logits/chosen": -2.49613618850708,
-      "logits/rejected": -2.4438865184783936,
-      "logps/chosen": -287.05419921875,
-      "logps/rejected": -249.3083953857422,
-      "loss": 0.6131,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.08329292386770248,
-      "rewards/margins": 0.29655909538269043,
-      "rewards/rejected": -0.3798519968986511,
+      "epoch": 2.81,
+      "learning_rate": 3.522205206738132e-08,
+      "logits/chosen": -2.4069762229919434,
+      "logits/rejected": -2.3731188774108887,
+      "logps/chosen": -275.3673400878906,
+      "logps/rejected": -230.1641387939453,
+      "loss": 0.568,
+      "rewards/accuracies": 0.723437488079071,
+      "rewards/chosen": -0.07594893872737885,
+      "rewards/margins": 0.4454987049102783,
+      "rewards/rejected": -0.5214475989341736,
       "step": 680
     },
     {
-      "epoch": 0.71,
-      "learning_rate": 4.236509758897818e-07,
-      "logits/chosen": -2.3793463706970215,
-      "logits/rejected": -2.387934684753418,
-      "logps/chosen": -260.1773986816406,
-      "logps/rejected": -218.5209197998047,
-      "loss": 0.602,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.06579776108264923,
-      "rewards/margins": 0.35870271921157837,
-      "rewards/rejected": -0.4245004653930664,
+      "epoch": 2.85,
+      "learning_rate": 2.7565084226646246e-08,
+      "logits/chosen": -2.3884053230285645,
+      "logits/rejected": -2.3761892318725586,
+      "logps/chosen": -264.29754638671875,
+      "logps/rejected": -236.7384033203125,
+      "loss": 0.5738,
+      "rewards/accuracies": 0.7093750238418579,
+      "rewards/chosen": -0.04798636958003044,
+      "rewards/margins": 0.462782621383667,
+      "rewards/rejected": -0.5107689499855042,
       "step": 690
     },
     {
-      "epoch": 0.72,
-      "learning_rate": 4.2173746651358586e-07,
-      "logits/chosen": -2.412266254425049,
-      "logits/rejected": -2.389812469482422,
-      "logps/chosen": -260.27349853515625,
-      "logps/rejected": -246.9514923095703,
-      "loss": 0.5861,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.10205063968896866,
-      "rewards/margins": 0.2957732677459717,
-      "rewards/rejected": -0.39782392978668213,
+      "epoch": 2.89,
+      "learning_rate": 1.9908116385911178e-08,
+      "logits/chosen": -2.4066903591156006,
+      "logits/rejected": -2.388782501220703,
+      "logps/chosen": -260.63067626953125,
+      "logps/rejected": -227.6019744873047,
+      "loss": 0.5721,
+      "rewards/accuracies": 0.667187511920929,
+      "rewards/chosen": -0.0927988812327385,
+      "rewards/margins": 0.37709805369377136,
+      "rewards/rejected": -0.46989694237709045,
       "step": 700
     },
     {
-      "epoch": 0.73,
-      "learning_rate": 4.198239571373899e-07,
-      "logits/chosen": -2.42134690284729,
-      "logits/rejected": -2.353253126144409,
-      "logps/chosen": -243.3867645263672,
-      "logps/rejected": -195.7749481201172,
-      "loss": 0.5847,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.05715983361005783,
-      "rewards/margins": 0.43060773611068726,
-      "rewards/rejected": -0.48776760697364807,
+      "epoch": 2.93,
+      "learning_rate": 1.225114854517611e-08,
+      "logits/chosen": -2.4368255138397217,
+      "logits/rejected": -2.371954917907715,
+      "logps/chosen": -280.86224365234375,
+      "logps/rejected": -221.8843536376953,
+      "loss": 0.5689,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": -0.06322827190160751,
+      "rewards/margins": 0.44322142004966736,
+      "rewards/rejected": -0.5064496397972107,
       "step": 710
     },
     {
-      "epoch": 0.74,
-      "learning_rate": 4.17910447761194e-07,
-      "logits/chosen": -2.4673993587493896,
-      "logits/rejected": -2.3654751777648926,
-      "logps/chosen": -267.1549377441406,
-      "logps/rejected": -255.4532012939453,
-      "loss": 0.5956,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.08059550076723099,
-      "rewards/margins": 0.34076714515686035,
-      "rewards/rejected": -0.42136263847351074,
+      "epoch": 2.97,
+      "learning_rate": 4.594180704441042e-09,
+      "logits/chosen": -2.4003658294677734,
+      "logits/rejected": -2.3783950805664062,
+      "logps/chosen": -257.96258544921875,
+      "logps/rejected": -225.0150146484375,
+      "loss": 0.5678,
+      "rewards/accuracies": 0.7015625238418579,
+      "rewards/chosen": -0.05350871756672859,
+      "rewards/margins": 0.414130300283432,
+      "rewards/rejected": -0.4676390290260315,
       "step": 720
     },
-    {
-      "epoch": 0.75,
-      "learning_rate": 4.1599693838499805e-07,
-      "logits/chosen": -2.45065975189209,
-      "logits/rejected": -2.4266138076782227,
-      "logps/chosen": -295.47137451171875,
-      "logps/rejected": -215.19296264648438,
-      "loss": 0.5811,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.061896733939647675,
-      "rewards/margins": 0.3598650097846985,
-      "rewards/rejected": -0.42176175117492676,
-      "step": 730
-    },
-    {
-      "epoch": 0.76,
-      "learning_rate": 4.140834290088021e-07,
-      "logits/chosen": -2.4394631385803223,
-      "logits/rejected": -2.417912006378174,
-      "logps/chosen": -277.1748046875,
-      "logps/rejected": -221.8069610595703,
-      "loss": 0.5631,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.08770271390676498,
-      "rewards/margins": 0.44523605704307556,
-      "rewards/rejected": -0.5329388380050659,
-      "step": 740
-    },
-    {
-      "epoch": 0.77,
-      "learning_rate": 4.121699196326062e-07,
-      "logits/chosen": -2.435689926147461,
-      "logits/rejected": -2.310481548309326,
-      "logps/chosen": -218.2103729248047,
-      "logps/rejected": -185.0491943359375,
-      "loss": 0.5802,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.05216900631785393,
-      "rewards/margins": 0.38854002952575684,
-      "rewards/rejected": -0.44070905447006226,
-      "step": 750
-    },
-    {
-      "epoch": 0.78,
-      "learning_rate": 4.1025641025641024e-07,
-      "logits/chosen": -2.399353504180908,
-      "logits/rejected": -2.4066669940948486,
-      "logps/chosen": -259.16229248046875,
-      "logps/rejected": -243.228515625,
-      "loss": 0.5769,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.06289193779230118,
-      "rewards/margins": 0.4069492220878601,
-      "rewards/rejected": -0.4698411524295807,
-      "step": 760
-    },
-    {
-      "epoch": 0.8,
-      "learning_rate": 4.083429008802143e-07,
-      "logits/chosen": -2.525341510772705,
-      "logits/rejected": -2.471456527709961,
-      "logps/chosen": -274.21099853515625,
-      "logps/rejected": -213.09451293945312,
-      "loss": 0.5545,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.041963692754507065,
-      "rewards/margins": 0.450117826461792,
-      "rewards/rejected": -0.4920814633369446,
-      "step": 770
-    },
-    {
-      "epoch": 0.81,
-      "learning_rate": 4.0642939150401836e-07,
-      "logits/chosen": -2.487657070159912,
-      "logits/rejected": -2.3538174629211426,
-      "logps/chosen": -290.19036865234375,
-      "logps/rejected": -217.69906616210938,
-      "loss": 0.5707,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.06529206037521362,
-      "rewards/margins": 0.44091683626174927,
-      "rewards/rejected": -0.5062088966369629,
-      "step": 780
-    },
-    {
-      "epoch": 0.82,
-      "learning_rate": 4.0451588212782237e-07,
-      "logits/chosen": -2.429466724395752,
-      "logits/rejected": -2.395692825317383,
-      "logps/chosen": -255.69052124023438,
-      "logps/rejected": -227.69168090820312,
-      "loss": 0.5409,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.0646648183465004,
-      "rewards/margins": 0.5062236189842224,
-      "rewards/rejected": -0.5708884596824646,
-      "step": 790
-    },
-    {
-      "epoch": 0.83,
-      "learning_rate": 4.0260237275162643e-07,
-      "logits/chosen": -2.368137836456299,
-      "logits/rejected": -2.3667895793914795,
-      "logps/chosen": -232.4878692626953,
-      "logps/rejected": -227.72323608398438,
-      "loss": 0.5741,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.057848911732435226,
-      "rewards/margins": 0.45716094970703125,
-      "rewards/rejected": -0.5150099396705627,
-      "step": 800
-    },
-    {
-      "epoch": 0.84,
-      "learning_rate": 4.006888633754305e-07,
-      "logits/chosen": -2.5232253074645996,
-      "logits/rejected": -2.4718704223632812,
-      "logps/chosen": -277.5395812988281,
-      "logps/rejected": -229.75863647460938,
-      "loss": 0.545,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.02210831269621849,
-      "rewards/margins": 0.5734663605690002,
-      "rewards/rejected": -0.5955746173858643,
-      "step": 810
-    },
-    {
-      "epoch": 0.85,
-      "learning_rate": 3.9877535399923456e-07,
-      "logits/chosen": -2.4078879356384277,
-      "logits/rejected": -2.3697750568389893,
-      "logps/chosen": -267.1474914550781,
-      "logps/rejected": -224.79318237304688,
-      "loss": 0.5611,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.05940120667219162,
-      "rewards/margins": 0.5237945318222046,
-      "rewards/rejected": -0.583195686340332,
-      "step": 820
-    },
-    {
-      "epoch": 0.86,
-      "learning_rate": 3.968618446230386e-07,
-      "logits/chosen": -2.441854953765869,
-      "logits/rejected": -2.413933277130127,
-      "logps/chosen": -319.1496276855469,
-      "logps/rejected": -247.3388671875,
-      "loss": 0.5411,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.00216462230309844,
-      "rewards/margins": 0.6005392074584961,
-      "rewards/rejected": -0.6027037501335144,
-      "step": 830
-    },
-    {
-      "epoch": 0.87,
-      "learning_rate": 3.949483352468427e-07,
-      "logits/chosen": -2.441795825958252,
-      "logits/rejected": -2.408433437347412,
-      "logps/chosen": -275.55364990234375,
-      "logps/rejected": -240.1023712158203,
-      "loss": 0.5996,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.18812677264213562,
-      "rewards/margins": 0.3052147328853607,
-      "rewards/rejected": -0.4933415353298187,
-      "step": 840
-    },
-    {
-      "epoch": 0.88,
-      "learning_rate": 3.9303482587064674e-07,
-      "logits/chosen": -2.4030299186706543,
-      "logits/rejected": -2.4201598167419434,
-      "logps/chosen": -282.9662170410156,
-      "logps/rejected": -238.91635131835938,
-      "loss": 0.5653,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.07580298185348511,
-      "rewards/margins": 0.48956719040870667,
-      "rewards/rejected": -0.5653700828552246,
-      "step": 850
-    },
-    {
-      "epoch": 0.89,
-      "learning_rate": 3.911213164944508e-07,
-      "logits/chosen": -2.4482102394104004,
-      "logits/rejected": -2.3696365356445312,
-      "logps/chosen": -291.52374267578125,
-      "logps/rejected": -245.3708953857422,
-      "loss": 0.5645,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.0693991631269455,
-      "rewards/margins": 0.5479902029037476,
-      "rewards/rejected": -0.6173893809318542,
-      "step": 860
-    },
-    {
-      "epoch": 0.9,
-      "learning_rate": 3.8920780711825487e-07,
-      "logits/chosen": -2.333220958709717,
-      "logits/rejected": -2.36674165725708,
-      "logps/chosen": -270.8648376464844,
-      "logps/rejected": -226.4284210205078,
-      "loss": 0.5623,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.07031848281621933,
-      "rewards/margins": 0.5230459570884705,
-      "rewards/rejected": -0.5933644771575928,
-      "step": 870
-    },
-    {
-      "epoch": 0.91,
-      "learning_rate": 3.8729429774205893e-07,
-      "logits/chosen": -2.414712905883789,
-      "logits/rejected": -2.3959202766418457,
-      "logps/chosen": -265.5609436035156,
-      "logps/rejected": -221.8790283203125,
-      "loss": 0.5583,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.09441053122282028,
-      "rewards/margins": 0.4818418025970459,
-      "rewards/rejected": -0.5762523412704468,
-      "step": 880
-    },
-    {
-      "epoch": 0.92,
-      "learning_rate": 3.8538078836586294e-07,
-      "logits/chosen": -2.4260947704315186,
-      "logits/rejected": -2.3521246910095215,
-      "logps/chosen": -255.67459106445312,
-      "logps/rejected": -249.5232696533203,
-      "loss": 0.5798,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.11774387210607529,
-      "rewards/margins": 0.41275423765182495,
-      "rewards/rejected": -0.5304980874061584,
-      "step": 890
-    },
-    {
-      "epoch": 0.93,
-      "learning_rate": 3.83467278989667e-07,
-      "logits/chosen": -2.4081029891967773,
-      "logits/rejected": -2.356966495513916,
-      "logps/chosen": -282.0208740234375,
-      "logps/rejected": -222.43765258789062,
-      "loss": 0.5496,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.0056116655468940735,
-      "rewards/margins": 0.560874879360199,
-      "rewards/rejected": -0.5664864778518677,
-      "step": 900
-    },
-    {
-      "epoch": 0.94,
-      "learning_rate": 3.8155376961347106e-07,
-      "logits/chosen": -2.4205422401428223,
-      "logits/rejected": -2.4086201190948486,
-      "logps/chosen": -250.5408477783203,
-      "logps/rejected": -203.41635131835938,
-      "loss": 0.5658,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.08207114040851593,
-      "rewards/margins": 0.5275572538375854,
-      "rewards/rejected": -0.6096283197402954,
-      "step": 910
-    },
-    {
-      "epoch": 0.95,
-      "learning_rate": 3.796402602372751e-07,
-      "logits/chosen": -2.4266042709350586,
-      "logits/rejected": -2.3435492515563965,
-      "logps/chosen": -272.265380859375,
-      "logps/rejected": -252.53268432617188,
-      "loss": 0.5561,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.08728349208831787,
-      "rewards/margins": 0.43014270067214966,
-      "rewards/rejected": -0.5174261927604675,
-      "step": 920
-    },
-    {
-      "epoch": 0.96,
-      "learning_rate": 3.777267508610792e-07,
-      "logits/chosen": -2.4939112663269043,
-      "logits/rejected": -2.428483247756958,
-      "logps/chosen": -270.8354797363281,
-      "logps/rejected": -260.45611572265625,
-      "loss": 0.5662,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.1041080579161644,
-      "rewards/margins": 0.5785307884216309,
-      "rewards/rejected": -0.6826388239860535,
-      "step": 930
-    },
-    {
-      "epoch": 0.97,
-      "learning_rate": 3.7581324148488325e-07,
-      "logits/chosen": -2.4214115142822266,
-      "logits/rejected": -2.374481201171875,
-      "logps/chosen": -299.26385498046875,
-      "logps/rejected": -224.3602294921875,
-      "loss": 0.5676,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.09809824079275131,
-      "rewards/margins": 0.4741215705871582,
-      "rewards/rejected": -0.5722197890281677,
-      "step": 940
-    },
-    {
-      "epoch": 0.98,
-      "learning_rate": 3.738997321086873e-07,
-      "logits/chosen": -2.405039072036743,
-      "logits/rejected": -2.36025333404541,
-      "logps/chosen": -257.98358154296875,
-      "logps/rejected": -224.39584350585938,
-      "loss": 0.5705,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.055282533168792725,
-      "rewards/margins": 0.41767168045043945,
-      "rewards/rejected": -0.4729541838169098,
-      "step": 950
-    },
-    {
-      "epoch": 0.99,
-      "learning_rate": 3.7198622273249137e-07,
-      "logits/chosen": -2.4322896003723145,
-      "logits/rejected": -2.4136605262756348,
-      "logps/chosen": -262.254150390625,
-      "logps/rejected": -195.317626953125,
-      "loss": 0.5501,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.09157253801822662,
-      "rewards/margins": 0.4116409420967102,
-      "rewards/rejected": -0.5032135248184204,
-      "step": 960
-    },
-    {
-      "epoch": 1.0,
-      "eval_logits/chosen": -2.1115331649780273,
-      "eval_logits/rejected": -1.9901020526885986,
-      "eval_logps/chosen": -265.7538146972656,
-      "eval_logps/rejected": -225.6846923828125,
-      "eval_loss": 0.5559706091880798,
-      "eval_rewards/accuracies": 0.7200000286102295,
-      "eval_rewards/chosen": -0.10930197685956955,
-      "eval_rewards/margins": 0.5307366251945496,
-      "eval_rewards/rejected": -0.6400385499000549,
-      "eval_runtime": 454.9548,
-      "eval_samples_per_second": 4.396,
-      "eval_steps_per_second": 0.275,
-      "step": 968
-    },
-    {
-      "epoch": 1.0,
-      "learning_rate": 3.7007271335629544e-07,
-      "logits/chosen": -2.393294334411621,
-      "logits/rejected": -2.2951369285583496,
-      "logps/chosen": -265.35040283203125,
-      "logps/rejected": -216.5760955810547,
-      "loss": 0.5536,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.0699644461274147,
-      "rewards/margins": 0.6380785703659058,
-      "rewards/rejected": -0.7080430388450623,
-      "step": 970
-    },
-    {
-      "epoch": 1.01,
-      "learning_rate": 3.681592039800995e-07,
-      "logits/chosen": -2.384970188140869,
-      "logits/rejected": -2.355285167694092,
-      "logps/chosen": -242.12600708007812,
-      "logps/rejected": -217.994873046875,
-      "loss": 0.5518,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.10410074144601822,
-      "rewards/margins": 0.4534371495246887,
-      "rewards/rejected": -0.557537853717804,
-      "step": 980
-    },
-    {
-      "epoch": 1.02,
-      "learning_rate": 3.662456946039035e-07,
-      "logits/chosen": -2.3399007320404053,
-      "logits/rejected": -2.3183391094207764,
-      "logps/chosen": -231.69064331054688,
-      "logps/rejected": -201.68441772460938,
-      "loss": 0.5997,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.2032332420349121,
-      "rewards/margins": 0.3764384686946869,
-      "rewards/rejected": -0.5796716809272766,
-      "step": 990
-    },
-    {
-      "epoch": 1.03,
-      "learning_rate": 3.6433218522770757e-07,
-      "logits/chosen": -2.3833537101745605,
-      "logits/rejected": -2.319551944732666,
-      "logps/chosen": -295.35052490234375,
-      "logps/rejected": -220.7740936279297,
-      "loss": 0.5815,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.07751850038766861,
-      "rewards/margins": 0.4878837466239929,
-      "rewards/rejected": -0.5654021501541138,
-      "step": 1000
-    },
-    {
-      "epoch": 1.04,
-      "learning_rate": 3.6241867585151163e-07,
-      "logits/chosen": -2.449373245239258,
-      "logits/rejected": -2.387481927871704,
-      "logps/chosen": -248.5247802734375,
-      "logps/rejected": -225.5816192626953,
-      "loss": 0.5582,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.13464747369289398,
-      "rewards/margins": 0.4785802960395813,
-      "rewards/rejected": -0.6132277250289917,
-      "step": 1010
-    },
-    {
-      "epoch": 1.05,
-      "learning_rate": 3.605051664753157e-07,
-      "logits/chosen": -2.4245686531066895,
-      "logits/rejected": -2.4296786785125732,
-      "logps/chosen": -251.46206665039062,
-      "logps/rejected": -202.94284057617188,
-      "loss": 0.5488,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.10227508842945099,
-      "rewards/margins": 0.5419632792472839,
-      "rewards/rejected": -0.6442384123802185,
-      "step": 1020
-    },
-    {
-      "epoch": 1.06,
-      "learning_rate": 3.5859165709911975e-07,
-      "logits/chosen": -2.3132331371307373,
-      "logits/rejected": -2.3190786838531494,
-      "logps/chosen": -292.53125,
-      "logps/rejected": -256.5845947265625,
-      "loss": 0.5407,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.04681607708334923,
-      "rewards/margins": 0.629582941532135,
-      "rewards/rejected": -0.6763990521430969,
-      "step": 1030
-    },
-    {
-      "epoch": 1.07,
-      "learning_rate": 3.566781477229238e-07,
-      "logits/chosen": -2.36743426322937,
-      "logits/rejected": -2.2601940631866455,
-      "logps/chosen": -259.8556213378906,
-      "logps/rejected": -235.2071075439453,
-      "loss": 0.5544,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.0872168317437172,
-      "rewards/margins": 0.5171287059783936,
-      "rewards/rejected": -0.6043455600738525,
-      "step": 1040
-    },
-    {
-      "epoch": 1.08,
-      "learning_rate": 3.547646383467279e-07,
-      "logits/chosen": -2.326066732406616,
-      "logits/rejected": -2.3307995796203613,
-      "logps/chosen": -256.3676452636719,
-      "logps/rejected": -235.98672485351562,
-      "loss": 0.528,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.039658013731241226,
-      "rewards/margins": 0.6266075372695923,
-      "rewards/rejected": -0.6662655472755432,
-      "step": 1050
-    },
-    {
-      "epoch": 1.09,
-      "learning_rate": 3.5285112897053194e-07,
-      "logits/chosen": -2.4012277126312256,
-      "logits/rejected": -2.287254810333252,
-      "logps/chosen": -275.3678894042969,
-      "logps/rejected": -235.6710662841797,
-      "loss": 0.5394,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.1649123877286911,
-      "rewards/margins": 0.7132270932197571,
-      "rewards/rejected": -0.8781394958496094,
-      "step": 1060
-    },
-    {
-      "epoch": 1.11,
-      "learning_rate": 3.50937619594336e-07,
-      "logits/chosen": -2.4277257919311523,
-      "logits/rejected": -2.3757331371307373,
-      "logps/chosen": -291.19110107421875,
-      "logps/rejected": -261.84320068359375,
-      "loss": 0.5486,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.08061722666025162,
-      "rewards/margins": 0.646703839302063,
-      "rewards/rejected": -0.7273210883140564,
-      "step": 1070
-    },
-    {
-      "epoch": 1.12,
-      "learning_rate": 3.4902411021814007e-07,
-      "logits/chosen": -2.4494857788085938,
-      "logits/rejected": -2.3594818115234375,
-      "logps/chosen": -294.79901123046875,
-      "logps/rejected": -229.1653594970703,
-      "loss": 0.5273,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.0071506015956401825,
-      "rewards/margins": 0.7936251759529114,
-      "rewards/rejected": -0.8007757067680359,
-      "step": 1080
-    },
-    {
-      "epoch": 1.13,
-      "learning_rate": 3.4711060084194413e-07,
-      "logits/chosen": -2.3939168453216553,
-      "logits/rejected": -2.403517961502075,
-      "logps/chosen": -272.8337707519531,
-      "logps/rejected": -228.14816284179688,
-      "loss": 0.569,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.05292476341128349,
-      "rewards/margins": 0.5845643281936646,
-      "rewards/rejected": -0.6374891996383667,
-      "step": 1090
-    },
-    {
-      "epoch": 1.14,
-      "learning_rate": 3.4519709146574814e-07,
-      "logits/chosen": -2.4750289916992188,
-      "logits/rejected": -2.3406853675842285,
-      "logps/chosen": -267.2197265625,
-      "logps/rejected": -247.76498413085938,
-      "loss": 0.5588,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.07975287735462189,
-      "rewards/margins": 0.5557757019996643,
-      "rewards/rejected": -0.6355286240577698,
-      "step": 1100
-    },
-    {
-      "epoch": 1.15,
-      "learning_rate": 3.432835820895522e-07,
-      "logits/chosen": -2.4085628986358643,
-      "logits/rejected": -2.3299522399902344,
-      "logps/chosen": -250.01791381835938,
-      "logps/rejected": -230.2222900390625,
-      "loss": 0.5427,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.09803862869739532,
-      "rewards/margins": 0.5516743659973145,
-      "rewards/rejected": -0.6497129797935486,
-      "step": 1110
-    },
-    {
-      "epoch": 1.16,
-      "learning_rate": 3.4137007271335626e-07,
-      "logits/chosen": -2.349724054336548,
-      "logits/rejected": -2.396789073944092,
-      "logps/chosen": -218.85134887695312,
-      "logps/rejected": -212.79843139648438,
-      "loss": 0.5405,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.17577314376831055,
-      "rewards/margins": 0.4191734790802002,
-      "rewards/rejected": -0.5949466228485107,
-      "step": 1120
-    },
-    {
-      "epoch": 1.17,
-      "learning_rate": 3.394565633371603e-07,
-      "logits/chosen": -2.3591291904449463,
-      "logits/rejected": -2.2995028495788574,
-      "logps/chosen": -293.353271484375,
-      "logps/rejected": -245.3329315185547,
-      "loss": 0.5378,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.1506257951259613,
-      "rewards/margins": 0.5927566289901733,
-      "rewards/rejected": -0.7433823347091675,
-      "step": 1130
-    },
-    {
-      "epoch": 1.18,
-      "learning_rate": 3.375430539609644e-07,
-      "logits/chosen": -2.281580924987793,
-      "logits/rejected": -2.3465418815612793,
-      "logps/chosen": -239.721923828125,
-      "logps/rejected": -211.04336547851562,
-      "loss": 0.5148,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.13178768754005432,
-      "rewards/margins": 0.6117417812347412,
-      "rewards/rejected": -0.7435294985771179,
-      "step": 1140
-    },
-    {
-      "epoch": 1.19,
-      "learning_rate": 3.3562954458476845e-07,
-      "logits/chosen": -2.4424147605895996,
-      "logits/rejected": -2.428922653198242,
-      "logps/chosen": -306.62567138671875,
-      "logps/rejected": -233.6839141845703,
-      "loss": 0.5608,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.14439252018928528,
-      "rewards/margins": 0.5881448984146118,
-      "rewards/rejected": -0.7325373888015747,
-      "step": 1150
-    },
-    {
-      "epoch": 1.2,
-      "learning_rate": 3.337160352085725e-07,
-      "logits/chosen": -2.4268736839294434,
-      "logits/rejected": -2.385798692703247,
-      "logps/chosen": -298.24273681640625,
-      "logps/rejected": -241.7369842529297,
-      "loss": 0.535,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.06826777756214142,
-      "rewards/margins": 0.7597671747207642,
-      "rewards/rejected": -0.8280348777770996,
-      "step": 1160
-    },
-    {
-      "epoch": 1.21,
-      "learning_rate": 3.3180252583237657e-07,
-      "logits/chosen": -2.3368301391601562,
-      "logits/rejected": -2.2904515266418457,
-      "logps/chosen": -277.94744873046875,
-      "logps/rejected": -231.4571533203125,
-      "loss": 0.5509,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.057608962059020996,
-      "rewards/margins": 0.6012957692146301,
-      "rewards/rejected": -0.6589046716690063,
-      "step": 1170
-    },
-    {
-      "epoch": 1.22,
-      "learning_rate": 3.2988901645618063e-07,
-      "logits/chosen": -2.3686211109161377,
-      "logits/rejected": -2.278508424758911,
-      "logps/chosen": -261.26556396484375,
-      "logps/rejected": -225.91049194335938,
-      "loss": 0.5406,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.1395711600780487,
-      "rewards/margins": 0.5758097171783447,
-      "rewards/rejected": -0.715380847454071,
-      "step": 1180
-    },
-    {
-      "epoch": 1.23,
-      "learning_rate": 3.279755070799847e-07,
-      "logits/chosen": -2.402057647705078,
-      "logits/rejected": -2.310411214828491,
-      "logps/chosen": -259.6847229003906,
-      "logps/rejected": -257.0271301269531,
-      "loss": 0.5279,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.1509382575750351,
-      "rewards/margins": 0.5303701162338257,
-      "rewards/rejected": -0.681308388710022,
-      "step": 1190
-    },
-    {
-      "epoch": 1.24,
-      "learning_rate": 3.260619977037887e-07,
-      "logits/chosen": -2.397024154663086,
-      "logits/rejected": -2.3406670093536377,
-      "logps/chosen": -251.27926635742188,
-      "logps/rejected": -214.0767822265625,
-      "loss": 0.5191,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.07334662973880768,
-      "rewards/margins": 0.7737401723861694,
-      "rewards/rejected": -0.8470869064331055,
-      "step": 1200
-    },
-    {
-      "epoch": 1.25,
-      "learning_rate": 3.2414848832759277e-07,
-      "logits/chosen": -2.42278790473938,
-      "logits/rejected": -2.2710700035095215,
-      "logps/chosen": -261.6346435546875,
-      "logps/rejected": -227.0576171875,
-      "loss": 0.5345,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.13957397639751434,
-      "rewards/margins": 0.585122287273407,
-      "rewards/rejected": -0.7246962785720825,
-      "step": 1210
-    },
-    {
-      "epoch": 1.26,
-      "learning_rate": 3.2223497895139683e-07,
-      "logits/chosen": -2.3876070976257324,
-      "logits/rejected": -2.3234522342681885,
-      "logps/chosen": -263.7557678222656,
-      "logps/rejected": -219.36367797851562,
-      "loss": 0.5538,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.12251804769039154,
-      "rewards/margins": 0.5337706804275513,
-      "rewards/rejected": -0.656288743019104,
-      "step": 1220
-    },
-    {
-      "epoch": 1.27,
-      "learning_rate": 3.203214695752009e-07,
-      "logits/chosen": -2.4647679328918457,
-      "logits/rejected": -2.385979175567627,
-      "logps/chosen": -262.8483581542969,
-      "logps/rejected": -230.214111328125,
-      "loss": 0.5692,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.15525886416435242,
-      "rewards/margins": 0.5802783370018005,
-      "rewards/rejected": -0.7355371713638306,
-      "step": 1230
-    },
-    {
-      "epoch": 1.28,
-      "learning_rate": 3.1840796019900495e-07,
-      "logits/chosen": -2.4262988567352295,
-      "logits/rejected": -2.3878419399261475,
-      "logps/chosen": -262.4114074707031,
-      "logps/rejected": -255.02749633789062,
-      "loss": 0.5478,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.11712455749511719,
-      "rewards/margins": 0.6756891012191772,
-      "rewards/rejected": -0.7928137183189392,
-      "step": 1240
-    },
-    {
-      "epoch": 1.29,
-      "learning_rate": 3.16494450822809e-07,
-      "logits/chosen": -2.35296893119812,
-      "logits/rejected": -2.3551628589630127,
-      "logps/chosen": -236.53445434570312,
-      "logps/rejected": -233.4728546142578,
-      "loss": 0.5355,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.18067090213298798,
-      "rewards/margins": 0.4853228032588959,
-      "rewards/rejected": -0.6659936308860779,
-      "step": 1250
-    },
-    {
-      "epoch": 1.3,
-      "learning_rate": 3.145809414466131e-07,
-      "logits/chosen": -2.48589825630188,
-      "logits/rejected": -2.478086233139038,
-      "logps/chosen": -272.775634765625,
-      "logps/rejected": -266.91058349609375,
-      "loss": 0.5399,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.14183524250984192,
-      "rewards/margins": 0.4844776690006256,
-      "rewards/rejected": -0.6263129711151123,
-      "step": 1260
-    },
-    {
-      "epoch": 1.31,
-      "learning_rate": 3.1266743207041714e-07,
-      "logits/chosen": -2.406167507171631,
-      "logits/rejected": -2.4039671421051025,
-      "logps/chosen": -247.5933074951172,
-      "logps/rejected": -226.58450317382812,
-      "loss": 0.5439,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.1417725682258606,
-      "rewards/margins": 0.6144998073577881,
-      "rewards/rejected": -0.7562723755836487,
-      "step": 1270
-    },
-    {
-      "epoch": 1.32,
-      "learning_rate": 3.107539226942212e-07,
-      "logits/chosen": -2.392794132232666,
-      "logits/rejected": -2.370959758758545,
-      "logps/chosen": -259.82183837890625,
-      "logps/rejected": -240.86178588867188,
-      "loss": 0.5788,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.14074555039405823,
-      "rewards/margins": 0.5021336674690247,
-      "rewards/rejected": -0.6428791880607605,
-      "step": 1280
-    },
-    {
-      "epoch": 1.33,
-      "learning_rate": 3.0884041331802526e-07,
-      "logits/chosen": -2.3568851947784424,
-      "logits/rejected": -2.2625174522399902,
-      "logps/chosen": -243.117431640625,
-      "logps/rejected": -216.640380859375,
-      "loss": 0.5542,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.17768898606300354,
-      "rewards/margins": 0.5181851983070374,
-      "rewards/rejected": -0.6958742141723633,
-      "step": 1290
-    },
-    {
-      "epoch": 1.34,
-      "learning_rate": 3.0692690394182927e-07,
-      "logits/chosen": -2.436131477355957,
-      "logits/rejected": -2.396608829498291,
-      "logps/chosen": -277.2726135253906,
-      "logps/rejected": -233.1066131591797,
-      "loss": 0.5558,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.06510978192090988,
-      "rewards/margins": 0.6768725514411926,
-      "rewards/rejected": -0.7419822812080383,
-      "step": 1300
-    },
-    {
-      "epoch": 1.35,
-      "learning_rate": 3.0501339456563334e-07,
-      "logits/chosen": -2.4156711101531982,
-      "logits/rejected": -2.271003007888794,
-      "logps/chosen": -269.3404541015625,
-      "logps/rejected": -243.42245483398438,
-      "loss": 0.534,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.16720613837242126,
-      "rewards/margins": 0.5710007548332214,
-      "rewards/rejected": -0.7382069230079651,
-      "step": 1310
-    },
-    {
-      "epoch": 1.36,
-      "learning_rate": 3.030998851894374e-07,
-      "logits/chosen": -2.352940082550049,
-      "logits/rejected": -2.363243579864502,
-      "logps/chosen": -229.7804718017578,
-      "logps/rejected": -221.0922393798828,
-      "loss": 0.5117,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.08244588226079941,
-      "rewards/margins": 0.668733537197113,
-      "rewards/rejected": -0.7511794567108154,
-      "step": 1320
-    },
-    {
-      "epoch": 1.37,
-      "learning_rate": 3.0118637581324146e-07,
-      "logits/chosen": -2.4218077659606934,
-      "logits/rejected": -2.36722469329834,
-      "logps/chosen": -257.25872802734375,
-      "logps/rejected": -223.78256225585938,
-      "loss": 0.539,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.08728824555873871,
-      "rewards/margins": 0.7230480909347534,
-      "rewards/rejected": -0.8103362917900085,
-      "step": 1330
-    },
-    {
-      "epoch": 1.38,
-      "learning_rate": 2.992728664370455e-07,
-      "logits/chosen": -2.4494032859802246,
-      "logits/rejected": -2.388096332550049,
-      "logps/chosen": -239.74057006835938,
-      "logps/rejected": -225.5271453857422,
-      "loss": 0.5603,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.21483702957630157,
-      "rewards/margins": 0.5343385338783264,
-      "rewards/rejected": -0.7491755485534668,
-      "step": 1340
-    },
-    {
-      "epoch": 1.39,
-      "learning_rate": 2.973593570608496e-07,
-      "logits/chosen": -2.3430349826812744,
-      "logits/rejected": -2.262885808944702,
-      "logps/chosen": -261.22149658203125,
-      "logps/rejected": -220.1973419189453,
-      "loss": 0.5515,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.11469510942697525,
-      "rewards/margins": 0.6628144979476929,
-      "rewards/rejected": -0.7775096297264099,
-      "step": 1350
-    },
-    {
-      "epoch": 1.4,
-      "learning_rate": 2.9544584768465365e-07,
-      "logits/chosen": -2.415313243865967,
-      "logits/rejected": -2.369680881500244,
-      "logps/chosen": -310.0767517089844,
-      "logps/rejected": -254.2592315673828,
-      "loss": 0.5553,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.1948031485080719,
-      "rewards/margins": 0.6389765739440918,
-      "rewards/rejected": -0.8337796926498413,
-      "step": 1360
-    },
-    {
-      "epoch": 1.41,
-      "learning_rate": 2.935323383084577e-07,
-      "logits/chosen": -2.3823018074035645,
-      "logits/rejected": -2.3315823078155518,
-      "logps/chosen": -290.1065368652344,
-      "logps/rejected": -242.30081176757812,
-      "loss": 0.5604,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.09571398794651031,
-      "rewards/margins": 0.5208708643913269,
-      "rewards/rejected": -0.6165848970413208,
-      "step": 1370
-    },
-    {
-      "epoch": 1.43,
-      "learning_rate": 2.9161882893226177e-07,
-      "logits/chosen": -2.3578696250915527,
-      "logits/rejected": -2.2521920204162598,
-      "logps/chosen": -240.7886962890625,
-      "logps/rejected": -204.34652709960938,
-      "loss": 0.544,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.20477895438671112,
-      "rewards/margins": 0.4378456473350525,
-      "rewards/rejected": -0.6426246166229248,
-      "step": 1380
-    },
-    {
-      "epoch": 1.44,
-      "learning_rate": 2.8970531955606583e-07,
-      "logits/chosen": -2.453507900238037,
-      "logits/rejected": -2.3540263175964355,
-      "logps/chosen": -270.1062927246094,
-      "logps/rejected": -231.31466674804688,
-      "loss": 0.5674,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.16483107209205627,
-      "rewards/margins": 0.64717036485672,
-      "rewards/rejected": -0.8120015263557434,
-      "step": 1390
-    },
-    {
-      "epoch": 1.45,
-      "learning_rate": 2.8779181017986984e-07,
-      "logits/chosen": -2.424034595489502,
-      "logits/rejected": -2.4023849964141846,
-      "logps/chosen": -269.05780029296875,
-      "logps/rejected": -239.2170867919922,
-      "loss": 0.5546,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.22925500571727753,
-      "rewards/margins": 0.5416269898414612,
-      "rewards/rejected": -0.7708818912506104,
-      "step": 1400
-    },
-    {
-      "epoch": 1.46,
-      "learning_rate": 2.858783008036739e-07,
-      "logits/chosen": -2.3142106533050537,
-      "logits/rejected": -2.261690616607666,
-      "logps/chosen": -278.53643798828125,
-      "logps/rejected": -220.97555541992188,
-      "loss": 0.5294,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.13496367633342743,
-      "rewards/margins": 0.6569581031799316,
-      "rewards/rejected": -0.7919217944145203,
-      "step": 1410
-    },
-    {
-      "epoch": 1.47,
-      "learning_rate": 2.8396479142747797e-07,
-      "logits/chosen": -2.3985562324523926,
-      "logits/rejected": -2.368497848510742,
-      "logps/chosen": -230.22122192382812,
-      "logps/rejected": -201.47998046875,
-      "loss": 0.5405,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.1572544276714325,
-      "rewards/margins": 0.6025274991989136,
-      "rewards/rejected": -0.7597819566726685,
-      "step": 1420
-    },
-    {
-      "epoch": 1.48,
-      "learning_rate": 2.8205128205128203e-07,
-      "logits/chosen": -2.3532938957214355,
-      "logits/rejected": -2.30509352684021,
-      "logps/chosen": -260.51312255859375,
-      "logps/rejected": -229.70376586914062,
-      "loss": 0.5412,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.2625270187854767,
-      "rewards/margins": 0.5138301253318787,
-      "rewards/rejected": -0.776357114315033,
-      "step": 1430
-    },
-    {
-      "epoch": 1.49,
-      "learning_rate": 2.801377726750861e-07,
-      "logits/chosen": -2.3864636421203613,
-      "logits/rejected": -2.316066265106201,
-      "logps/chosen": -256.13763427734375,
-      "logps/rejected": -217.63235473632812,
-      "loss": 0.5431,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.1825113743543625,
-      "rewards/margins": 0.49288827180862427,
-      "rewards/rejected": -0.675399661064148,
-      "step": 1440
-    },
-    {
-      "epoch": 1.5,
-      "learning_rate": 2.7822426329889015e-07,
-      "logits/chosen": -2.428596019744873,
-      "logits/rejected": -2.3228797912597656,
-      "logps/chosen": -294.236328125,
-      "logps/rejected": -235.548095703125,
-      "loss": 0.5636,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.1488821804523468,
-      "rewards/margins": 0.5950523018836975,
-      "rewards/rejected": -0.7439345121383667,
-      "step": 1450
-    },
-    {
-      "epoch": 1.51,
-      "learning_rate": 2.763107539226942e-07,
-      "logits/chosen": -2.380725145339966,
-      "logits/rejected": -2.3312487602233887,
-      "logps/chosen": -280.84490966796875,
-      "logps/rejected": -242.58340454101562,
-      "loss": 0.5586,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.13354410231113434,
-      "rewards/margins": 0.6448094844818115,
-      "rewards/rejected": -0.7783535718917847,
-      "step": 1460
-    },
-    {
-      "epoch": 1.52,
-      "learning_rate": 2.743972445464983e-07,
-      "logits/chosen": -2.4229865074157715,
-      "logits/rejected": -2.410762071609497,
-      "logps/chosen": -254.83706665039062,
-      "logps/rejected": -232.5323486328125,
-      "loss": 0.5598,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.1900000274181366,
-      "rewards/margins": 0.4888847768306732,
-      "rewards/rejected": -0.678884744644165,
-      "step": 1470
-    },
-    {
-      "epoch": 1.53,
-      "learning_rate": 2.7248373517030234e-07,
-      "logits/chosen": -2.475088119506836,
-      "logits/rejected": -2.4195237159729004,
-      "logps/chosen": -274.3759765625,
-      "logps/rejected": -237.31689453125,
-      "loss": 0.5401,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.16161084175109863,
-      "rewards/margins": 0.6231001615524292,
-      "rewards/rejected": -0.7847110629081726,
-      "step": 1480
-    },
-    {
-      "epoch": 1.54,
-      "learning_rate": 2.705702257941064e-07,
-      "logits/chosen": -2.432378053665161,
-      "logits/rejected": -2.4352517127990723,
-      "logps/chosen": -311.6330261230469,
-      "logps/rejected": -246.75796508789062,
-      "loss": 0.5356,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.16773328185081482,
-      "rewards/margins": 0.6822336912155151,
-      "rewards/rejected": -0.8499670028686523,
-      "step": 1490
-    },
-    {
-      "epoch": 1.55,
-      "learning_rate": 2.686567164179104e-07,
-      "logits/chosen": -2.348602771759033,
-      "logits/rejected": -2.275367259979248,
-      "logps/chosen": -259.00390625,
-      "logps/rejected": -219.0964813232422,
-      "loss": 0.5445,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.16603624820709229,
-      "rewards/margins": 0.6049584150314331,
-      "rewards/rejected": -0.7709946632385254,
-      "step": 1500
-    },
-    {
-      "epoch": 1.56,
-      "learning_rate": 2.6674320704171447e-07,
-      "logits/chosen": -2.416759729385376,
-      "logits/rejected": -2.3592278957366943,
-      "logps/chosen": -259.57403564453125,
-      "logps/rejected": -206.39962768554688,
-      "loss": 0.5209,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.10807810723781586,
-      "rewards/margins": 0.7587023973464966,
-      "rewards/rejected": -0.8667804598808289,
-      "step": 1510
-    },
-    {
-      "epoch": 1.57,
-      "learning_rate": 2.6482969766551853e-07,
-      "logits/chosen": -2.4495720863342285,
-      "logits/rejected": -2.447631359100342,
-      "logps/chosen": -274.3368225097656,
-      "logps/rejected": -237.6719970703125,
-      "loss": 0.5443,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.08986170589923859,
-      "rewards/margins": 0.7171138525009155,
-      "rewards/rejected": -0.8069756627082825,
-      "step": 1520
-    },
-    {
-      "epoch": 1.58,
-      "learning_rate": 2.629161882893226e-07,
-      "logits/chosen": -2.4459540843963623,
-      "logits/rejected": -2.371903419494629,
-      "logps/chosen": -280.4138488769531,
-      "logps/rejected": -214.16226196289062,
-      "loss": 0.5159,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.15295711159706116,
-      "rewards/margins": 0.7536977529525757,
-      "rewards/rejected": -0.906654953956604,
-      "step": 1530
-    },
-    {
-      "epoch": 1.59,
-      "learning_rate": 2.6100267891312666e-07,
-      "logits/chosen": -2.3349404335021973,
-      "logits/rejected": -2.303260087966919,
-      "logps/chosen": -269.13787841796875,
-      "logps/rejected": -208.63107299804688,
-      "loss": 0.5546,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.12116660922765732,
-      "rewards/margins": 0.6888124942779541,
-      "rewards/rejected": -0.8099790811538696,
-      "step": 1540
-    },
-    {
-      "epoch": 1.6,
-      "learning_rate": 2.590891695369307e-07,
-      "logits/chosen": -2.450225830078125,
-      "logits/rejected": -2.361754894256592,
-      "logps/chosen": -256.0200500488281,
-      "logps/rejected": -210.36026000976562,
-      "loss": 0.529,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.1600186824798584,
-      "rewards/margins": 0.5653270483016968,
-      "rewards/rejected": -0.7253457903862,
-      "step": 1550
-    },
-    {
-      "epoch": 1.61,
-      "learning_rate": 2.571756601607348e-07,
-      "logits/chosen": -2.4175140857696533,
-      "logits/rejected": -2.3866875171661377,
-      "logps/chosen": -294.81744384765625,
-      "logps/rejected": -217.4511260986328,
-      "loss": 0.5455,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.18921521306037903,
-      "rewards/margins": 0.6538978815078735,
-      "rewards/rejected": -0.843113124370575,
-      "step": 1560
-    },
-    {
-      "epoch": 1.62,
-      "learning_rate": 2.5526215078453884e-07,
-      "logits/chosen": -2.4297268390655518,
-      "logits/rejected": -2.3942902088165283,
-      "logps/chosen": -301.64642333984375,
-      "logps/rejected": -237.7666015625,
-      "loss": 0.5349,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.10356118530035019,
-      "rewards/margins": 0.7558127045631409,
-      "rewards/rejected": -0.859373927116394,
-      "step": 1570
-    },
-    {
-      "epoch": 1.63,
-      "learning_rate": 2.533486414083429e-07,
-      "logits/chosen": -2.4258458614349365,
-      "logits/rejected": -2.3717398643493652,
-      "logps/chosen": -306.97442626953125,
-      "logps/rejected": -249.22109985351562,
-      "loss": 0.5161,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.07514607906341553,
-      "rewards/margins": 0.7315629720687866,
-      "rewards/rejected": -0.8067091107368469,
-      "step": 1580
-    },
-    {
-      "epoch": 1.64,
-      "learning_rate": 2.5143513203214697e-07,
-      "logits/chosen": -2.3663864135742188,
-      "logits/rejected": -2.326347827911377,
-      "logps/chosen": -275.7830505371094,
-      "logps/rejected": -225.2949676513672,
-      "loss": 0.5606,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.20903043448925018,
-      "rewards/margins": 0.5634015202522278,
-      "rewards/rejected": -0.7724319696426392,
-      "step": 1590
-    },
-    {
-      "epoch": 1.65,
-      "learning_rate": 2.49521622655951e-07,
-      "logits/chosen": -2.492893934249878,
-      "logits/rejected": -2.3561127185821533,
-      "logps/chosen": -293.12384033203125,
-      "logps/rejected": -262.7926940917969,
-      "loss": 0.553,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.04525697976350784,
-      "rewards/margins": 0.8012422323226929,
-      "rewards/rejected": -0.8464992642402649,
-      "step": 1600
-    },
-    {
-      "epoch": 1.66,
-      "learning_rate": 2.4760811327975504e-07,
-      "logits/chosen": -2.4678902626037598,
-      "logits/rejected": -2.3764843940734863,
-      "logps/chosen": -273.65106201171875,
-      "logps/rejected": -253.1365966796875,
-      "loss": 0.5352,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.13813751935958862,
-      "rewards/margins": 0.5831281542778015,
-      "rewards/rejected": -0.7212656736373901,
-      "step": 1610
-    },
-    {
-      "epoch": 1.67,
-      "learning_rate": 2.456946039035591e-07,
-      "logits/chosen": -2.466249942779541,
-      "logits/rejected": -2.436295747756958,
-      "logps/chosen": -290.6951599121094,
-      "logps/rejected": -257.230712890625,
-      "loss": 0.5668,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.10182980448007584,
-      "rewards/margins": 0.7511130571365356,
-      "rewards/rejected": -0.852942943572998,
-      "step": 1620
-    },
-    {
-      "epoch": 1.68,
-      "learning_rate": 2.4378109452736316e-07,
-      "logits/chosen": -2.3190982341766357,
-      "logits/rejected": -2.3256137371063232,
-      "logps/chosen": -272.5343322753906,
-      "logps/rejected": -198.2899932861328,
-      "loss": 0.5291,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.10047291219234467,
-      "rewards/margins": 0.7110286951065063,
-      "rewards/rejected": -0.8115016222000122,
-      "step": 1630
-    },
-    {
-      "epoch": 1.69,
-      "learning_rate": 2.418675851511672e-07,
-      "logits/chosen": -2.3069424629211426,
-      "logits/rejected": -2.312654495239258,
-      "logps/chosen": -281.74322509765625,
-      "logps/rejected": -221.7678680419922,
-      "loss": 0.5624,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.17993247509002686,
-      "rewards/margins": 0.6608012318611145,
-      "rewards/rejected": -0.8407337069511414,
-      "step": 1640
-    },
-    {
-      "epoch": 1.7,
-      "learning_rate": 2.399540757749713e-07,
-      "logits/chosen": -2.372399091720581,
-      "logits/rejected": -2.2994461059570312,
-      "logps/chosen": -250.9709014892578,
-      "logps/rejected": -216.40042114257812,
-      "loss": 0.5261,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.07177358865737915,
-      "rewards/margins": 0.9095686078071594,
-      "rewards/rejected": -0.9813421964645386,
-      "step": 1650
-    },
-    {
-      "epoch": 1.71,
-      "learning_rate": 2.3804056639877535e-07,
-      "logits/chosen": -2.333723545074463,
-      "logits/rejected": -2.250431537628174,
-      "logps/chosen": -269.04998779296875,
-      "logps/rejected": -230.35256958007812,
-      "loss": 0.5566,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.1327822506427765,
-      "rewards/margins": 0.7238941192626953,
-      "rewards/rejected": -0.8566763997077942,
-      "step": 1660
-    },
-    {
-      "epoch": 1.72,
-      "learning_rate": 2.361270570225794e-07,
-      "logits/chosen": -2.3128609657287598,
-      "logits/rejected": -2.2538418769836426,
-      "logps/chosen": -263.09417724609375,
-      "logps/rejected": -223.70974731445312,
-      "loss": 0.5049,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.11414393037557602,
-      "rewards/margins": 0.789380669593811,
-      "rewards/rejected": -0.9035245776176453,
-      "step": 1670
-    },
-    {
-      "epoch": 1.74,
-      "learning_rate": 2.3421354764638345e-07,
-      "logits/chosen": -2.4389374256134033,
-      "logits/rejected": -2.3650295734405518,
-      "logps/chosen": -296.4449462890625,
-      "logps/rejected": -244.5340576171875,
-      "loss": 0.554,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.16256913542747498,
-      "rewards/margins": 0.6594110727310181,
-      "rewards/rejected": -0.8219801783561707,
-      "step": 1680
-    },
-    {
-      "epoch": 1.75,
-      "learning_rate": 2.323000382701875e-07,
-      "logits/chosen": -2.38859224319458,
-      "logits/rejected": -2.275116443634033,
-      "logps/chosen": -289.535400390625,
-      "logps/rejected": -226.6898956298828,
-      "loss": 0.5174,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.11396412551403046,
-      "rewards/margins": 0.6983034014701843,
-      "rewards/rejected": -0.8122674822807312,
-      "step": 1690
-    },
-    {
-      "epoch": 1.76,
-      "learning_rate": 2.3038652889399157e-07,
-      "logits/chosen": -2.3573622703552246,
-      "logits/rejected": -2.368932008743286,
-      "logps/chosen": -259.396240234375,
-      "logps/rejected": -235.6220245361328,
-      "loss": 0.5219,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.0638931542634964,
-      "rewards/margins": 0.7549349069595337,
-      "rewards/rejected": -0.8188279867172241,
-      "step": 1700
-    },
-    {
-      "epoch": 1.77,
-      "learning_rate": 2.2847301951779563e-07,
-      "logits/chosen": -2.3194305896759033,
-      "logits/rejected": -2.271104574203491,
-      "logps/chosen": -230.3657989501953,
-      "logps/rejected": -218.594970703125,
-      "loss": 0.5364,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.23464179039001465,
-      "rewards/margins": 0.5825188755989075,
-      "rewards/rejected": -0.8171606063842773,
-      "step": 1710
-    },
-    {
-      "epoch": 1.78,
-      "learning_rate": 2.265595101415997e-07,
-      "logits/chosen": -2.3034956455230713,
-      "logits/rejected": -2.26176118850708,
-      "logps/chosen": -263.59234619140625,
-      "logps/rejected": -207.0500946044922,
-      "loss": 0.5405,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.1522129625082016,
-      "rewards/margins": 0.8050564527511597,
-      "rewards/rejected": -0.957269549369812,
-      "step": 1720
-    },
-    {
-      "epoch": 1.79,
-      "learning_rate": 2.2464600076540373e-07,
-      "logits/chosen": -2.3417623043060303,
-      "logits/rejected": -2.3644707202911377,
-      "logps/chosen": -272.0294494628906,
-      "logps/rejected": -233.199951171875,
-      "loss": 0.5376,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.14386221766471863,
-      "rewards/margins": 0.7099758386611938,
-      "rewards/rejected": -0.8538379669189453,
-      "step": 1730
-    },
-    {
-      "epoch": 1.8,
-      "learning_rate": 2.227324913892078e-07,
-      "logits/chosen": -2.4512619972229004,
-      "logits/rejected": -2.333153247833252,
-      "logps/chosen": -262.7388610839844,
-      "logps/rejected": -233.94564819335938,
-      "loss": 0.5274,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.09474427998065948,
-      "rewards/margins": 0.678414523601532,
-      "rewards/rejected": -0.773158848285675,
-      "step": 1740
-    },
-    {
-      "epoch": 1.81,
-      "learning_rate": 2.2081898201301186e-07,
-      "logits/chosen": -2.400722026824951,
-      "logits/rejected": -2.328423023223877,
-      "logps/chosen": -251.57211303710938,
-      "logps/rejected": -219.1768035888672,
-      "loss": 0.5368,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.13836149871349335,
-      "rewards/margins": 0.6494232416152954,
-      "rewards/rejected": -0.78778475522995,
-      "step": 1750
-    },
-    {
-      "epoch": 1.82,
-      "learning_rate": 2.1890547263681592e-07,
-      "logits/chosen": -2.4296817779541016,
-      "logits/rejected": -2.367189407348633,
-      "logps/chosen": -271.3897399902344,
-      "logps/rejected": -215.4512481689453,
-      "loss": 0.5408,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.08417050540447235,
-      "rewards/margins": 0.754088282585144,
-      "rewards/rejected": -0.8382587432861328,
-      "step": 1760
-    },
-    {
-      "epoch": 1.83,
-      "learning_rate": 2.1699196326061998e-07,
-      "logits/chosen": -2.3716368675231934,
-      "logits/rejected": -2.3670201301574707,
-      "logps/chosen": -265.5472106933594,
-      "logps/rejected": -241.4296875,
-      "loss": 0.5596,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.14254868030548096,
-      "rewards/margins": 0.701485276222229,
-      "rewards/rejected": -0.8440340161323547,
-      "step": 1770
-    },
-    {
-      "epoch": 1.84,
-      "learning_rate": 2.1507845388442402e-07,
-      "logits/chosen": -2.396054744720459,
-      "logits/rejected": -2.285252332687378,
-      "logps/chosen": -287.78558349609375,
-      "logps/rejected": -233.22732543945312,
-      "loss": 0.5295,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.057570360600948334,
-      "rewards/margins": 0.7760987281799316,
-      "rewards/rejected": -0.8336690664291382,
-      "step": 1780
-    },
-    {
-      "epoch": 1.85,
-      "learning_rate": 2.1316494450822808e-07,
-      "logits/chosen": -2.3319549560546875,
-      "logits/rejected": -2.3298840522766113,
-      "logps/chosen": -269.1609191894531,
-      "logps/rejected": -242.9523468017578,
-      "loss": 0.5454,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.12093748152256012,
-      "rewards/margins": 0.7203799486160278,
-      "rewards/rejected": -0.8413174748420715,
-      "step": 1790
-    },
-    {
-      "epoch": 1.86,
-      "learning_rate": 2.1125143513203214e-07,
-      "logits/chosen": -2.4473745822906494,
-      "logits/rejected": -2.4215996265411377,
-      "logps/chosen": -266.4071960449219,
-      "logps/rejected": -224.84756469726562,
-      "loss": 0.5133,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.14717252552509308,
-      "rewards/margins": 0.730695366859436,
-      "rewards/rejected": -0.8778678774833679,
-      "step": 1800
-    },
-    {
-      "epoch": 1.87,
-      "learning_rate": 2.093379257558362e-07,
-      "logits/chosen": -2.426567316055298,
-      "logits/rejected": -2.4301810264587402,
-      "logps/chosen": -274.27569580078125,
-      "logps/rejected": -237.6707305908203,
-      "loss": 0.5553,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.19985240697860718,
-      "rewards/margins": 0.6345885992050171,
-      "rewards/rejected": -0.8344410061836243,
-      "step": 1810
-    },
-    {
-      "epoch": 1.88,
-      "learning_rate": 2.0742441637964026e-07,
-      "logits/chosen": -2.32918119430542,
-      "logits/rejected": -2.315727710723877,
-      "logps/chosen": -299.13201904296875,
-      "logps/rejected": -238.65219116210938,
-      "loss": 0.5101,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.08389053493738174,
-      "rewards/margins": 0.9151546359062195,
-      "rewards/rejected": -0.999045193195343,
-      "step": 1820
-    },
-    {
-      "epoch": 1.89,
-      "learning_rate": 2.055109070034443e-07,
-      "logits/chosen": -2.4138271808624268,
-      "logits/rejected": -2.395538091659546,
-      "logps/chosen": -287.26788330078125,
-      "logps/rejected": -256.14886474609375,
-      "loss": 0.5232,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.10842672735452652,
-      "rewards/margins": 0.7822098731994629,
-      "rewards/rejected": -0.8906365633010864,
-      "step": 1830
-    },
-    {
-      "epoch": 1.9,
-      "learning_rate": 2.0359739762724836e-07,
-      "logits/chosen": -2.415120840072632,
-      "logits/rejected": -2.359282970428467,
-      "logps/chosen": -279.69464111328125,
-      "logps/rejected": -237.9990234375,
-      "loss": 0.5288,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.1128331646323204,
-      "rewards/margins": 0.7950798869132996,
-      "rewards/rejected": -0.9079130291938782,
-      "step": 1840
-    },
-    {
-      "epoch": 1.91,
-      "learning_rate": 2.0168388825105242e-07,
-      "logits/chosen": -2.342097520828247,
-      "logits/rejected": -2.2842659950256348,
-      "logps/chosen": -260.4648742675781,
-      "logps/rejected": -264.97601318359375,
-      "loss": 0.516,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.14475753903388977,
-      "rewards/margins": 0.8012211918830872,
-      "rewards/rejected": -0.9459787607192993,
-      "step": 1850
-    },
-    {
-      "epoch": 1.92,
-      "learning_rate": 1.997703788748565e-07,
-      "logits/chosen": -2.3480544090270996,
-      "logits/rejected": -2.300661325454712,
-      "logps/chosen": -278.40704345703125,
-      "logps/rejected": -228.9430694580078,
-      "loss": 0.5331,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.2291232794523239,
-      "rewards/margins": 0.6820493936538696,
-      "rewards/rejected": -0.9111726880073547,
-      "step": 1860
-    },
-    {
-      "epoch": 1.93,
-      "learning_rate": 1.9785686949866055e-07,
-      "logits/chosen": -2.374687910079956,
-      "logits/rejected": -2.449584484100342,
-      "logps/chosen": -275.783447265625,
-      "logps/rejected": -233.29403686523438,
-      "loss": 0.5627,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.1256851851940155,
-      "rewards/margins": 0.5714830160140991,
-      "rewards/rejected": -0.6971681714057922,
-      "step": 1870
-    },
-    {
-      "epoch": 1.94,
-      "learning_rate": 1.9594336012246458e-07,
-      "logits/chosen": -2.4475274085998535,
-      "logits/rejected": -2.3420703411102295,
-      "logps/chosen": -270.1884460449219,
-      "logps/rejected": -235.0321502685547,
-      "loss": 0.5117,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.15421505272388458,
-      "rewards/margins": 0.7101847529411316,
-      "rewards/rejected": -0.864399790763855,
-      "step": 1880
-    },
-    {
-      "epoch": 1.95,
-      "learning_rate": 1.9402985074626865e-07,
-      "logits/chosen": -2.379213333129883,
-      "logits/rejected": -2.3949854373931885,
-      "logps/chosen": -269.8036804199219,
-      "logps/rejected": -228.36856079101562,
-      "loss": 0.5536,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.1839611977338791,
-      "rewards/margins": 0.684939980506897,
-      "rewards/rejected": -0.8689011335372925,
-      "step": 1890
-    },
-    {
-      "epoch": 1.96,
-      "learning_rate": 1.921163413700727e-07,
-      "logits/chosen": -2.3342185020446777,
-      "logits/rejected": -2.291137218475342,
-      "logps/chosen": -280.77459716796875,
-      "logps/rejected": -209.2417755126953,
-      "loss": 0.5782,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.22546765208244324,
-      "rewards/margins": 0.6627296209335327,
-      "rewards/rejected": -0.8881972432136536,
-      "step": 1900
-    },
-    {
-      "epoch": 1.97,
-      "learning_rate": 1.9020283199387677e-07,
-      "logits/chosen": -2.4452133178710938,
-      "logits/rejected": -2.3817238807678223,
-      "logps/chosen": -262.40533447265625,
-      "logps/rejected": -233.92431640625,
-      "loss": 0.552,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.05952071025967598,
-      "rewards/margins": 0.7161270976066589,
-      "rewards/rejected": -0.7756478190422058,
-      "step": 1910
-    },
-    {
-      "epoch": 1.98,
-      "learning_rate": 1.8828932261768083e-07,
-      "logits/chosen": -2.432097911834717,
-      "logits/rejected": -2.338268995285034,
-      "logps/chosen": -266.4609375,
-      "logps/rejected": -244.36953735351562,
-      "loss": 0.5319,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.1674240082502365,
-      "rewards/margins": 0.6536375284194946,
-      "rewards/rejected": -0.8210614919662476,
-      "step": 1920
-    },
-    {
-      "epoch": 1.99,
-      "learning_rate": 1.8637581324148487e-07,
-      "logits/chosen": -2.4438672065734863,
-      "logits/rejected": -2.4159255027770996,
-      "logps/chosen": -265.9490051269531,
-      "logps/rejected": -240.15011596679688,
-      "loss": 0.5412,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.11111648380756378,
-      "rewards/margins": 0.8190618753433228,
-      "rewards/rejected": -0.9301783442497253,
-      "step": 1930
-    },
-    {
-      "epoch": 2.0,
-      "eval_logits/chosen": -2.0748343467712402,
-      "eval_logits/rejected": -1.9509462118148804,
-      "eval_logps/chosen": -266.15826416015625,
-      "eval_logps/rejected": -227.92453002929688,
-      "eval_loss": 0.5318232774734497,
-      "eval_rewards/accuracies": 0.7419999837875366,
-      "eval_rewards/chosen": -0.1497425138950348,
-      "eval_rewards/margins": 0.7142792344093323,
-      "eval_rewards/rejected": -0.8640217185020447,
-      "eval_runtime": 456.7367,
-      "eval_samples_per_second": 4.379,
-      "eval_steps_per_second": 0.274,
-      "step": 1936
-    },
-    {
-      "epoch": 2.0,
-      "learning_rate": 1.8446230386528893e-07,
-      "logits/chosen": -2.3549647331237793,
-      "logits/rejected": -2.3090908527374268,
-      "logps/chosen": -239.0998992919922,
-      "logps/rejected": -226.9877166748047,
-      "loss": 0.5444,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.196995347738266,
-      "rewards/margins": 0.598027229309082,
-      "rewards/rejected": -0.7950225472450256,
-      "step": 1940
-    },
-    {
-      "epoch": 2.01,
-      "learning_rate": 1.82548794489093e-07,
-      "logits/chosen": -2.307945489883423,
-      "logits/rejected": -2.3319668769836426,
-      "logps/chosen": -243.7688446044922,
-      "logps/rejected": -234.90756225585938,
-      "loss": 0.5395,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.20119981467723846,
-      "rewards/margins": 0.630052924156189,
-      "rewards/rejected": -0.8312527537345886,
-      "step": 1950
-    },
-    {
-      "epoch": 2.02,
-      "learning_rate": 1.8063528511289706e-07,
-      "logits/chosen": -2.358461380004883,
-      "logits/rejected": -2.277463912963867,
-      "logps/chosen": -286.9745178222656,
-      "logps/rejected": -259.64605712890625,
-      "loss": 0.5319,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.10196640342473984,
-      "rewards/margins": 0.7701470851898193,
-      "rewards/rejected": -0.8721135258674622,
-      "step": 1960
-    },
-    {
-      "epoch": 2.03,
-      "learning_rate": 1.7872177573670112e-07,
-      "logits/chosen": -2.4066734313964844,
-      "logits/rejected": -2.4170022010803223,
-      "logps/chosen": -252.1322479248047,
-      "logps/rejected": -230.71755981445312,
-      "loss": 0.5399,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.08831964433193207,
-      "rewards/margins": 0.6736435890197754,
-      "rewards/rejected": -0.7619632482528687,
-      "step": 1970
-    },
-    {
-      "epoch": 2.04,
-      "learning_rate": 1.7680826636050515e-07,
-      "logits/chosen": -2.313403844833374,
-      "logits/rejected": -2.280029773712158,
-      "logps/chosen": -278.5584716796875,
-      "logps/rejected": -234.1926727294922,
-      "loss": 0.5248,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.07849719375371933,
-      "rewards/margins": 0.7140364050865173,
-      "rewards/rejected": -0.7925336360931396,
-      "step": 1980
-    },
-    {
-      "epoch": 2.06,
-      "learning_rate": 1.7489475698430921e-07,
-      "logits/chosen": -2.332839012145996,
-      "logits/rejected": -2.3671531677246094,
-      "logps/chosen": -249.85073852539062,
-      "logps/rejected": -249.09408569335938,
-      "loss": 0.5556,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.13992241024971008,
-      "rewards/margins": 0.6826204061508179,
-      "rewards/rejected": -0.8225427865982056,
-      "step": 1990
-    },
-    {
-      "epoch": 2.07,
-      "learning_rate": 1.7298124760811328e-07,
-      "logits/chosen": -2.40509295463562,
-      "logits/rejected": -2.385143756866455,
-      "logps/chosen": -277.4422302246094,
-      "logps/rejected": -224.10305786132812,
-      "loss": 0.539,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.1345512419939041,
-      "rewards/margins": 0.709915280342102,
-      "rewards/rejected": -0.8444665670394897,
-      "step": 2000
-    },
-    {
-      "epoch": 2.08,
-      "learning_rate": 1.7106773823191734e-07,
-      "logits/chosen": -2.438018321990967,
-      "logits/rejected": -2.372896909713745,
-      "logps/chosen": -270.50103759765625,
-      "logps/rejected": -240.79336547851562,
-      "loss": 0.5304,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.14716067910194397,
-      "rewards/margins": 0.8182382583618164,
-      "rewards/rejected": -0.965398907661438,
-      "step": 2010
-    },
-    {
-      "epoch": 2.09,
-      "learning_rate": 1.691542288557214e-07,
-      "logits/chosen": -2.41938853263855,
-      "logits/rejected": -2.322054624557495,
-      "logps/chosen": -297.6514892578125,
-      "logps/rejected": -230.926513671875,
-      "loss": 0.544,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.17855019867420197,
-      "rewards/margins": 0.7301106452941895,
-      "rewards/rejected": -0.908660888671875,
-      "step": 2020
-    },
-    {
-      "epoch": 2.1,
-      "learning_rate": 1.6724071947952544e-07,
-      "logits/chosen": -2.4344213008880615,
-      "logits/rejected": -2.275275468826294,
-      "logps/chosen": -240.5940704345703,
-      "logps/rejected": -232.0761260986328,
-      "loss": 0.5354,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.1535104215145111,
-      "rewards/margins": 0.6651692390441895,
-      "rewards/rejected": -0.818679690361023,
-      "step": 2030
-    },
-    {
-      "epoch": 2.11,
-      "learning_rate": 1.653272101033295e-07,
-      "logits/chosen": -2.4676742553710938,
-      "logits/rejected": -2.3419291973114014,
-      "logps/chosen": -276.0610656738281,
-      "logps/rejected": -212.14321899414062,
-      "loss": 0.5129,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.09980995208024979,
-      "rewards/margins": 0.677939772605896,
-      "rewards/rejected": -0.7777497172355652,
-      "step": 2040
-    },
-    {
-      "epoch": 2.12,
-      "learning_rate": 1.6341370072713356e-07,
-      "logits/chosen": -2.471161365509033,
-      "logits/rejected": -2.415436267852783,
-      "logps/chosen": -302.96063232421875,
-      "logps/rejected": -248.37313842773438,
-      "loss": 0.5382,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.08729290962219238,
-      "rewards/margins": 0.8466998934745789,
-      "rewards/rejected": -0.9339929819107056,
-      "step": 2050
-    },
-    {
-      "epoch": 2.13,
-      "learning_rate": 1.6150019135093762e-07,
-      "logits/chosen": -2.4522697925567627,
-      "logits/rejected": -2.3724026679992676,
-      "logps/chosen": -304.5672912597656,
-      "logps/rejected": -232.2679443359375,
-      "loss": 0.5288,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.1347544938325882,
-      "rewards/margins": 0.7216586470603943,
-      "rewards/rejected": -0.8564130663871765,
-      "step": 2060
-    },
-    {
-      "epoch": 2.14,
-      "learning_rate": 1.5958668197474169e-07,
-      "logits/chosen": -2.4622063636779785,
-      "logits/rejected": -2.3807263374328613,
-      "logps/chosen": -272.76177978515625,
-      "logps/rejected": -222.5310516357422,
-      "loss": 0.5348,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.09426755458116531,
-      "rewards/margins": 0.8474025726318359,
-      "rewards/rejected": -0.9416700601577759,
-      "step": 2070
-    },
-    {
-      "epoch": 2.15,
-      "learning_rate": 1.5767317259854572e-07,
-      "logits/chosen": -2.4014036655426025,
-      "logits/rejected": -2.3767526149749756,
-      "logps/chosen": -262.97088623046875,
-      "logps/rejected": -233.16796875,
-      "loss": 0.5329,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.1530064344406128,
-      "rewards/margins": 0.7272775769233704,
-      "rewards/rejected": -0.8802839517593384,
-      "step": 2080
-    },
-    {
-      "epoch": 2.16,
-      "learning_rate": 1.5575966322234978e-07,
-      "logits/chosen": -2.326734781265259,
-      "logits/rejected": -2.2565207481384277,
-      "logps/chosen": -259.03985595703125,
-      "logps/rejected": -247.90286254882812,
-      "loss": 0.5285,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.1133735179901123,
-      "rewards/margins": 0.7463991641998291,
-      "rewards/rejected": -0.8597726821899414,
-      "step": 2090
-    },
-    {
-      "epoch": 2.17,
-      "learning_rate": 1.5384615384615385e-07,
-      "logits/chosen": -2.378979444503784,
-      "logits/rejected": -2.2936904430389404,
-      "logps/chosen": -238.01724243164062,
-      "logps/rejected": -218.4258575439453,
-      "loss": 0.52,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.17722460627555847,
-      "rewards/margins": 0.7926307320594788,
-      "rewards/rejected": -0.9698553085327148,
-      "step": 2100
-    },
-    {
-      "epoch": 2.18,
-      "learning_rate": 1.519326444699579e-07,
-      "logits/chosen": -2.386594295501709,
-      "logits/rejected": -2.346097469329834,
-      "logps/chosen": -279.75958251953125,
-      "logps/rejected": -252.79776000976562,
-      "loss": 0.5138,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.10955911874771118,
-      "rewards/margins": 0.7847574949264526,
-      "rewards/rejected": -0.8943166732788086,
-      "step": 2110
-    },
-    {
-      "epoch": 2.19,
-      "learning_rate": 1.5001913509376197e-07,
-      "logits/chosen": -2.2668120861053467,
-      "logits/rejected": -2.29984188079834,
-      "logps/chosen": -242.16360473632812,
-      "logps/rejected": -219.63143920898438,
-      "loss": 0.5344,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.03612351417541504,
-      "rewards/margins": 0.8138107061386108,
-      "rewards/rejected": -0.8499342799186707,
-      "step": 2120
-    },
-    {
-      "epoch": 2.2,
-      "learning_rate": 1.4810562571756603e-07,
-      "logits/chosen": -2.405428647994995,
-      "logits/rejected": -2.382072925567627,
-      "logps/chosen": -260.3352355957031,
-      "logps/rejected": -238.9024658203125,
-      "loss": 0.5394,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.1940733641386032,
-      "rewards/margins": 0.6490978002548218,
-      "rewards/rejected": -0.843171238899231,
-      "step": 2130
-    },
-    {
-      "epoch": 2.21,
-      "learning_rate": 1.4619211634137007e-07,
-      "logits/chosen": -2.417595624923706,
-      "logits/rejected": -2.346921920776367,
-      "logps/chosen": -296.86602783203125,
-      "logps/rejected": -244.6204071044922,
-      "loss": 0.5207,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.08298005163669586,
-      "rewards/margins": 0.777125358581543,
-      "rewards/rejected": -0.8601053357124329,
-      "step": 2140
-    },
-    {
-      "epoch": 2.22,
-      "learning_rate": 1.4427860696517413e-07,
-      "logits/chosen": -2.310767650604248,
-      "logits/rejected": -2.2727725505828857,
-      "logps/chosen": -276.6751708984375,
-      "logps/rejected": -244.1461944580078,
-      "loss": 0.5488,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.23952357470989227,
-      "rewards/margins": 0.5698575973510742,
-      "rewards/rejected": -0.8093811273574829,
-      "step": 2150
-    },
-    {
-      "epoch": 2.23,
-      "learning_rate": 1.423650975889782e-07,
-      "logits/chosen": -2.413959503173828,
-      "logits/rejected": -2.317779302597046,
-      "logps/chosen": -301.21697998046875,
-      "logps/rejected": -230.67611694335938,
-      "loss": 0.5223,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.07718826830387115,
-      "rewards/margins": 0.7954361438751221,
-      "rewards/rejected": -0.872624397277832,
-      "step": 2160
-    },
-    {
-      "epoch": 2.24,
-      "learning_rate": 1.4045158821278225e-07,
-      "logits/chosen": -2.43796443939209,
-      "logits/rejected": -2.3855981826782227,
-      "logps/chosen": -281.7301025390625,
-      "logps/rejected": -252.88900756835938,
-      "loss": 0.5216,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.07020233571529388,
-      "rewards/margins": 0.8178524971008301,
-      "rewards/rejected": -0.8880547285079956,
-      "step": 2170
-    },
-    {
-      "epoch": 2.25,
-      "learning_rate": 1.3853807883658632e-07,
-      "logits/chosen": -2.381850481033325,
-      "logits/rejected": -2.3632988929748535,
-      "logps/chosen": -286.48199462890625,
-      "logps/rejected": -263.02325439453125,
-      "loss": 0.5582,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.06794221699237823,
-      "rewards/margins": 0.675620436668396,
-      "rewards/rejected": -0.743562638759613,
-      "step": 2180
-    },
-    {
-      "epoch": 2.26,
-      "learning_rate": 1.3662456946039035e-07,
-      "logits/chosen": -2.3416738510131836,
-      "logits/rejected": -2.3051633834838867,
-      "logps/chosen": -244.4949188232422,
-      "logps/rejected": -206.51089477539062,
-      "loss": 0.5339,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.12611594796180725,
-      "rewards/margins": 0.6276872158050537,
-      "rewards/rejected": -0.7538031339645386,
-      "step": 2190
-    },
-    {
-      "epoch": 2.27,
-      "learning_rate": 1.3471106008419441e-07,
-      "logits/chosen": -2.3636796474456787,
-      "logits/rejected": -2.3292782306671143,
-      "logps/chosen": -273.59027099609375,
-      "logps/rejected": -260.6876220703125,
-      "loss": 0.5276,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.08774266391992569,
-      "rewards/margins": 0.6979103088378906,
-      "rewards/rejected": -0.7856529951095581,
-      "step": 2200
-    },
-    {
-      "epoch": 2.28,
-      "learning_rate": 1.3279755070799848e-07,
-      "logits/chosen": -2.3285207748413086,
-      "logits/rejected": -2.2629952430725098,
-      "logps/chosen": -284.96600341796875,
-      "logps/rejected": -232.8168182373047,
-      "loss": 0.5429,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.10798223316669464,
-      "rewards/margins": 0.7428350448608398,
-      "rewards/rejected": -0.8508173227310181,
-      "step": 2210
-    },
-    {
-      "epoch": 2.29,
-      "learning_rate": 1.3088404133180254e-07,
-      "logits/chosen": -2.2993249893188477,
-      "logits/rejected": -2.2543981075286865,
-      "logps/chosen": -306.201171875,
-      "logps/rejected": -226.76223754882812,
-      "loss": 0.483,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.06968189030885696,
-      "rewards/margins": 0.9336912035942078,
-      "rewards/rejected": -1.003373146057129,
-      "step": 2220
-    },
-    {
-      "epoch": 2.3,
-      "learning_rate": 1.289705319556066e-07,
-      "logits/chosen": -2.3141911029815674,
-      "logits/rejected": -2.251255512237549,
-      "logps/chosen": -252.2708282470703,
-      "logps/rejected": -221.80953979492188,
-      "loss": 0.5277,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.14586138725280762,
-      "rewards/margins": 0.6129014492034912,
-      "rewards/rejected": -0.7587628960609436,
-      "step": 2230
-    },
-    {
-      "epoch": 2.31,
-      "learning_rate": 1.2705702257941064e-07,
-      "logits/chosen": -2.373944044113159,
-      "logits/rejected": -2.362863779067993,
-      "logps/chosen": -257.61285400390625,
-      "logps/rejected": -234.9366455078125,
-      "loss": 0.5122,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.1866065412759781,
-      "rewards/margins": 0.6813262701034546,
-      "rewards/rejected": -0.8679327964782715,
-      "step": 2240
-    },
-    {
-      "epoch": 2.32,
-      "learning_rate": 1.251435132032147e-07,
-      "logits/chosen": -2.3465607166290283,
-      "logits/rejected": -2.338548421859741,
-      "logps/chosen": -273.64178466796875,
-      "logps/rejected": -259.8929748535156,
-      "loss": 0.5306,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.0799819678068161,
-      "rewards/margins": 0.7552547454833984,
-      "rewards/rejected": -0.8352367281913757,
-      "step": 2250
-    },
-    {
-      "epoch": 2.33,
-      "learning_rate": 1.2323000382701873e-07,
-      "logits/chosen": -2.3777191638946533,
-      "logits/rejected": -2.3778316974639893,
-      "logps/chosen": -270.16326904296875,
-      "logps/rejected": -229.1251983642578,
-      "loss": 0.5575,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.2065315544605255,
-      "rewards/margins": 0.6355454921722412,
-      "rewards/rejected": -0.8420770764350891,
-      "step": 2260
-    },
-    {
-      "epoch": 2.34,
-      "learning_rate": 1.213164944508228e-07,
-      "logits/chosen": -2.4501662254333496,
-      "logits/rejected": -2.3497557640075684,
-      "logps/chosen": -266.2723083496094,
-      "logps/rejected": -269.751708984375,
-      "loss": 0.5113,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.2602607309818268,
-      "rewards/margins": 0.6484657526016235,
-      "rewards/rejected": -0.9087265133857727,
-      "step": 2270
-    },
-    {
-      "epoch": 2.35,
-      "learning_rate": 1.1940298507462686e-07,
-      "logits/chosen": -2.2452497482299805,
-      "logits/rejected": -2.276188850402832,
-      "logps/chosen": -261.48919677734375,
-      "logps/rejected": -211.9011688232422,
-      "loss": 0.5202,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.0977492704987526,
-      "rewards/margins": 0.8213205337524414,
-      "rewards/rejected": -0.9190698862075806,
-      "step": 2280
-    },
-    {
-      "epoch": 2.37,
-      "learning_rate": 1.1748947569843092e-07,
-      "logits/chosen": -2.39143967628479,
-      "logits/rejected": -2.307307481765747,
-      "logps/chosen": -263.43524169921875,
-      "logps/rejected": -225.5679473876953,
-      "loss": 0.5088,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.12165951728820801,
-      "rewards/margins": 0.8993788957595825,
-      "rewards/rejected": -1.021038293838501,
-      "step": 2290
-    },
-    {
-      "epoch": 2.38,
-      "learning_rate": 1.1557596632223497e-07,
-      "logits/chosen": -2.361375331878662,
-      "logits/rejected": -2.375532627105713,
-      "logps/chosen": -272.00408935546875,
-      "logps/rejected": -224.09286499023438,
-      "loss": 0.5223,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.13877837359905243,
-      "rewards/margins": 0.8421686291694641,
-      "rewards/rejected": -0.9809468984603882,
-      "step": 2300
-    },
-    {
-      "epoch": 2.39,
-      "learning_rate": 1.1366245694603903e-07,
-      "logits/chosen": -2.4291224479675293,
-      "logits/rejected": -2.245253801345825,
-      "logps/chosen": -265.1824645996094,
-      "logps/rejected": -249.3574676513672,
-      "loss": 0.5579,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.24909143149852753,
-      "rewards/margins": 0.6685684323310852,
-      "rewards/rejected": -0.9176598787307739,
-      "step": 2310
-    },
-    {
-      "epoch": 2.4,
-      "learning_rate": 1.1174894756984308e-07,
-      "logits/chosen": -2.2975034713745117,
-      "logits/rejected": -2.394157886505127,
-      "logps/chosen": -254.5048370361328,
-      "logps/rejected": -224.20376586914062,
-      "loss": 0.5215,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.18976199626922607,
-      "rewards/margins": 0.8317477107048035,
-      "rewards/rejected": -1.0215097665786743,
-      "step": 2320
-    },
-    {
-      "epoch": 2.41,
-      "learning_rate": 1.0983543819364714e-07,
-      "logits/chosen": -2.4635863304138184,
-      "logits/rejected": -2.4494881629943848,
-      "logps/chosen": -278.895263671875,
-      "logps/rejected": -231.98135375976562,
-      "loss": 0.4772,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.08000790327787399,
-      "rewards/margins": 0.9092146158218384,
-      "rewards/rejected": -0.989222526550293,
-      "step": 2330
-    },
-    {
-      "epoch": 2.42,
-      "learning_rate": 1.079219288174512e-07,
-      "logits/chosen": -2.3342247009277344,
-      "logits/rejected": -2.319608211517334,
-      "logps/chosen": -247.49368286132812,
-      "logps/rejected": -209.24111938476562,
-      "loss": 0.5371,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.20573286712169647,
-      "rewards/margins": 0.5834436416625977,
-      "rewards/rejected": -0.7891765236854553,
-      "step": 2340
-    },
-    {
-      "epoch": 2.43,
-      "learning_rate": 1.0600841944125525e-07,
-      "logits/chosen": -2.436828136444092,
-      "logits/rejected": -2.3965752124786377,
-      "logps/chosen": -271.6636047363281,
-      "logps/rejected": -221.7318572998047,
-      "loss": 0.5528,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.20491096377372742,
-      "rewards/margins": 0.6560606360435486,
-      "rewards/rejected": -0.8609716296195984,
-      "step": 2350
-    },
-    {
-      "epoch": 2.44,
-      "learning_rate": 1.0409491006505931e-07,
-      "logits/chosen": -2.400538682937622,
-      "logits/rejected": -2.284837484359741,
-      "logps/chosen": -283.9150390625,
-      "logps/rejected": -229.0013885498047,
-      "loss": 0.5457,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.1943405717611313,
-      "rewards/margins": 0.7493225336074829,
-      "rewards/rejected": -0.9436630010604858,
-      "step": 2360
-    },
-    {
-      "epoch": 2.45,
-      "learning_rate": 1.0218140068886336e-07,
-      "logits/chosen": -2.377981662750244,
-      "logits/rejected": -2.2820467948913574,
-      "logps/chosen": -260.76312255859375,
-      "logps/rejected": -221.4712371826172,
-      "loss": 0.5534,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.21287551522254944,
-      "rewards/margins": 0.5487321615219116,
-      "rewards/rejected": -0.7616077661514282,
-      "step": 2370
-    },
-    {
-      "epoch": 2.46,
-      "learning_rate": 1.0026789131266743e-07,
-      "logits/chosen": -2.295510768890381,
-      "logits/rejected": -2.329360246658325,
-      "logps/chosen": -252.51107788085938,
-      "logps/rejected": -242.34616088867188,
-      "loss": 0.5429,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.1736653447151184,
-      "rewards/margins": 0.6323011517524719,
-      "rewards/rejected": -0.8059664964675903,
-      "step": 2380
-    },
-    {
-      "epoch": 2.47,
-      "learning_rate": 9.835438193647149e-08,
-      "logits/chosen": -2.3245301246643066,
-      "logits/rejected": -2.372647523880005,
-      "logps/chosen": -255.889892578125,
-      "logps/rejected": -230.44961547851562,
-      "loss": 0.5579,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.15318112075328827,
-      "rewards/margins": 0.6755190491676331,
-      "rewards/rejected": -0.8287001848220825,
-      "step": 2390
-    },
-    {
-      "epoch": 2.48,
-      "learning_rate": 9.644087256027554e-08,
-      "logits/chosen": -2.393808364868164,
-      "logits/rejected": -2.387742280960083,
-      "logps/chosen": -239.28219604492188,
-      "logps/rejected": -218.7506103515625,
-      "loss": 0.536,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.13704940676689148,
-      "rewards/margins": 0.6621817946434021,
-      "rewards/rejected": -0.7992312908172607,
-      "step": 2400
-    },
-    {
-      "epoch": 2.49,
-      "learning_rate": 9.45273631840796e-08,
-      "logits/chosen": -2.3953936100006104,
-      "logits/rejected": -2.4050917625427246,
-      "logps/chosen": -278.6383361816406,
-      "logps/rejected": -231.466552734375,
-      "loss": 0.5329,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.07729502022266388,
-      "rewards/margins": 0.736717700958252,
-      "rewards/rejected": -0.8140127062797546,
-      "step": 2410
-    },
-    {
-      "epoch": 2.5,
-      "learning_rate": 9.261385380788366e-08,
-      "logits/chosen": -2.276078939437866,
-      "logits/rejected": -2.2849624156951904,
-      "logps/chosen": -239.13525390625,
-      "logps/rejected": -222.2736053466797,
-      "loss": 0.5249,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.2353070229291916,
-      "rewards/margins": 0.6328269243240356,
-      "rewards/rejected": -0.8681339025497437,
-      "step": 2420
-    },
-    {
-      "epoch": 2.51,
-      "learning_rate": 9.070034443168771e-08,
-      "logits/chosen": -2.387070894241333,
-      "logits/rejected": -2.37086820602417,
-      "logps/chosen": -261.84112548828125,
-      "logps/rejected": -227.5956268310547,
-      "loss": 0.557,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.2481728047132492,
-      "rewards/margins": 0.6492807865142822,
-      "rewards/rejected": -0.8974536657333374,
-      "step": 2430
-    },
-    {
-      "epoch": 2.52,
-      "learning_rate": 8.878683505549177e-08,
-      "logits/chosen": -2.2902541160583496,
-      "logits/rejected": -2.3299739360809326,
-      "logps/chosen": -267.1860656738281,
-      "logps/rejected": -226.9349822998047,
-      "loss": 0.5107,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.10977288335561752,
-      "rewards/margins": 0.7155038714408875,
-      "rewards/rejected": -0.8252767324447632,
-      "step": 2440
-    },
-    {
-      "epoch": 2.53,
-      "learning_rate": 8.687332567929582e-08,
-      "logits/chosen": -2.3206660747528076,
-      "logits/rejected": -2.3494954109191895,
-      "logps/chosen": -291.82305908203125,
-      "logps/rejected": -246.83761596679688,
-      "loss": 0.5355,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.044042427092790604,
-      "rewards/margins": 0.900373637676239,
-      "rewards/rejected": -0.9444161653518677,
-      "step": 2450
-    },
-    {
-      "epoch": 2.54,
-      "learning_rate": 8.495981630309988e-08,
-      "logits/chosen": -2.2711682319641113,
-      "logits/rejected": -2.290236473083496,
-      "logps/chosen": -300.72979736328125,
-      "logps/rejected": -218.26022338867188,
-      "loss": 0.5448,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.1306503415107727,
-      "rewards/margins": 0.5897533297538757,
-      "rewards/rejected": -0.7204037308692932,
-      "step": 2460
-    },
-    {
-      "epoch": 2.55,
-      "learning_rate": 8.304630692690395e-08,
-      "logits/chosen": -2.317441463470459,
-      "logits/rejected": -2.2715091705322266,
-      "logps/chosen": -262.0405578613281,
-      "logps/rejected": -205.5135955810547,
-      "loss": 0.5286,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.14162805676460266,
-      "rewards/margins": 0.7206326723098755,
-      "rewards/rejected": -0.8622606992721558,
-      "step": 2470
-    },
-    {
-      "epoch": 2.56,
-      "learning_rate": 8.1132797550708e-08,
-      "logits/chosen": -2.3969407081604004,
-      "logits/rejected": -2.3611950874328613,
-      "logps/chosen": -267.4500427246094,
-      "logps/rejected": -221.7566375732422,
-      "loss": 0.5277,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.15396210551261902,
-      "rewards/margins": 0.7070397138595581,
-      "rewards/rejected": -0.8610017895698547,
-      "step": 2480
-    },
-    {
-      "epoch": 2.57,
-      "learning_rate": 7.921928817451206e-08,
-      "logits/chosen": -2.4169304370880127,
-      "logits/rejected": -2.321960926055908,
-      "logps/chosen": -277.09588623046875,
-      "logps/rejected": -227.67959594726562,
-      "loss": 0.5088,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.17860722541809082,
-      "rewards/margins": 0.7987648248672485,
-      "rewards/rejected": -0.9773720502853394,
-      "step": 2490
-    },
-    {
-      "epoch": 2.58,
-      "learning_rate": 7.73057787983161e-08,
-      "logits/chosen": -2.3953964710235596,
-      "logits/rejected": -2.301238536834717,
-      "logps/chosen": -292.7876892089844,
-      "logps/rejected": -266.57476806640625,
-      "loss": 0.5165,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.08117020130157471,
-      "rewards/margins": 0.7519367933273315,
-      "rewards/rejected": -0.8331069946289062,
-      "step": 2500
-    },
-    {
-      "epoch": 2.59,
-      "learning_rate": 7.539226942212017e-08,
-      "logits/chosen": -2.3565969467163086,
-      "logits/rejected": -2.3130733966827393,
-      "logps/chosen": -243.97903442382812,
-      "logps/rejected": -219.99301147460938,
-      "loss": 0.5185,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.11240849643945694,
-      "rewards/margins": 0.8732815980911255,
-      "rewards/rejected": -0.9856899976730347,
-      "step": 2510
-    },
-    {
-      "epoch": 2.6,
-      "learning_rate": 7.347876004592423e-08,
-      "logits/chosen": -2.4705801010131836,
-      "logits/rejected": -2.40214467048645,
-      "logps/chosen": -280.0275573730469,
-      "logps/rejected": -232.6969451904297,
-      "loss": 0.5403,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.14689256250858307,
-      "rewards/margins": 0.7363706231117249,
-      "rewards/rejected": -0.883263111114502,
-      "step": 2520
-    },
-    {
-      "epoch": 2.61,
-      "learning_rate": 7.156525066972828e-08,
-      "logits/chosen": -2.299910068511963,
-      "logits/rejected": -2.3627266883850098,
-      "logps/chosen": -242.22579956054688,
-      "logps/rejected": -227.53482055664062,
-      "loss": 0.5402,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.088679239153862,
-      "rewards/margins": 0.6972663402557373,
-      "rewards/rejected": -0.7859456539154053,
-      "step": 2530
-    },
-    {
-      "epoch": 2.62,
-      "learning_rate": 6.965174129353234e-08,
-      "logits/chosen": -2.3531718254089355,
-      "logits/rejected": -2.2922921180725098,
-      "logps/chosen": -302.03350830078125,
-      "logps/rejected": -258.0994567871094,
-      "loss": 0.5287,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.10454048961400986,
-      "rewards/margins": 0.8333840370178223,
-      "rewards/rejected": -0.9379245042800903,
-      "step": 2540
-    },
-    {
-      "epoch": 2.63,
-      "learning_rate": 6.773823191733639e-08,
-      "logits/chosen": -2.284903049468994,
-      "logits/rejected": -2.2148680686950684,
-      "logps/chosen": -259.69036865234375,
-      "logps/rejected": -222.1662139892578,
-      "loss": 0.5297,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.24571335315704346,
-      "rewards/margins": 0.7015795707702637,
-      "rewards/rejected": -0.9472929239273071,
-      "step": 2550
-    },
-    {
-      "epoch": 2.64,
-      "learning_rate": 6.582472254114045e-08,
-      "logits/chosen": -2.3052189350128174,
-      "logits/rejected": -2.3175923824310303,
-      "logps/chosen": -252.48291015625,
-      "logps/rejected": -223.8980712890625,
-      "loss": 0.5649,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.12469018995761871,
-      "rewards/margins": 0.6323705911636353,
-      "rewards/rejected": -0.757060706615448,
-      "step": 2560
-    },
-    {
-      "epoch": 2.65,
-      "learning_rate": 6.391121316494451e-08,
-      "logits/chosen": -2.3261055946350098,
-      "logits/rejected": -2.3110244274139404,
-      "logps/chosen": -295.005859375,
-      "logps/rejected": -239.6460418701172,
-      "loss": 0.5248,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.10888131707906723,
-      "rewards/margins": 0.8443535566329956,
-      "rewards/rejected": -0.953234851360321,
-      "step": 2570
-    },
-    {
-      "epoch": 2.66,
-      "learning_rate": 6.199770378874856e-08,
-      "logits/chosen": -2.377129077911377,
-      "logits/rejected": -2.2884180545806885,
-      "logps/chosen": -272.9131164550781,
-      "logps/rejected": -225.4416961669922,
-      "loss": 0.5302,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.07070966064929962,
-      "rewards/margins": 0.9023112058639526,
-      "rewards/rejected": -0.9730209112167358,
-      "step": 2580
-    },
-    {
-      "epoch": 2.67,
-      "learning_rate": 6.008419441255262e-08,
-      "logits/chosen": -2.3185131549835205,
-      "logits/rejected": -2.260045051574707,
-      "logps/chosen": -299.19635009765625,
-      "logps/rejected": -217.824951171875,
-      "loss": 0.5244,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.07015608996152878,
-      "rewards/margins": 0.8233577013015747,
-      "rewards/rejected": -0.8935137987136841,
-      "step": 2590
-    },
-    {
-      "epoch": 2.69,
-      "learning_rate": 5.817068503635668e-08,
-      "logits/chosen": -2.4003567695617676,
-      "logits/rejected": -2.3623907566070557,
-      "logps/chosen": -288.9065856933594,
-      "logps/rejected": -222.6360626220703,
-      "loss": 0.515,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.12081106007099152,
-      "rewards/margins": 0.8377307057380676,
-      "rewards/rejected": -0.958541750907898,
-      "step": 2600
-    },
-    {
-      "epoch": 2.7,
-      "learning_rate": 5.6257175660160735e-08,
-      "logits/chosen": -2.420441150665283,
-      "logits/rejected": -2.37391996383667,
-      "logps/chosen": -295.19952392578125,
-      "logps/rejected": -238.8549041748047,
-      "loss": 0.516,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.18193045258522034,
-      "rewards/margins": 0.7073780298233032,
-      "rewards/rejected": -0.8893085718154907,
-      "step": 2610
-    },
-    {
-      "epoch": 2.71,
-      "learning_rate": 5.4343666283964784e-08,
-      "logits/chosen": -2.4277868270874023,
-      "logits/rejected": -2.3764290809631348,
-      "logps/chosen": -263.9327697753906,
-      "logps/rejected": -249.3427734375,
-      "loss": 0.5214,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.07714501768350601,
-      "rewards/margins": 0.7166513204574585,
-      "rewards/rejected": -0.7937964200973511,
-      "step": 2620
-    },
-    {
-      "epoch": 2.72,
-      "learning_rate": 5.243015690776884e-08,
-      "logits/chosen": -2.467040538787842,
-      "logits/rejected": -2.406569719314575,
-      "logps/chosen": -285.0174865722656,
-      "logps/rejected": -235.902587890625,
-      "loss": 0.5146,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.15382210910320282,
-      "rewards/margins": 0.7074260711669922,
-      "rewards/rejected": -0.8612481951713562,
-      "step": 2630
-    },
-    {
-      "epoch": 2.73,
-      "learning_rate": 5.05166475315729e-08,
-      "logits/chosen": -2.3759026527404785,
-      "logits/rejected": -2.323474884033203,
-      "logps/chosen": -263.00689697265625,
-      "logps/rejected": -218.7880859375,
-      "loss": 0.5388,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.2826555371284485,
-      "rewards/margins": 0.6195917129516602,
-      "rewards/rejected": -0.9022472500801086,
-      "step": 2640
-    },
-    {
-      "epoch": 2.74,
-      "learning_rate": 4.860313815537696e-08,
-      "logits/chosen": -2.4053902626037598,
-      "logits/rejected": -2.36479115486145,
-      "logps/chosen": -284.4691467285156,
-      "logps/rejected": -266.75579833984375,
-      "loss": 0.5364,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.11270469427108765,
-      "rewards/margins": 0.7914165258407593,
-      "rewards/rejected": -0.9041212201118469,
-      "step": 2650
-    },
-    {
-      "epoch": 2.75,
-      "learning_rate": 4.668962877918101e-08,
-      "logits/chosen": -2.380582332611084,
-      "logits/rejected": -2.2785563468933105,
-      "logps/chosen": -288.0699768066406,
-      "logps/rejected": -278.1865234375,
-      "loss": 0.5236,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.12235669791698456,
-      "rewards/margins": 0.8113776445388794,
-      "rewards/rejected": -0.9337342381477356,
-      "step": 2660
-    },
-    {
-      "epoch": 2.76,
-      "learning_rate": 4.477611940298507e-08,
-      "logits/chosen": -2.4007792472839355,
-      "logits/rejected": -2.3088767528533936,
-      "logps/chosen": -245.1487274169922,
-      "logps/rejected": -227.521728515625,
-      "loss": 0.4938,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.13851900398731232,
-      "rewards/margins": 0.8027512431144714,
-      "rewards/rejected": -0.9412702322006226,
-      "step": 2670
-    },
-    {
-      "epoch": 2.77,
-      "learning_rate": 4.2862610026789124e-08,
-      "logits/chosen": -2.4124245643615723,
-      "logits/rejected": -2.296288013458252,
-      "logps/chosen": -259.74029541015625,
-      "logps/rejected": -223.14627075195312,
-      "loss": 0.5476,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.1544261872768402,
-      "rewards/margins": 0.7074520587921143,
-      "rewards/rejected": -0.8618782162666321,
-      "step": 2680
-    },
-    {
-      "epoch": 2.78,
-      "learning_rate": 4.0949100650593186e-08,
-      "logits/chosen": -2.3861641883850098,
-      "logits/rejected": -2.328504800796509,
-      "logps/chosen": -256.6800231933594,
-      "logps/rejected": -243.67514038085938,
-      "loss": 0.546,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.25118106603622437,
-      "rewards/margins": 0.5502891540527344,
-      "rewards/rejected": -0.8014701008796692,
-      "step": 2690
-    },
-    {
-      "epoch": 2.79,
-      "learning_rate": 3.903559127439724e-08,
-      "logits/chosen": -2.294541358947754,
-      "logits/rejected": -2.2779922485351562,
-      "logps/chosen": -273.6397399902344,
-      "logps/rejected": -230.7729949951172,
-      "loss": 0.5064,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.11550472676753998,
-      "rewards/margins": 0.7522660493850708,
-      "rewards/rejected": -0.867770791053772,
-      "step": 2700
-    },
-    {
-      "epoch": 2.8,
-      "learning_rate": 3.71220818982013e-08,
-      "logits/chosen": -2.406259536743164,
-      "logits/rejected": -2.3714747428894043,
-      "logps/chosen": -273.5616760253906,
-      "logps/rejected": -218.18923950195312,
-      "loss": 0.5481,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.23518535494804382,
-      "rewards/margins": 0.6195680499076843,
-      "rewards/rejected": -0.8547533750534058,
-      "step": 2710
-    },
-    {
-      "epoch": 2.81,
-      "learning_rate": 3.520857252200535e-08,
-      "logits/chosen": -2.363008975982666,
-      "logits/rejected": -2.3325300216674805,
-      "logps/chosen": -300.9599609375,
-      "logps/rejected": -242.6225128173828,
-      "loss": 0.5323,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.039127424359321594,
-      "rewards/margins": 0.9830048680305481,
-      "rewards/rejected": -1.022132158279419,
-      "step": 2720
-    },
-    {
-      "epoch": 2.82,
-      "learning_rate": 3.3295063145809414e-08,
-      "logits/chosen": -2.280790328979492,
-      "logits/rejected": -2.225207805633545,
-      "logps/chosen": -266.9983825683594,
-      "logps/rejected": -244.36279296875,
-      "loss": 0.5561,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.14134183526039124,
-      "rewards/margins": 0.7063352465629578,
-      "rewards/rejected": -0.8476771116256714,
-      "step": 2730
-    },
-    {
-      "epoch": 2.83,
-      "learning_rate": 3.138155376961347e-08,
-      "logits/chosen": -2.288053512573242,
-      "logits/rejected": -2.3323187828063965,
-      "logps/chosen": -234.13241577148438,
-      "logps/rejected": -226.0674285888672,
-      "loss": 0.5667,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.14560513198375702,
-      "rewards/margins": 0.7158702611923218,
-      "rewards/rejected": -0.8614753484725952,
-      "step": 2740
-    },
-    {
-      "epoch": 2.84,
-      "learning_rate": 2.9468044393417525e-08,
-      "logits/chosen": -2.4299161434173584,
-      "logits/rejected": -2.40264892578125,
-      "logps/chosen": -273.2800598144531,
-      "logps/rejected": -262.6374206542969,
-      "loss": 0.5196,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.06616147607564926,
-      "rewards/margins": 0.8120719194412231,
-      "rewards/rejected": -0.8782333135604858,
-      "step": 2750
-    },
-    {
-      "epoch": 2.85,
-      "learning_rate": 2.755453501722158e-08,
-      "logits/chosen": -2.37566876411438,
-      "logits/rejected": -2.36662220954895,
-      "logps/chosen": -285.37725830078125,
-      "logps/rejected": -228.5401611328125,
-      "loss": 0.5099,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.09863577038049698,
-      "rewards/margins": 0.8224757313728333,
-      "rewards/rejected": -0.9211114645004272,
-      "step": 2760
-    },
-    {
-      "epoch": 2.86,
-      "learning_rate": 2.564102564102564e-08,
-      "logits/chosen": -2.399296283721924,
-      "logits/rejected": -2.3836464881896973,
-      "logps/chosen": -258.1005554199219,
-      "logps/rejected": -225.07723999023438,
-      "loss": 0.524,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.18826046586036682,
-      "rewards/margins": 0.6354348659515381,
-      "rewards/rejected": -0.8236953020095825,
-      "step": 2770
-    },
-    {
-      "epoch": 2.87,
-      "learning_rate": 2.3727516264829695e-08,
-      "logits/chosen": -2.320814609527588,
-      "logits/rejected": -2.295502185821533,
-      "logps/chosen": -238.77200317382812,
-      "logps/rejected": -234.020263671875,
-      "loss": 0.5344,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.1772747039794922,
-      "rewards/margins": 0.5961662530899048,
-      "rewards/rejected": -0.7734408974647522,
-      "step": 2780
-    },
-    {
-      "epoch": 2.88,
-      "learning_rate": 2.1814006888633754e-08,
-      "logits/chosen": -2.334709405899048,
-      "logits/rejected": -2.3521342277526855,
-      "logps/chosen": -263.27349853515625,
-      "logps/rejected": -233.0219268798828,
-      "loss": 0.5471,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.2036319077014923,
-      "rewards/margins": 0.6492605805397034,
-      "rewards/rejected": -0.8528925180435181,
-      "step": 2790
-    },
-    {
-      "epoch": 2.89,
-      "learning_rate": 1.990049751243781e-08,
-      "logits/chosen": -2.3961801528930664,
-      "logits/rejected": -2.344447612762451,
-      "logps/chosen": -285.4869079589844,
-      "logps/rejected": -231.99276733398438,
-      "loss": 0.5319,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.11304112523794174,
-      "rewards/margins": 0.6869500875473022,
-      "rewards/rejected": -0.7999913096427917,
-      "step": 2800
-    },
-    {
-      "epoch": 2.9,
-      "learning_rate": 1.7986988136241865e-08,
-      "logits/chosen": -2.4139771461486816,
-      "logits/rejected": -2.357755661010742,
-      "logps/chosen": -262.9169006347656,
-      "logps/rejected": -225.28134155273438,
-      "loss": 0.4933,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.029026124626398087,
-      "rewards/margins": 0.874340832233429,
-      "rewards/rejected": -0.9033668637275696,
-      "step": 2810
-    },
-    {
-      "epoch": 2.91,
-      "learning_rate": 1.6073478760045924e-08,
-      "logits/chosen": -2.2996811866760254,
-      "logits/rejected": -2.224923849105835,
-      "logps/chosen": -285.3755798339844,
-      "logps/rejected": -201.12803649902344,
-      "loss": 0.5432,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.24773137271404266,
-      "rewards/margins": 0.5589512586593628,
-      "rewards/rejected": -0.8066825866699219,
-      "step": 2820
-    },
-    {
-      "epoch": 2.92,
-      "learning_rate": 1.4159969383849981e-08,
-      "logits/chosen": -2.493635654449463,
-      "logits/rejected": -2.4172415733337402,
-      "logps/chosen": -312.3724670410156,
-      "logps/rejected": -248.0670623779297,
-      "loss": 0.5265,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.10385121405124664,
-      "rewards/margins": 0.8151585459709167,
-      "rewards/rejected": -0.9190096855163574,
-      "step": 2830
-    },
-    {
-      "epoch": 2.93,
-      "learning_rate": 1.2246460007654037e-08,
-      "logits/chosen": -2.365940570831299,
-      "logits/rejected": -2.3057992458343506,
-      "logps/chosen": -266.408447265625,
-      "logps/rejected": -227.32492065429688,
-      "loss": 0.571,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.23473727703094482,
-      "rewards/margins": 0.5884072184562683,
-      "rewards/rejected": -0.8231445550918579,
-      "step": 2840
-    },
-    {
-      "epoch": 2.94,
-      "learning_rate": 1.0332950631458094e-08,
-      "logits/chosen": -2.306847095489502,
-      "logits/rejected": -2.352964401245117,
-      "logps/chosen": -273.6145935058594,
-      "logps/rejected": -230.6118927001953,
-      "loss": 0.5353,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.1490267664194107,
-      "rewards/margins": 0.7372574210166931,
-      "rewards/rejected": -0.8862841725349426,
-      "step": 2850
-    },
-    {
-      "epoch": 2.95,
-      "learning_rate": 8.419441255262151e-09,
-      "logits/chosen": -2.2987818717956543,
-      "logits/rejected": -2.319676160812378,
-      "logps/chosen": -267.0763854980469,
-      "logps/rejected": -219.8603973388672,
-      "loss": 0.5045,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.09555543214082718,
-      "rewards/margins": 0.6565554738044739,
-      "rewards/rejected": -0.7521108388900757,
-      "step": 2860
-    },
-    {
-      "epoch": 2.96,
-      "learning_rate": 6.505931879066207e-09,
-      "logits/chosen": -2.403472900390625,
-      "logits/rejected": -2.3466238975524902,
-      "logps/chosen": -235.06271362304688,
-      "logps/rejected": -236.23068237304688,
-      "loss": 0.5334,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.24678435921669006,
-      "rewards/margins": 0.6536287665367126,
-      "rewards/rejected": -0.9004132151603699,
-      "step": 2870
-    },
-    {
-      "epoch": 2.97,
-      "learning_rate": 4.592422502870264e-09,
-      "logits/chosen": -2.413404941558838,
-      "logits/rejected": -2.317659854888916,
-      "logps/chosen": -259.46697998046875,
-      "logps/rejected": -227.43246459960938,
-      "loss": 0.5463,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.05970311164855957,
-      "rewards/margins": 0.6795765161514282,
-      "rewards/rejected": -0.7392796277999878,
-      "step": 2880
-    },
-    {
-      "epoch": 2.98,
-      "learning_rate": 2.6789131266743202e-09,
-      "logits/chosen": -2.3422017097473145,
-      "logits/rejected": -2.2731831073760986,
-      "logps/chosen": -233.3965301513672,
-      "logps/rejected": -225.33566284179688,
-      "loss": 0.5157,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.2789511978626251,
-      "rewards/margins": 0.6096433401107788,
-      "rewards/rejected": -0.8885944485664368,
-      "step": 2890
-    },
-    {
-      "epoch": 3.0,
-      "learning_rate": 7.654037504783773e-10,
-      "logits/chosen": -2.3140885829925537,
-      "logits/rejected": -2.351783275604248,
-      "logps/chosen": -238.41293334960938,
-      "logps/rejected": -245.66201782226562,
-      "loss": 0.5454,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.1346302330493927,
-      "rewards/margins": 0.7680613398551941,
-      "rewards/rejected": -0.9026915431022644,
-      "step": 2900
-    },
     {
       "epoch": 3.0,
-      "eval_logits/chosen": -2.0658671855926514,
-      "eval_logits/rejected": -1.9411793947219849,
-      "eval_logps/chosen": -266.13037109375,
-      "eval_logps/rejected": -228.2694091796875,
-      "eval_loss": 0.5265706777572632,
-      "eval_rewards/accuracies": 0.7459999918937683,
-      "eval_rewards/chosen": -0.1469534933567047,
-      "eval_rewards/margins": 0.7515553832054138,
-      "eval_rewards/rejected": -0.8985088467597961,
-      "eval_runtime": 452.2504,
-      "eval_samples_per_second": 4.422,
-      "eval_steps_per_second": 0.276,
-      "step": 2904
+      "eval_logits/chosen": -2.1217968463897705,
+      "eval_logits/rejected": -2.000995397567749,
+      "eval_logps/chosen": -265.43603515625,
+      "eval_logps/rejected": -224.63743591308594,
+      "eval_loss": 0.5667475461959839,
+      "eval_rewards/accuracies": 0.7059999704360962,
+      "eval_rewards/chosen": -0.07752041518688202,
+      "eval_rewards/margins": 0.4577913284301758,
+      "eval_rewards/rejected": -0.5353116989135742,
+      "eval_runtime": 444.751,
+      "eval_samples_per_second": 4.497,
+      "eval_steps_per_second": 0.281,
+      "step": 726
     },
     {
       "epoch": 3.0,
-      "step": 2904,
+      "step": 726,
       "total_flos": 0.0,
-      "train_loss": 0.5642705829018732,
-      "train_runtime": 71197.0849,
-      "train_samples_per_second": 2.611,
-      "train_steps_per_second": 0.041
+      "train_loss": 0.6037136622532668,
+      "train_runtime": 61083.6691,
+      "train_samples_per_second": 3.043,
+      "train_steps_per_second": 0.012
     }
   ],
   "logging_steps": 10,
-  "max_steps": 2904,
+  "max_steps": 726,
   "num_train_epochs": 3,
   "save_steps": 500,
   "total_flos": 0.0,