{
  "best_metric": 0.42165058851242065,
  "best_model_checkpoint": "/mnt/data/shesj/Trained/RL4CoT/DPO/Meta_13B_numglueCorrect_extend_10lang_v3longer.json/checkpoint-2000",
  "epoch": 0.5,
  "eval_steps": 100,
  "global_step": 2000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1e-08,
      "logits/chosen": -1.932613730430603,
      "logits/rejected": -1.9839212894439697,
      "logps/chosen": -21.470577239990234,
      "logps/rejected": -29.716251373291016,
      "loss": 0.6934,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00021913403179496527,
      "rewards/margins": -0.000711185741238296,
      "rewards/rejected": 0.0009303201222792268,
      "step": 5
    },
    {
      "epoch": 0.0,
      "learning_rate": 2e-08,
      "logits/chosen": -1.9172611236572266,
      "logits/rejected": -1.9188216924667358,
      "logps/chosen": -26.2304744720459,
      "logps/rejected": -34.4053955078125,
      "loss": 0.6927,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.001622863463126123,
      "rewards/margins": 0.0004410705587361008,
      "rewards/rejected": 0.0011817925842478871,
      "step": 10
    },
    {
      "epoch": 0.0,
      "learning_rate": 3e-08,
      "logits/chosen": -1.8847434520721436,
      "logits/rejected": -1.9170100688934326,
      "logps/chosen": -27.089553833007812,
      "logps/rejected": -27.409826278686523,
      "loss": 0.6935,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.001969739096239209,
      "rewards/margins": -0.001115257851779461,
      "rewards/rejected": -0.0008544811280444264,
      "step": 15
    },
    {
      "epoch": 0.01,
      "learning_rate": 4e-08,
      "logits/chosen": -1.9766782522201538,
      "logits/rejected": -2.03544282913208,
      "logps/chosen": -24.145034790039062,
      "logps/rejected": -29.940826416015625,
      "loss": 0.6929,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0021944832988083363,
      "rewards/margins": -0.004052413627505302,
      "rewards/rejected": 0.001857930445112288,
      "step": 20
    },
    {
      "epoch": 0.01,
      "learning_rate": 5e-08,
      "logits/chosen": -1.9266796112060547,
      "logits/rejected": -1.9567911624908447,
      "logps/chosen": -23.183719635009766,
      "logps/rejected": -30.98269271850586,
      "loss": 0.6936,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.000239828834310174,
      "rewards/margins": -0.0014836899936199188,
      "rewards/rejected": 0.0012438612757250667,
      "step": 25
    },
    {
      "epoch": 0.01,
      "learning_rate": 6e-08,
      "logits/chosen": -1.906097650527954,
      "logits/rejected": -1.9456329345703125,
      "logps/chosen": -24.585407257080078,
      "logps/rejected": -38.85862731933594,
      "loss": 0.6929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0007270004716701806,
      "rewards/margins": -0.0001574301568325609,
      "rewards/rejected": 0.0008844308322295547,
      "step": 30
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.999999999999999e-08,
      "logits/chosen": -1.8719940185546875,
      "logits/rejected": -1.9031072854995728,
      "logps/chosen": -26.398326873779297,
      "logps/rejected": -26.643457412719727,
      "loss": 0.6934,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.00045877936645410955,
      "rewards/margins": 0.0018442294094711542,
      "rewards/rejected": -0.0023030086886137724,
      "step": 35
    },
    {
      "epoch": 0.01,
      "learning_rate": 8e-08,
      "logits/chosen": -1.8939685821533203,
      "logits/rejected": -1.9583580493927002,
      "logps/chosen": -25.56913185119629,
      "logps/rejected": -30.93756103515625,
      "loss": 0.693,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.0020271444227546453,
      "rewards/margins": 0.0006519665475934744,
      "rewards/rejected": 0.0013751781079918146,
      "step": 40
    },
    {
      "epoch": 0.01,
      "learning_rate": 9e-08,
      "logits/chosen": -1.9265903234481812,
      "logits/rejected": -1.931652307510376,
      "logps/chosen": -22.316240310668945,
      "logps/rejected": -28.648876190185547,
      "loss": 0.6926,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.002130888169631362,
      "rewards/margins": -0.001213808893226087,
      "rewards/rejected": 0.0033446974121034145,
      "step": 45
    },
    {
      "epoch": 0.01,
      "learning_rate": 1e-07,
      "logits/chosen": -1.8686259984970093,
      "logits/rejected": -1.9298280477523804,
      "logps/chosen": -23.921396255493164,
      "logps/rejected": -31.777599334716797,
      "loss": 0.6926,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.0009275235352106392,
      "rewards/margins": -0.0007253309595398605,
      "rewards/rejected": 0.001652854261919856,
      "step": 50
    },
    {
      "epoch": 0.01,
      "learning_rate": 1.1e-07,
      "logits/chosen": -1.9727710485458374,
      "logits/rejected": -1.9917280673980713,
      "logps/chosen": -22.190906524658203,
      "logps/rejected": -31.58829116821289,
      "loss": 0.6933,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0005749320844188333,
      "rewards/margins": -0.0007338084396906197,
      "rewards/rejected": 0.000158876326167956,
      "step": 55
    },
    {
      "epoch": 0.01,
      "learning_rate": 1.2e-07,
      "logits/chosen": -1.9801238775253296,
      "logits/rejected": -2.005495309829712,
      "logps/chosen": -27.862258911132812,
      "logps/rejected": -25.942520141601562,
      "loss": 0.6926,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0010454825824126601,
      "rewards/margins": -0.0008343329536728561,
      "rewards/rejected": 0.0018798153614625335,
      "step": 60
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.3e-07,
      "logits/chosen": -1.9689757823944092,
      "logits/rejected": -1.9881080389022827,
      "logps/chosen": -21.018028259277344,
      "logps/rejected": -24.91114044189453,
      "loss": 0.6926,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.0009749111486598849,
      "rewards/margins": -0.0024081047158688307,
      "rewards/rejected": 0.0014331938000395894,
      "step": 65
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.3999999999999998e-07,
      "logits/chosen": -1.8362280130386353,
      "logits/rejected": -1.8990787267684937,
      "logps/chosen": -19.452341079711914,
      "logps/rejected": -31.218761444091797,
      "loss": 0.6923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.001963708084076643,
      "rewards/margins": -0.0008558571571484208,
      "rewards/rejected": 0.002819565124809742,
      "step": 70
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.5e-07,
      "logits/chosen": -1.926926612854004,
      "logits/rejected": -1.923295259475708,
      "logps/chosen": -30.22551918029785,
      "logps/rejected": -29.960784912109375,
      "loss": 0.6923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002189194317907095,
      "rewards/margins": 0.00112381752114743,
      "rewards/rejected": 0.0010653762146830559,
      "step": 75
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.6e-07,
      "logits/chosen": -1.9044420719146729,
      "logits/rejected": -2.015061616897583,
      "logps/chosen": -21.34702491760254,
      "logps/rejected": -34.192962646484375,
      "loss": 0.6914,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0026284088380634785,
      "rewards/margins": 0.0029561687260866165,
      "rewards/rejected": -0.0003277602372691035,
      "step": 80
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.7e-07,
      "logits/chosen": -1.9384136199951172,
      "logits/rejected": -1.9547865390777588,
      "logps/chosen": -22.821643829345703,
      "logps/rejected": -27.573680877685547,
      "loss": 0.6915,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0042790137231349945,
      "rewards/margins": 0.003715975908562541,
      "rewards/rejected": 0.0005630379309877753,
      "step": 85
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.8e-07,
      "logits/chosen": -1.934216856956482,
      "logits/rejected": -1.9454669952392578,
      "logps/chosen": -25.801891326904297,
      "logps/rejected": -33.503868103027344,
      "loss": 0.6896,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.005253393668681383,
      "rewards/margins": 0.009739309549331665,
      "rewards/rejected": -0.004485915414988995,
      "step": 90
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -1.9906108379364014,
      "logits/rejected": -1.982187271118164,
      "logps/chosen": -20.808486938476562,
      "logps/rejected": -22.990875244140625,
      "loss": 0.6907,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.007240566425025463,
      "rewards/margins": 0.009975696913897991,
      "rewards/rejected": -0.002735130488872528,
      "step": 95
    },
    {
      "epoch": 0.03,
      "learning_rate": 2e-07,
      "logits/chosen": -1.9031318426132202,
      "logits/rejected": -1.9641430377960205,
      "logps/chosen": -27.743423461914062,
      "logps/rejected": -33.22098159790039,
      "loss": 0.6904,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.007883915677666664,
      "rewards/margins": 0.010012111626565456,
      "rewards/rejected": -0.002128196880221367,
      "step": 100
    },
    {
      "epoch": 0.03,
      "eval_logits/chosen": -2.094839572906494,
      "eval_logits/rejected": -2.1590890884399414,
      "eval_logps/chosen": -27.199792861938477,
      "eval_logps/rejected": -33.886505126953125,
      "eval_loss": 0.6891224384307861,
      "eval_rewards/accuracies": 0.6281948685646057,
      "eval_rewards/chosen": 0.007588829845190048,
      "eval_rewards/margins": 0.007552009075880051,
      "eval_rewards/rejected": 3.682050737552345e-05,
      "eval_runtime": 620.6125,
      "eval_samples_per_second": 32.226,
      "eval_steps_per_second": 0.504,
      "step": 100
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.9999658256641745e-07,
      "logits/chosen": -1.8730093240737915,
      "logits/rejected": -1.9344890117645264,
      "logps/chosen": -20.13459014892578,
      "logps/rejected": -33.617027282714844,
      "loss": 0.6879,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.00706650223582983,
      "rewards/margins": 0.0060449643060564995,
      "rewards/rejected": 0.0010215395595878363,
      "step": 105
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.999863304992469e-07,
      "logits/chosen": -1.9270174503326416,
      "logits/rejected": -1.9574453830718994,
      "logps/chosen": -27.71604347229004,
      "logps/rejected": -25.006996154785156,
      "loss": 0.6878,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006603191141039133,
      "rewards/margins": 0.008059519343078136,
      "rewards/rejected": -0.0014563293661922216,
      "step": 110
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.9996924449920347e-07,
      "logits/chosen": -1.9630305767059326,
      "logits/rejected": -1.9570411443710327,
      "logps/chosen": -26.463726043701172,
      "logps/rejected": -32.061275482177734,
      "loss": 0.6863,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.014216710813343525,
      "rewards/margins": 0.009634643793106079,
      "rewards/rejected": 0.004582066088914871,
      "step": 115
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.999453257340926e-07,
      "logits/chosen": -1.8999840021133423,
      "logits/rejected": -1.9795329570770264,
      "logps/chosen": -25.98947525024414,
      "logps/rejected": -28.222991943359375,
      "loss": 0.6856,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.01323631964623928,
      "rewards/margins": 0.011925769969820976,
      "rewards/rejected": 0.00131055002566427,
      "step": 120
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.9991457583873009e-07,
      "logits/chosen": -1.8996551036834717,
      "logits/rejected": -1.955409049987793,
      "logps/chosen": -22.584300994873047,
      "logps/rejected": -33.386985778808594,
      "loss": 0.6865,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.013754159212112427,
      "rewards/margins": 0.023814406245946884,
      "rewards/rejected": -0.010060247965157032,
      "step": 125
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.9987699691483047e-07,
      "logits/chosen": -2.0506348609924316,
      "logits/rejected": -2.0362610816955566,
      "logps/chosen": -23.7604923248291,
      "logps/rejected": -26.14568519592285,
      "loss": 0.6857,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00911510456353426,
      "rewards/margins": 0.018404189497232437,
      "rewards/rejected": -0.009289086796343327,
      "step": 130
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.9983259153086325e-07,
      "logits/chosen": -1.9324474334716797,
      "logits/rejected": -1.9770278930664062,
      "logps/chosen": -22.019317626953125,
      "logps/rejected": -32.28319549560547,
      "loss": 0.6833,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.01499349158257246,
      "rewards/margins": 0.011077550239861012,
      "rewards/rejected": 0.003915940877050161,
      "step": 135
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9978136272187745e-07,
      "logits/chosen": -1.8848316669464111,
      "logits/rejected": -1.9437576532363892,
      "logps/chosen": -23.589109420776367,
      "logps/rejected": -35.33000946044922,
      "loss": 0.6827,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.023860031738877296,
      "rewards/margins": 0.032189734280109406,
      "rewards/rejected": -0.008329702541232109,
      "step": 140
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.997233139892941e-07,
      "logits/chosen": -1.9527661800384521,
      "logits/rejected": -1.9712032079696655,
      "logps/chosen": -26.96938705444336,
      "logps/rejected": -37.38798522949219,
      "loss": 0.6792,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.029867831617593765,
      "rewards/margins": 0.0479811429977417,
      "rewards/rejected": -0.018113311380147934,
      "step": 145
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9965844930066698e-07,
      "logits/chosen": -1.9537423849105835,
      "logits/rejected": -2.0033607482910156,
      "logps/chosen": -24.348556518554688,
      "logps/rejected": -33.808815002441406,
      "loss": 0.6799,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.020862484350800514,
      "rewards/margins": 0.031281448900699615,
      "rewards/rejected": -0.010418963618576527,
      "step": 150
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9958677308941136e-07,
      "logits/chosen": -1.8261839151382446,
      "logits/rejected": -1.9302173852920532,
      "logps/chosen": -29.987186431884766,
      "logps/rejected": -35.99931335449219,
      "loss": 0.6799,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.019242849200963974,
      "rewards/margins": 0.02805192768573761,
      "rewards/rejected": -0.008809077553451061,
      "step": 155
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9950829025450114e-07,
      "logits/chosen": -1.9289829730987549,
      "logits/rejected": -1.947072982788086,
      "logps/chosen": -28.187902450561523,
      "logps/rejected": -34.05443572998047,
      "loss": 0.6765,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.021738989278674126,
      "rewards/margins": 0.030259471386671066,
      "rewards/rejected": -0.008520480245351791,
      "step": 160
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9942300616013377e-07,
      "logits/chosen": -1.9248237609863281,
      "logits/rejected": -1.9657666683197021,
      "logps/chosen": -24.501365661621094,
      "logps/rejected": -35.636444091796875,
      "loss": 0.6758,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.027700508013367653,
      "rewards/margins": 0.04301094263792038,
      "rewards/rejected": -0.015310434624552727,
      "step": 165
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.993309266353638e-07,
      "logits/chosen": -1.9632354974746704,
      "logits/rejected": -2.003535747528076,
      "logps/chosen": -23.00834846496582,
      "logps/rejected": -35.10881805419922,
      "loss": 0.672,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.022433171048760414,
      "rewards/margins": 0.03842338174581528,
      "rewards/rejected": -0.01599021628499031,
      "step": 170
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.992320579737045e-07,
      "logits/chosen": -1.924059510231018,
      "logits/rejected": -1.9485867023468018,
      "logps/chosen": -23.606395721435547,
      "logps/rejected": -34.96836471557617,
      "loss": 0.6707,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.020746735855937004,
      "rewards/margins": 0.03996802866458893,
      "rewards/rejected": -0.019221294671297073,
      "step": 175
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9912640693269751e-07,
      "logits/chosen": -1.8697850704193115,
      "logits/rejected": -1.8567920923233032,
      "logps/chosen": -21.75259017944336,
      "logps/rejected": -30.40744400024414,
      "loss": 0.6734,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.01639285869896412,
      "rewards/margins": 0.043617911636829376,
      "rewards/rejected": -0.027225052937865257,
      "step": 180
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.9901398073345117e-07,
      "logits/chosen": -1.9471813440322876,
      "logits/rejected": -1.9353182315826416,
      "logps/chosen": -25.765697479248047,
      "logps/rejected": -29.618860244750977,
      "loss": 0.668,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02380688488483429,
      "rewards/margins": 0.049822211265563965,
      "rewards/rejected": -0.026015322655439377,
      "step": 185
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.9889478706014683e-07,
      "logits/chosen": -1.9487874507904053,
      "logits/rejected": -2.0098330974578857,
      "logps/chosen": -21.570528030395508,
      "logps/rejected": -33.25171661376953,
      "loss": 0.6662,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.026225054636597633,
      "rewards/margins": 0.06671885401010513,
      "rewards/rejected": -0.04049379751086235,
      "step": 190
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.9876883405951376e-07,
      "logits/chosen": -1.8715938329696655,
      "logits/rejected": -1.9319870471954346,
      "logps/chosen": -26.760677337646484,
      "logps/rejected": -33.19749069213867,
      "loss": 0.668,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.023825686424970627,
      "rewards/margins": 0.051926322281360626,
      "rewards/rejected": -0.02810063399374485,
      "step": 195
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.9863613034027222e-07,
      "logits/chosen": -1.8788840770721436,
      "logits/rejected": -1.8988853693008423,
      "logps/chosen": -25.21091079711914,
      "logps/rejected": -29.254589080810547,
      "loss": 0.6593,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.018495548516511917,
      "rewards/margins": 0.06201624125242233,
      "rewards/rejected": -0.04352068901062012,
      "step": 200
    },
    {
      "epoch": 0.05,
      "eval_logits/chosen": -2.055267333984375,
      "eval_logits/rejected": -2.1217849254608154,
      "eval_logps/chosen": -27.035058975219727,
      "eval_logps/rejected": -34.23577880859375,
      "eval_loss": 0.6638627648353577,
      "eval_rewards/accuracies": 0.7012779712677002,
      "eval_rewards/chosen": 0.024062257260084152,
      "eval_rewards/margins": 0.05895264819264412,
      "eval_rewards/rejected": -0.03489040210843086,
      "eval_runtime": 624.7249,
      "eval_samples_per_second": 32.014,
      "eval_steps_per_second": 0.501,
      "step": 200
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.9849668497254518e-07,
      "logits/chosen": -1.934955358505249,
      "logits/rejected": -1.962149977684021,
      "logps/chosen": -27.25844955444336,
      "logps/rejected": -34.96973419189453,
      "loss": 0.6654,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0293357465416193,
      "rewards/margins": 0.07454489171504974,
      "rewards/rejected": -0.045209143310785294,
      "step": 205
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.9835050748723822e-07,
      "logits/chosen": -1.9069238901138306,
      "logits/rejected": -1.9731998443603516,
      "logps/chosen": -24.134227752685547,
      "logps/rejected": -35.1556282043457,
      "loss": 0.6578,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.034762006253004074,
      "rewards/margins": 0.08553215861320496,
      "rewards/rejected": -0.05077015236020088,
      "step": 210
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.9819760787538837e-07,
      "logits/chosen": -1.871299147605896,
      "logits/rejected": -1.9281699657440186,
      "logps/chosen": -23.06464195251465,
      "logps/rejected": -31.450504302978516,
      "loss": 0.6563,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.022200025618076324,
      "rewards/margins": 0.08272843807935715,
      "rewards/rejected": -0.06052841991186142,
      "step": 215
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.9803799658748093e-07,
      "logits/chosen": -1.8807361125946045,
      "logits/rejected": -1.9069770574569702,
      "logps/chosen": -30.42557716369629,
      "logps/rejected": -34.229042053222656,
      "loss": 0.6562,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.022107623517513275,
      "rewards/margins": 0.08593301475048065,
      "rewards/rejected": -0.06382538378238678,
      "step": 220
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.9787168453273545e-07,
      "logits/chosen": -1.8091869354248047,
      "logits/rejected": -1.8331130743026733,
      "logps/chosen": -25.224321365356445,
      "logps/rejected": -30.4110107421875,
      "loss": 0.6554,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0017654001712799072,
      "rewards/margins": 0.052088379859924316,
      "rewards/rejected": -0.053853780031204224,
      "step": 225
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.9769868307835993e-07,
      "logits/chosen": -1.9052034616470337,
      "logits/rejected": -1.948081612586975,
      "logps/chosen": -25.112337112426758,
      "logps/rejected": -33.975223541259766,
      "loss": 0.6495,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.021494796499609947,
      "rewards/margins": 0.09832977503538132,
      "rewards/rejected": -0.07683496922254562,
      "step": 230
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.9751900404877398e-07,
      "logits/chosen": -1.9521013498306274,
      "logits/rejected": -2.0165581703186035,
      "logps/chosen": -24.376867294311523,
      "logps/rejected": -31.251483917236328,
      "loss": 0.6525,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.03034518100321293,
      "rewards/margins": 0.07310134917497635,
      "rewards/rejected": -0.04275617375969887,
      "step": 235
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.9733265972480058e-07,
      "logits/chosen": -1.8078219890594482,
      "logits/rejected": -1.84963858127594,
      "logps/chosen": -26.859643936157227,
      "logps/rejected": -39.10490036010742,
      "loss": 0.6486,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02189042791724205,
      "rewards/margins": 0.10619709640741348,
      "rewards/rejected": -0.08430664986371994,
      "step": 240
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.9713966284282674e-07,
      "logits/chosen": -1.8735325336456299,
      "logits/rejected": -1.8725016117095947,
      "logps/chosen": -26.303363800048828,
      "logps/rejected": -34.02263641357422,
      "loss": 0.6508,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.002572746481746435,
      "rewards/margins": 0.08564522862434387,
      "rewards/rejected": -0.08821798115968704,
      "step": 245
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.9694002659393302e-07,
      "logits/chosen": -1.8189780712127686,
      "logits/rejected": -1.8682382106781006,
      "logps/chosen": -23.835880279541016,
      "logps/rejected": -33.8657112121582,
      "loss": 0.6466,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.0005129946512170136,
      "rewards/margins": 0.1281592845916748,
      "rewards/rejected": -0.1276462972164154,
      "step": 250
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.9673376462299182e-07,
      "logits/chosen": -1.8371633291244507,
      "logits/rejected": -1.9105684757232666,
      "logps/chosen": -21.575557708740234,
      "logps/rejected": -34.56597137451172,
      "loss": 0.6423,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.007006521336734295,
      "rewards/margins": 0.09751694649457932,
      "rewards/rejected": -0.09051042795181274,
      "step": 255
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.9652089102773487e-07,
      "logits/chosen": -1.8369518518447876,
      "logits/rejected": -1.8599889278411865,
      "logps/chosen": -28.181167602539062,
      "logps/rejected": -31.22989273071289,
      "loss": 0.6411,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.025061314925551414,
      "rewards/margins": 0.1553947627544403,
      "rewards/rejected": -0.13033345341682434,
      "step": 260
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.963014203577896e-07,
      "logits/chosen": -1.8845443725585938,
      "logits/rejected": -1.8823344707489014,
      "logps/chosen": -29.47086524963379,
      "logps/rejected": -35.28539276123047,
      "loss": 0.6358,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.017991356551647186,
      "rewards/margins": 0.1316610425710678,
      "rewards/rejected": -0.11366970837116241,
      "step": 265
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.9607536761368482e-07,
      "logits/chosen": -1.8176910877227783,
      "logits/rejected": -1.8355636596679688,
      "logps/chosen": -25.24968910217285,
      "logps/rejected": -33.39419174194336,
      "loss": 0.6364,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.005754109006375074,
      "rewards/margins": 0.15424194931983948,
      "rewards/rejected": -0.148487851023674,
      "step": 270
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.9584274824582527e-07,
      "logits/chosen": -1.850232720375061,
      "logits/rejected": -1.9631538391113281,
      "logps/chosen": -24.343835830688477,
      "logps/rejected": -33.17558670043945,
      "loss": 0.6367,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0063208164647221565,
      "rewards/margins": 0.12795674800872803,
      "rewards/rejected": -0.1342775821685791,
      "step": 275
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.9560357815343574e-07,
      "logits/chosen": -1.855934739112854,
      "logits/rejected": -1.874148964881897,
      "logps/chosen": -22.502605438232422,
      "logps/rejected": -28.427841186523438,
      "loss": 0.6266,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.024706363677978516,
      "rewards/margins": 0.14013074338436127,
      "rewards/rejected": -0.1648370921611786,
      "step": 280
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.9535787368347442e-07,
      "logits/chosen": -1.8721545934677124,
      "logits/rejected": -1.900692343711853,
      "logps/chosen": -25.19928550720215,
      "logps/rejected": -34.262535095214844,
      "loss": 0.6278,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.032768525183200836,
      "rewards/margins": 0.1182633638381958,
      "rewards/rejected": -0.15103188157081604,
      "step": 285
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.9510565162951537e-07,
      "logits/chosen": -1.8483591079711914,
      "logits/rejected": -1.8752696514129639,
      "logps/chosen": -20.428386688232422,
      "logps/rejected": -36.55931854248047,
      "loss": 0.6365,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.036745864897966385,
      "rewards/margins": 0.11163260042667389,
      "rewards/rejected": -0.14837846159934998,
      "step": 290
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.9484692923060094e-07,
      "logits/chosen": -1.8256515264511108,
      "logits/rejected": -1.9205400943756104,
      "logps/chosen": -23.362619400024414,
      "logps/rejected": -31.759166717529297,
      "loss": 0.6362,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0479150153696537,
      "rewards/margins": 0.17140944302082062,
      "rewards/rejected": -0.21932446956634521,
      "step": 295
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.9458172417006346e-07,
      "logits/chosen": -1.7794628143310547,
      "logits/rejected": -1.7937818765640259,
      "logps/chosen": -25.80463218688965,
      "logps/rejected": -35.583526611328125,
      "loss": 0.6272,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07188913971185684,
      "rewards/margins": 0.11425880342721939,
      "rewards/rejected": -0.18614795804023743,
      "step": 300
    },
    {
      "epoch": 0.07,
      "eval_logits/chosen": -1.982098937034607,
      "eval_logits/rejected": -2.0512313842773438,
      "eval_logps/chosen": -27.691436767578125,
      "eval_logps/rejected": -35.85608673095703,
      "eval_loss": 0.6240853667259216,
      "eval_rewards/accuracies": 0.710463285446167,
      "eval_rewards/chosen": -0.041575513780117035,
      "eval_rewards/margins": 0.15534532070159912,
      "eval_rewards/rejected": -0.19692084193229675,
      "eval_runtime": 628.2953,
      "eval_samples_per_second": 31.832,
      "eval_steps_per_second": 0.498,
      "step": 300
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.943100545743165e-07,
      "logits/chosen": -1.760960340499878,
      "logits/rejected": -1.850353479385376,
      "logps/chosen": -21.75943374633789,
      "logps/rejected": -36.918704986572266,
      "loss": 0.6199,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.015559827908873558,
      "rewards/margins": 0.18496355414390564,
      "rewards/rejected": -0.20052340626716614,
      "step": 305
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.9403193901161612e-07,
      "logits/chosen": -1.8915361166000366,
      "logits/rejected": -1.9225527048110962,
      "logps/chosen": -25.573026657104492,
      "logps/rejected": -33.04106903076172,
      "loss": 0.6159,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.014883624389767647,
      "rewards/margins": 0.21884708106517792,
      "rewards/rejected": -0.2337307184934616,
      "step": 310
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.9374739649079154e-07,
      "logits/chosen": -1.7862402200698853,
      "logits/rejected": -1.8242515325546265,
      "logps/chosen": -22.60375213623047,
      "logps/rejected": -28.420673370361328,
      "loss": 0.6187,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.027039390057325363,
      "rewards/margins": 0.17555785179138184,
      "rewards/rejected": -0.2025972604751587,
      "step": 315
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.9345644645994608e-07,
      "logits/chosen": -1.8331549167633057,
      "logits/rejected": -1.8311151266098022,
      "logps/chosen": -25.251590728759766,
      "logps/rejected": -35.493736267089844,
      "loss": 0.6248,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.04046057537198067,
      "rewards/margins": 0.22045457363128662,
      "rewards/rejected": -0.2609151601791382,
      "step": 320
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.9315910880512788e-07,
      "logits/chosen": -1.797435998916626,
      "logits/rejected": -1.8667093515396118,
      "logps/chosen": -25.154064178466797,
      "logps/rejected": -32.20864486694336,
      "loss": 0.6174,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.055550407618284225,
      "rewards/margins": 0.21562273800373077,
      "rewards/rejected": -0.2711731493473053,
      "step": 325
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.928554038489707e-07,
      "logits/chosen": -1.6940425634384155,
      "logits/rejected": -1.744011640548706,
      "logps/chosen": -28.64211654663086,
      "logps/rejected": -36.03167724609375,
      "loss": 0.6163,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.10549549013376236,
      "rewards/margins": 0.20552174746990204,
      "rewards/rejected": -0.3110172152519226,
      "step": 330
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.9254535234930483e-07,
      "logits/chosen": -1.8530375957489014,
      "logits/rejected": -1.8774633407592773,
      "logps/chosen": -24.82217788696289,
      "logps/rejected": -34.68824005126953,
      "loss": 0.6181,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.10517600923776627,
      "rewards/margins": 0.17466488480567932,
      "rewards/rejected": -0.279840886592865,
      "step": 335
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.9222897549773846e-07,
      "logits/chosen": -1.7554054260253906,
      "logits/rejected": -1.7775160074234009,
      "logps/chosen": -33.513790130615234,
      "logps/rejected": -38.4843864440918,
      "loss": 0.6049,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.11450239270925522,
      "rewards/margins": 0.21290548145771027,
      "rewards/rejected": -0.3274078667163849,
      "step": 340
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.9190629491820908e-07,
      "logits/chosen": -1.8356316089630127,
      "logits/rejected": -1.8637549877166748,
      "logps/chosen": -23.877962112426758,
      "logps/rejected": -37.12334060668945,
      "loss": 0.6083,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1105366125702858,
      "rewards/margins": 0.2586100697517395,
      "rewards/rejected": -0.3691467046737671,
      "step": 345
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.9157733266550572e-07,
      "logits/chosen": -1.8048322200775146,
      "logits/rejected": -1.8302946090698242,
      "logps/chosen": -23.856672286987305,
      "logps/rejected": -29.770706176757812,
      "loss": 0.5973,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10654797405004501,
      "rewards/margins": 0.24383535981178284,
      "rewards/rejected": -0.35038334131240845,
      "step": 350
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.9124211122376135e-07,
      "logits/chosen": -1.732420563697815,
      "logits/rejected": -1.8045669794082642,
      "logps/chosen": -27.87255859375,
      "logps/rejected": -47.80708694458008,
      "loss": 0.5929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06705266982316971,
      "rewards/margins": 0.3158617913722992,
      "rewards/rejected": -0.3829144537448883,
      "step": 355
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.9090065350491624e-07,
      "logits/chosen": -1.6546525955200195,
      "logits/rejected": -1.715736746788025,
      "logps/chosen": -23.659460067749023,
      "logps/rejected": -41.387020111083984,
      "loss": 0.5963,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08905567228794098,
      "rewards/margins": 0.23344358801841736,
      "rewards/rejected": -0.32249927520751953,
      "step": 360
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.905529828471519e-07,
      "logits/chosen": -1.7011845111846924,
      "logits/rejected": -1.7666597366333008,
      "logps/chosen": -22.377338409423828,
      "logps/rejected": -40.995094299316406,
      "loss": 0.5949,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.052688825875520706,
      "rewards/margins": 0.33948010206222534,
      "rewards/rejected": -0.39216893911361694,
      "step": 365
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.901991230132959e-07,
      "logits/chosen": -1.810546875,
      "logits/rejected": -1.8701536655426025,
      "logps/chosen": -25.03759765625,
      "logps/rejected": -37.37371826171875,
      "loss": 0.6012,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04496913030743599,
      "rewards/margins": 0.305519163608551,
      "rewards/rejected": -0.3504883348941803,
      "step": 370
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.8983909818919788e-07,
      "logits/chosen": -1.8299157619476318,
      "logits/rejected": -1.8482189178466797,
      "logps/chosen": -25.418384552001953,
      "logps/rejected": -38.091896057128906,
      "loss": 0.5861,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.1421210616827011,
      "rewards/margins": 0.312614381313324,
      "rewards/rejected": -0.4547354280948639,
      "step": 375
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.8947293298207635e-07,
      "logits/chosen": -1.6938432455062866,
      "logits/rejected": -1.7125260829925537,
      "logps/chosen": -23.936098098754883,
      "logps/rejected": -40.08384323120117,
      "loss": 0.5878,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.17097172141075134,
      "rewards/margins": 0.3229272663593292,
      "rewards/rejected": -0.49389901757240295,
      "step": 380
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.8910065241883678e-07,
      "logits/chosen": -1.744314193725586,
      "logits/rejected": -1.8095916509628296,
      "logps/chosen": -29.93277359008789,
      "logps/rejected": -34.6977424621582,
      "loss": 0.579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.20626993477344513,
      "rewards/margins": 0.2772943675518036,
      "rewards/rejected": -0.48356422781944275,
      "step": 385
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.8872228194436116e-07,
      "logits/chosen": -1.7233030796051025,
      "logits/rejected": -1.7732064723968506,
      "logps/chosen": -27.3117733001709,
      "logps/rejected": -28.547231674194336,
      "loss": 0.568,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10007091611623764,
      "rewards/margins": 0.3294708728790283,
      "rewards/rejected": -0.42954176664352417,
      "step": 390
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.8833784741976886e-07,
      "logits/chosen": -1.771653175354004,
      "logits/rejected": -1.793135404586792,
      "logps/chosen": -25.894977569580078,
      "logps/rejected": -33.55714797973633,
      "loss": 0.5755,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25736480951309204,
      "rewards/margins": 0.29228243231773376,
      "rewards/rejected": -0.5496472120285034,
      "step": 395
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.8794737512064888e-07,
      "logits/chosen": -1.744651436805725,
      "logits/rejected": -1.8249235153198242,
      "logps/chosen": -31.24419593811035,
      "logps/rejected": -39.73127365112305,
      "loss": 0.5806,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.23053815960884094,
      "rewards/margins": 0.34528475999832153,
      "rewards/rejected": -0.5758228898048401,
      "step": 400
    },
    {
      "epoch": 0.1,
      "eval_logits/chosen": -1.8933817148208618,
      "eval_logits/rejected": -1.9633389711380005,
      "eval_logps/chosen": -29.42821502685547,
      "eval_logps/rejected": -39.12174987792969,
      "eval_loss": 0.5772050023078918,
      "eval_rewards/accuracies": 0.7336261868476868,
      "eval_rewards/chosen": -0.2152535617351532,
      "eval_rewards/margins": 0.3082338869571686,
      "eval_rewards/rejected": -0.5234874486923218,
      "eval_runtime": 638.6461,
      "eval_samples_per_second": 31.316,
      "eval_steps_per_second": 0.49,
      "step": 400
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.875508917352643e-07,
      "logits/chosen": -1.7457950115203857,
      "logits/rejected": -1.7916702032089233,
      "logps/chosen": -28.520294189453125,
      "logps/rejected": -34.520931243896484,
      "loss": 0.5825,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.28079259395599365,
      "rewards/margins": 0.21762211620807648,
      "rewards/rejected": -0.49841469526290894,
      "step": 405
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.871484243627277e-07,
      "logits/chosen": -1.7049251794815063,
      "logits/rejected": -1.7206684350967407,
      "logps/chosen": -30.74674415588379,
      "logps/rejected": -39.634422302246094,
      "loss": 0.5677,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2962929606437683,
      "rewards/margins": 0.273542582988739,
      "rewards/rejected": -0.5698355436325073,
      "step": 410
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.867400005111495e-07,
      "logits/chosen": -1.771710991859436,
      "logits/rejected": -1.7792901992797852,
      "logps/chosen": -24.596803665161133,
      "logps/rejected": -34.267784118652344,
      "loss": 0.5644,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.27595609426498413,
      "rewards/margins": 0.3286646008491516,
      "rewards/rejected": -0.6046206951141357,
      "step": 415
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.8632564809575738e-07,
      "logits/chosen": -1.6727784872055054,
      "logits/rejected": -1.763475775718689,
      "logps/chosen": -25.139801025390625,
      "logps/rejected": -46.42198944091797,
      "loss": 0.5476,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.2259303778409958,
      "rewards/margins": 0.4986729025840759,
      "rewards/rejected": -0.7246032953262329,
      "step": 420
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.859053954369885e-07,
      "logits/chosen": -1.7414724826812744,
      "logits/rejected": -1.7918275594711304,
      "logps/chosen": -24.54043960571289,
      "logps/rejected": -50.85098648071289,
      "loss": 0.5723,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.23223881423473358,
      "rewards/margins": 0.47227388620376587,
      "rewards/rejected": -0.7045127153396606,
      "step": 425
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.854792712585539e-07,
      "logits/chosen": -1.645516037940979,
      "logits/rejected": -1.693722128868103,
      "logps/chosen": -25.945995330810547,
      "logps/rejected": -45.90827560424805,
      "loss": 0.5483,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3240446448326111,
      "rewards/margins": 0.5380430221557617,
      "rewards/rejected": -0.8620877265930176,
      "step": 430
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.8504730468547506e-07,
      "logits/chosen": -1.649815320968628,
      "logits/rejected": -1.6857585906982422,
      "logps/chosen": -23.51533317565918,
      "logps/rejected": -34.9762077331543,
      "loss": 0.561,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3314105272293091,
      "rewards/margins": 0.3385566771030426,
      "rewards/rejected": -0.6699672341346741,
      "step": 435
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.846095252420935e-07,
      "logits/chosen": -1.6907812356948853,
      "logits/rejected": -1.6901006698608398,
      "logps/chosen": -26.697391510009766,
      "logps/rejected": -35.67798614501953,
      "loss": 0.5664,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23831510543823242,
      "rewards/margins": 0.37860625982284546,
      "rewards/rejected": -0.6169213056564331,
      "step": 440
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.841659628500527e-07,
      "logits/chosen": -1.7292283773422241,
      "logits/rejected": -1.761568307876587,
      "logps/chosen": -31.302043914794922,
      "logps/rejected": -42.45463943481445,
      "loss": 0.5407,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.361013263463974,
      "rewards/margins": 0.4722154140472412,
      "rewards/rejected": -0.833228588104248,
      "step": 445
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.8371664782625284e-07,
      "logits/chosen": -1.641015648841858,
      "logits/rejected": -1.6467399597167969,
      "logps/chosen": -28.206012725830078,
      "logps/rejected": -35.136924743652344,
      "loss": 0.5402,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2760232388973236,
      "rewards/margins": 0.4164581298828125,
      "rewards/rejected": -0.6924813985824585,
      "step": 450
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.8326161088077904e-07,
      "logits/chosen": -1.6614574193954468,
      "logits/rejected": -1.7363828420639038,
      "logps/chosen": -22.726844787597656,
      "logps/rejected": -38.93574142456055,
      "loss": 0.543,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.32301831245422363,
      "rewards/margins": 0.4177462160587311,
      "rewards/rejected": -0.7407644987106323,
      "step": 455
    },
    {
      "epoch": 0.12,
      "learning_rate": 1.82800883114802e-07,
      "logits/chosen": -1.6789630651474,
      "logits/rejected": -1.7167510986328125,
      "logps/chosen": -27.532705307006836,
      "logps/rejected": -46.597877502441406,
      "loss": 0.5481,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.32634788751602173,
      "rewards/margins": 0.510971188545227,
      "rewards/rejected": -0.8373190760612488,
      "step": 460
    },
    {
      "epoch": 0.12,
      "learning_rate": 1.8233449601845256e-07,
      "logits/chosen": -1.6799752712249756,
      "logits/rejected": -1.746375322341919,
      "logps/chosen": -31.223033905029297,
      "logps/rejected": -39.73358917236328,
      "loss": 0.5161,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.41861018538475037,
      "rewards/margins": 0.43975359201431274,
      "rewards/rejected": -0.8583638072013855,
      "step": 465
    },
    {
      "epoch": 0.12,
      "learning_rate": 1.8186248146866925e-07,
      "logits/chosen": -1.7066395282745361,
      "logits/rejected": -1.7620627880096436,
      "logps/chosen": -27.414352416992188,
      "logps/rejected": -46.25200653076172,
      "loss": 0.5246,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.38034266233444214,
      "rewards/margins": 0.5347098112106323,
      "rewards/rejected": -0.9150525331497192,
      "step": 470
    },
    {
      "epoch": 0.12,
      "learning_rate": 1.8138487172701948e-07,
      "logits/chosen": -1.6645714044570923,
      "logits/rejected": -1.6783406734466553,
      "logps/chosen": -25.467668533325195,
      "logps/rejected": -42.95486831665039,
      "loss": 0.535,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.3398158550262451,
      "rewards/margins": 0.697130560874939,
      "rewards/rejected": -1.0369462966918945,
      "step": 475
    },
    {
      "epoch": 0.12,
      "learning_rate": 1.8090169943749475e-07,
      "logits/chosen": -1.717818021774292,
      "logits/rejected": -1.761845350265503,
      "logps/chosen": -26.850399017333984,
      "logps/rejected": -42.4122200012207,
      "loss": 0.5404,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.5769149661064148,
      "rewards/margins": 0.5164484977722168,
      "rewards/rejected": -1.0933634042739868,
      "step": 480
    },
    {
      "epoch": 0.12,
      "learning_rate": 1.8041299762427914e-07,
      "logits/chosen": -1.5914769172668457,
      "logits/rejected": -1.6343708038330078,
      "logps/chosen": -28.54681396484375,
      "logps/rejected": -37.71539306640625,
      "loss": 0.5383,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3866100609302521,
      "rewards/margins": 0.4422999918460846,
      "rewards/rejected": -0.8289100527763367,
      "step": 485
    },
    {
      "epoch": 0.12,
      "learning_rate": 1.7991879968949247e-07,
      "logits/chosen": -1.6161174774169922,
      "logits/rejected": -1.6184766292572021,
      "logps/chosen": -34.267799377441406,
      "logps/rejected": -42.01969528198242,
      "loss": 0.5468,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5237308740615845,
      "rewards/margins": 0.4367442727088928,
      "rewards/rejected": -0.960474967956543,
      "step": 490
    },
    {
      "epoch": 0.12,
      "learning_rate": 1.794191394109071e-07,
      "logits/chosen": -1.6576045751571655,
      "logits/rejected": -1.727998971939087,
      "logps/chosen": -30.341400146484375,
      "logps/rejected": -38.22698211669922,
      "loss": 0.5517,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.4688238203525543,
      "rewards/margins": 0.46207141876220703,
      "rewards/rejected": -0.9308953285217285,
      "step": 495
    },
    {
      "epoch": 0.12,
      "learning_rate": 1.7891405093963936e-07,
      "logits/chosen": -1.6647958755493164,
      "logits/rejected": -1.6835273504257202,
      "logps/chosen": -33.16368865966797,
      "logps/rejected": -39.30922317504883,
      "loss": 0.5317,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5163825750350952,
      "rewards/margins": 0.40946492552757263,
      "rewards/rejected": -0.9258475303649902,
      "step": 500
    },
    {
      "epoch": 0.12,
      "eval_logits/chosen": -1.815245270729065,
      "eval_logits/rejected": -1.8842970132827759,
      "eval_logps/chosen": -32.26655578613281,
      "eval_logps/rejected": -44.04635238647461,
      "eval_loss": 0.5326652526855469,
      "eval_rewards/accuracies": 0.7432108521461487,
      "eval_rewards/chosen": -0.49908730387687683,
      "eval_rewards/margins": 0.5168607831001282,
      "eval_rewards/rejected": -1.0159480571746826,
      "eval_runtime": 642.3458,
      "eval_samples_per_second": 31.136,
      "eval_steps_per_second": 0.487,
      "step": 500
    },
    {
      "epoch": 0.13,
      "learning_rate": 1.7840356879781529e-07,
      "logits/chosen": -1.679195761680603,
      "logits/rejected": -1.7675142288208008,
      "logps/chosen": -35.0252799987793,
      "logps/rejected": -42.8774299621582,
      "loss": 0.5327,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5109128355979919,
      "rewards/margins": 0.5178115963935852,
      "rewards/rejected": -1.0287244319915771,
      "step": 505
    },
    {
      "epoch": 0.13,
      "learning_rate": 1.7788772787621125e-07,
      "logits/chosen": -1.5657098293304443,
      "logits/rejected": -1.6311728954315186,
      "logps/chosen": -25.362163543701172,
      "logps/rejected": -38.46403884887695,
      "loss": 0.5062,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.43067851662635803,
      "rewards/margins": 0.4838063716888428,
      "rewards/rejected": -0.9144848585128784,
      "step": 510
    },
    {
      "epoch": 0.13,
      "learning_rate": 1.7736656343186894e-07,
      "logits/chosen": -1.680122971534729,
      "logits/rejected": -1.6815965175628662,
      "logps/chosen": -32.139137268066406,
      "logps/rejected": -46.62664794921875,
      "loss": 0.5077,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.5889835357666016,
      "rewards/margins": 0.4796602725982666,
      "rewards/rejected": -1.0686438083648682,
      "step": 515
    },
    {
      "epoch": 0.13,
      "learning_rate": 1.768401110856859e-07,
      "logits/chosen": -1.6242386102676392,
      "logits/rejected": -1.6613088846206665,
      "logps/chosen": -27.435977935791016,
      "logps/rejected": -44.428260803222656,
      "loss": 0.5094,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.4436129033565521,
      "rewards/margins": 0.7085938453674316,
      "rewards/rejected": -1.1522066593170166,
      "step": 520
    },
    {
      "epoch": 0.13,
      "learning_rate": 1.7630840681998066e-07,
      "logits/chosen": -1.6392037868499756,
      "logits/rejected": -1.6771599054336548,
      "logps/chosen": -30.593151092529297,
      "logps/rejected": -38.12665557861328,
      "loss": 0.5225,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5331975221633911,
      "rewards/margins": 0.5177993178367615,
      "rewards/rejected": -1.0509967803955078,
      "step": 525
    },
    {
      "epoch": 0.13,
      "learning_rate": 1.7577148697603348e-07,
      "logits/chosen": -1.653996229171753,
      "logits/rejected": -1.6901382207870483,
      "logps/chosen": -28.415035247802734,
      "logps/rejected": -43.633609771728516,
      "loss": 0.5314,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.5270938873291016,
      "rewards/margins": 0.6642566919326782,
      "rewards/rejected": -1.1913506984710693,
      "step": 530
    },
    {
      "epoch": 0.13,
      "learning_rate": 1.7522938825160247e-07,
      "logits/chosen": -1.5846532583236694,
      "logits/rejected": -1.6490291357040405,
      "logps/chosen": -29.801273345947266,
      "logps/rejected": -42.26642990112305,
      "loss": 0.5399,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.44840726256370544,
      "rewards/margins": 0.6187388300895691,
      "rewards/rejected": -1.0671460628509521,
      "step": 535
    },
    {
      "epoch": 0.14,
      "learning_rate": 1.7468214769841538e-07,
      "logits/chosen": -1.68500554561615,
      "logits/rejected": -1.7133398056030273,
      "logps/chosen": -32.34290313720703,
      "logps/rejected": -42.54985046386719,
      "loss": 0.5033,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.6745347380638123,
      "rewards/margins": 0.5442879796028137,
      "rewards/rejected": -1.218822717666626,
      "step": 540
    },
    {
      "epoch": 0.14,
      "learning_rate": 1.7412980271963708e-07,
      "logits/chosen": -1.6446959972381592,
      "logits/rejected": -1.6709632873535156,
      "logps/chosen": -32.75804901123047,
      "logps/rejected": -41.7109260559082,
      "loss": 0.503,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6608411073684692,
      "rewards/margins": 0.5818773508071899,
      "rewards/rejected": -1.2427183389663696,
      "step": 545
    },
    {
      "epoch": 0.14,
      "learning_rate": 1.7357239106731316e-07,
      "logits/chosen": -1.5884132385253906,
      "logits/rejected": -1.6537491083145142,
      "logps/chosen": -33.371700286865234,
      "logps/rejected": -49.154361724853516,
      "loss": 0.5306,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.7718857526779175,
      "rewards/margins": 0.5803537368774414,
      "rewards/rejected": -1.3522393703460693,
      "step": 550
    },
    {
      "epoch": 0.14,
      "learning_rate": 1.7300995083978961e-07,
      "logits/chosen": -1.5982125997543335,
      "logits/rejected": -1.6282612085342407,
      "logps/chosen": -30.794885635375977,
      "logps/rejected": -46.758148193359375,
      "loss": 0.4896,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7337169051170349,
      "rewards/margins": 0.6327022314071655,
      "rewards/rejected": -1.3664191961288452,
      "step": 555
    },
    {
      "epoch": 0.14,
      "learning_rate": 1.724425204791089e-07,
      "logits/chosen": -1.573806881904602,
      "logits/rejected": -1.6021778583526611,
      "logps/chosen": -32.741111755371094,
      "logps/rejected": -41.9412727355957,
      "loss": 0.5134,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6988576650619507,
      "rewards/margins": 0.574633777141571,
      "rewards/rejected": -1.273491382598877,
      "step": 560
    },
    {
      "epoch": 0.14,
      "learning_rate": 1.7187013876838238e-07,
      "logits/chosen": -1.633247971534729,
      "logits/rejected": -1.6215112209320068,
      "logps/chosen": -28.081262588500977,
      "logps/rejected": -45.24711227416992,
      "loss": 0.4877,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.5856736302375793,
      "rewards/margins": 0.7724601030349731,
      "rewards/rejected": -1.3581336736679077,
      "step": 565
    },
    {
      "epoch": 0.14,
      "learning_rate": 1.712928448291397e-07,
      "logits/chosen": -1.654266357421875,
      "logits/rejected": -1.6817476749420166,
      "logps/chosen": -33.884037017822266,
      "logps/rejected": -48.278202056884766,
      "loss": 0.5093,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6435326337814331,
      "rewards/margins": 0.8194659948348999,
      "rewards/rejected": -1.462998628616333,
      "step": 570
    },
    {
      "epoch": 0.14,
      "learning_rate": 1.7071067811865473e-07,
      "logits/chosen": -1.6069905757904053,
      "logits/rejected": -1.6091623306274414,
      "logps/chosen": -32.96613311767578,
      "logps/rejected": -47.254539489746094,
      "loss": 0.5124,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.7916733026504517,
      "rewards/margins": 0.6568180322647095,
      "rewards/rejected": -1.4484912157058716,
      "step": 575
    },
    {
      "epoch": 0.14,
      "learning_rate": 1.7012367842724884e-07,
      "logits/chosen": -1.5957623720169067,
      "logits/rejected": -1.5968074798583984,
      "logps/chosen": -26.651874542236328,
      "logps/rejected": -47.75643539428711,
      "loss": 0.4962,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.7904036641120911,
      "rewards/margins": 0.7701337933540344,
      "rewards/rejected": -1.560537338256836,
      "step": 580
    },
    {
      "epoch": 0.15,
      "learning_rate": 1.695318858755712e-07,
      "logits/chosen": -1.6403392553329468,
      "logits/rejected": -1.6463661193847656,
      "logps/chosen": -30.842029571533203,
      "logps/rejected": -45.244266510009766,
      "loss": 0.5166,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8566893339157104,
      "rewards/margins": 0.6294564008712769,
      "rewards/rejected": -1.4861459732055664,
      "step": 585
    },
    {
      "epoch": 0.15,
      "learning_rate": 1.6893534091185658e-07,
      "logits/chosen": -1.6482681035995483,
      "logits/rejected": -1.665355920791626,
      "logps/chosen": -31.461956024169922,
      "logps/rejected": -42.383148193359375,
      "loss": 0.4946,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7341334223747253,
      "rewards/margins": 0.5131980180740356,
      "rewards/rejected": -1.2473313808441162,
      "step": 590
    },
    {
      "epoch": 0.15,
      "learning_rate": 1.6833408430916082e-07,
      "logits/chosen": -1.6054385900497437,
      "logits/rejected": -1.6243665218353271,
      "logps/chosen": -27.294775009155273,
      "logps/rejected": -40.987762451171875,
      "loss": 0.4672,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.7351181507110596,
      "rewards/margins": 0.7994521856307983,
      "rewards/rejected": -1.5345704555511475,
      "step": 595
    },
    {
      "epoch": 0.15,
      "learning_rate": 1.6772815716257412e-07,
      "logits/chosen": -1.59373140335083,
      "logits/rejected": -1.6416209936141968,
      "logps/chosen": -31.89492416381836,
      "logps/rejected": -47.964393615722656,
      "loss": 0.4991,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.7752366065979004,
      "rewards/margins": 0.85821533203125,
      "rewards/rejected": -1.6334518194198608,
      "step": 600
    },
    {
      "epoch": 0.15,
      "eval_logits/chosen": -1.7715307474136353,
      "eval_logits/rejected": -1.8386889696121216,
      "eval_logps/chosen": -35.16425704956055,
      "eval_logps/rejected": -49.16897201538086,
      "eval_loss": 0.4996708929538727,
      "eval_rewards/accuracies": 0.75,
      "eval_rewards/chosen": -0.7888572812080383,
      "eval_rewards/margins": 0.739352285861969,
      "eval_rewards/rejected": -1.5282095670700073,
      "eval_runtime": 651.3576,
      "eval_samples_per_second": 30.705,
      "eval_steps_per_second": 0.481,
      "step": 600
    },
    {
      "epoch": 0.15,
      "learning_rate": 1.6711760088641197e-07,
      "logits/chosen": -1.601464033126831,
      "logits/rejected": -1.649553894996643,
      "logps/chosen": -32.55518341064453,
      "logps/rejected": -48.19125747680664,
      "loss": 0.4954,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.970151424407959,
      "rewards/margins": 0.6675958633422852,
      "rewards/rejected": -1.6377474069595337,
      "step": 605
    },
    {
      "epoch": 0.15,
      "learning_rate": 1.665024572113848e-07,
      "logits/chosen": -1.6783527135849,
      "logits/rejected": -1.722983956336975,
      "logps/chosen": -35.63679885864258,
      "logps/rejected": -50.843589782714844,
      "loss": 0.5021,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.8517330884933472,
      "rewards/margins": 0.754002034664154,
      "rewards/rejected": -1.605735182762146,
      "step": 610
    },
    {
      "epoch": 0.15,
      "learning_rate": 1.6588276818174578e-07,
      "logits/chosen": -1.605347990989685,
      "logits/rejected": -1.6674606800079346,
      "logps/chosen": -30.550785064697266,
      "logps/rejected": -53.16766357421875,
      "loss": 0.4996,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.7929040193557739,
      "rewards/margins": 0.869250476360321,
      "rewards/rejected": -1.6621545553207397,
      "step": 615
    },
    {
      "epoch": 0.15,
      "learning_rate": 1.6525857615241686e-07,
      "logits/chosen": -1.6046507358551025,
      "logits/rejected": -1.65438973903656,
      "logps/chosen": -37.572898864746094,
      "logps/rejected": -51.364967346191406,
      "loss": 0.4857,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.8328970074653625,
      "rewards/margins": 0.8492236137390137,
      "rewards/rejected": -1.6821205615997314,
      "step": 620
    },
    {
      "epoch": 0.16,
      "learning_rate": 1.6462992378609406e-07,
      "logits/chosen": -1.600648283958435,
      "logits/rejected": -1.6318552494049072,
      "logps/chosen": -30.713436126708984,
      "logps/rejected": -50.46702194213867,
      "loss": 0.4972,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8259468078613281,
      "rewards/margins": 0.843370258808136,
      "rewards/rejected": -1.6693168878555298,
      "step": 625
    },
    {
      "epoch": 0.16,
      "learning_rate": 1.6399685405033166e-07,
      "logits/chosen": -1.5601131916046143,
      "logits/rejected": -1.6074775457382202,
      "logps/chosen": -31.211395263671875,
      "logps/rejected": -45.20989227294922,
      "loss": 0.4976,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8995653986930847,
      "rewards/margins": 0.750219464302063,
      "rewards/rejected": -1.649784803390503,
      "step": 630
    },
    {
      "epoch": 0.16,
      "learning_rate": 1.6335941021460504e-07,
      "logits/chosen": -1.5442253351211548,
      "logits/rejected": -1.632916808128357,
      "logps/chosen": -31.537792205810547,
      "logps/rejected": -47.98058319091797,
      "loss": 0.4946,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9447064399719238,
      "rewards/margins": 0.827142596244812,
      "rewards/rejected": -1.7718490362167358,
      "step": 635
    },
    {
      "epoch": 0.16,
      "learning_rate": 1.627176358473537e-07,
      "logits/chosen": -1.6395527124404907,
      "logits/rejected": -1.6777455806732178,
      "logps/chosen": -33.45917510986328,
      "logps/rejected": -52.585418701171875,
      "loss": 0.4725,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.7429215312004089,
      "rewards/margins": 0.9078781008720398,
      "rewards/rejected": -1.6507995128631592,
      "step": 640
    },
    {
      "epoch": 0.16,
      "learning_rate": 1.6207157481300312e-07,
      "logits/chosen": -1.7033193111419678,
      "logits/rejected": -1.7614482641220093,
      "logps/chosen": -30.210342407226562,
      "logps/rejected": -49.7141227722168,
      "loss": 0.4813,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8633533716201782,
      "rewards/margins": 0.9977296590805054,
      "rewards/rejected": -1.8610830307006836,
      "step": 645
    },
    {
      "epoch": 0.16,
      "learning_rate": 1.614212712689668e-07,
      "logits/chosen": -1.6111904382705688,
      "logits/rejected": -1.6214799880981445,
      "logps/chosen": -30.123058319091797,
      "logps/rejected": -46.51262664794922,
      "loss": 0.5166,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7468183636665344,
      "rewards/margins": 0.8520328402519226,
      "rewards/rejected": -1.598851203918457,
      "step": 650
    },
    {
      "epoch": 0.16,
      "learning_rate": 1.607667696626281e-07,
      "logits/chosen": -1.5769963264465332,
      "logits/rejected": -1.642521858215332,
      "logps/chosen": -28.669681549072266,
      "logps/rejected": -44.05248260498047,
      "loss": 0.4719,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8426681756973267,
      "rewards/margins": 0.7874193787574768,
      "rewards/rejected": -1.6300876140594482,
      "step": 655
    },
    {
      "epoch": 0.17,
      "learning_rate": 1.601081147283025e-07,
      "logits/chosen": -1.6030101776123047,
      "logits/rejected": -1.647033452987671,
      "logps/chosen": -31.833459854125977,
      "logps/rejected": -47.821189880371094,
      "loss": 0.4791,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.983397364616394,
      "rewards/margins": 0.9160215258598328,
      "rewards/rejected": -1.8994190692901611,
      "step": 660
    },
    {
      "epoch": 0.17,
      "learning_rate": 1.594453514841798e-07,
      "logits/chosen": -1.6228950023651123,
      "logits/rejected": -1.6250451803207397,
      "logps/chosen": -33.878692626953125,
      "logps/rejected": -42.81220245361328,
      "loss": 0.4711,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.8752596974372864,
      "rewards/margins": 0.6205729842185974,
      "rewards/rejected": -1.4958326816558838,
      "step": 665
    },
    {
      "epoch": 0.17,
      "learning_rate": 1.5877852522924732e-07,
      "logits/chosen": -1.5953247547149658,
      "logits/rejected": -1.6342957019805908,
      "logps/chosen": -32.81747817993164,
      "logps/rejected": -63.63328170776367,
      "loss": 0.46,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.7662860155105591,
      "rewards/margins": 1.2044267654418945,
      "rewards/rejected": -1.9707129001617432,
      "step": 670
    },
    {
      "epoch": 0.17,
      "learning_rate": 1.5810768154019382e-07,
      "logits/chosen": -1.570973515510559,
      "logits/rejected": -1.5865790843963623,
      "logps/chosen": -33.97815704345703,
      "logps/rejected": -49.3431396484375,
      "loss": 0.4737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9393548965454102,
      "rewards/margins": 0.8772164583206177,
      "rewards/rejected": -1.8165714740753174,
      "step": 675
    },
    {
      "epoch": 0.17,
      "learning_rate": 1.5743286626829435e-07,
      "logits/chosen": -1.6847432851791382,
      "logits/rejected": -1.703770399093628,
      "logps/chosen": -36.89822769165039,
      "logps/rejected": -53.3497428894043,
      "loss": 0.482,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1248753070831299,
      "rewards/margins": 0.8558415174484253,
      "rewards/rejected": -1.9807169437408447,
      "step": 680
    },
    {
      "epoch": 0.17,
      "learning_rate": 1.5675412553627636e-07,
      "logits/chosen": -1.5613644123077393,
      "logits/rejected": -1.569200873374939,
      "logps/chosen": -32.62022018432617,
      "logps/rejected": -50.714637756347656,
      "loss": 0.5126,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.9382123947143555,
      "rewards/margins": 0.8733282089233398,
      "rewards/rejected": -1.8115408420562744,
      "step": 685
    },
    {
      "epoch": 0.17,
      "learning_rate": 1.5607150573516727e-07,
      "logits/chosen": -1.6045001745224,
      "logits/rejected": -1.6578142642974854,
      "logps/chosen": -34.34115219116211,
      "logps/rejected": -49.66770553588867,
      "loss": 0.4586,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0255018472671509,
      "rewards/margins": 1.1245434284210205,
      "rewards/rejected": -2.150045156478882,
      "step": 690
    },
    {
      "epoch": 0.17,
      "learning_rate": 1.5538505352112372e-07,
      "logits/chosen": -1.575278639793396,
      "logits/rejected": -1.6097004413604736,
      "logps/chosen": -30.383869171142578,
      "logps/rejected": -47.15066146850586,
      "loss": 0.4524,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.8728625178337097,
      "rewards/margins": 0.9022108316421509,
      "rewards/rejected": -1.7750734090805054,
      "step": 695
    },
    {
      "epoch": 0.17,
      "learning_rate": 1.546948158122427e-07,
      "logits/chosen": -1.5903997421264648,
      "logits/rejected": -1.6119863986968994,
      "logps/chosen": -37.304439544677734,
      "logps/rejected": -55.2794075012207,
      "loss": 0.4853,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.9771662950515747,
      "rewards/margins": 0.9782206416130066,
      "rewards/rejected": -1.9553868770599365,
      "step": 700
    },
    {
      "epoch": 0.17,
      "eval_logits/chosen": -1.759588360786438,
      "eval_logits/rejected": -1.8255704641342163,
      "eval_logps/chosen": -36.62710189819336,
      "eval_logps/rejected": -52.2473030090332,
      "eval_loss": 0.48141878843307495,
      "eval_rewards/accuracies": 0.7531948685646057,
      "eval_rewards/chosen": -0.9351421594619751,
      "eval_rewards/margins": 0.9009013175964355,
      "eval_rewards/rejected": -1.836043357849121,
      "eval_runtime": 648.2488,
      "eval_samples_per_second": 30.852,
      "eval_steps_per_second": 0.483,
      "step": 700
    },
    {
      "epoch": 0.18,
      "learning_rate": 1.540008397853547e-07,
      "logits/chosen": -1.6052520275115967,
      "logits/rejected": -1.6869052648544312,
      "logps/chosen": -31.37619972229004,
      "logps/rejected": -50.86903762817383,
      "loss": 0.4618,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.8874313235282898,
      "rewards/margins": 1.1413828134536743,
      "rewards/rejected": -2.0288138389587402,
      "step": 705
    },
    {
      "epoch": 0.18,
      "learning_rate": 1.5330317287279937e-07,
      "logits/chosen": -1.6188942193984985,
      "logits/rejected": -1.6576578617095947,
      "logps/chosen": -27.7589054107666,
      "logps/rejected": -45.0649528503418,
      "loss": 0.4524,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8606212735176086,
      "rewards/margins": 0.9031599164009094,
      "rewards/rejected": -1.7637813091278076,
      "step": 710
    },
    {
      "epoch": 0.18,
      "learning_rate": 1.526018627591834e-07,
      "logits/chosen": -1.6113197803497314,
      "logits/rejected": -1.6310360431671143,
      "logps/chosen": -31.3461971282959,
      "logps/rejected": -54.99705123901367,
      "loss": 0.4833,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.0064165592193604,
      "rewards/margins": 1.113956093788147,
      "rewards/rejected": -2.1203725337982178,
      "step": 715
    },
    {
      "epoch": 0.18,
      "learning_rate": 1.5189695737812152e-07,
      "logits/chosen": -1.5996081829071045,
      "logits/rejected": -1.6568666696548462,
      "logps/chosen": -40.4968147277832,
      "logps/rejected": -51.13838577270508,
      "loss": 0.4633,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.1271008253097534,
      "rewards/margins": 0.8033941388130188,
      "rewards/rejected": -1.930495023727417,
      "step": 720
    },
    {
      "epoch": 0.18,
      "learning_rate": 1.511885049089601e-07,
      "logits/chosen": -1.6359695196151733,
      "logits/rejected": -1.7165886163711548,
      "logps/chosen": -32.879817962646484,
      "logps/rejected": -48.68081283569336,
      "loss": 0.4635,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.958130955696106,
      "rewards/margins": 1.11000657081604,
      "rewards/rejected": -2.0681374073028564,
      "step": 725
    },
    {
      "epoch": 0.18,
      "learning_rate": 1.5047655377348439e-07,
      "logits/chosen": -1.6750876903533936,
      "logits/rejected": -1.6934881210327148,
      "logps/chosen": -32.825477600097656,
      "logps/rejected": -48.85386276245117,
      "loss": 0.4997,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.9325745701789856,
      "rewards/margins": 0.9969514608383179,
      "rewards/rejected": -1.9295259714126587,
      "step": 730
    },
    {
      "epoch": 0.18,
      "learning_rate": 1.4976115263260874e-07,
      "logits/chosen": -1.5678603649139404,
      "logits/rejected": -1.661233901977539,
      "logps/chosen": -36.65790557861328,
      "logps/rejected": -48.384620666503906,
      "loss": 0.4576,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.1775093078613281,
      "rewards/margins": 0.8282756805419922,
      "rewards/rejected": -2.0057852268218994,
      "step": 735
    },
    {
      "epoch": 0.18,
      "learning_rate": 1.4904235038305082e-07,
      "logits/chosen": -1.6477429866790771,
      "logits/rejected": -1.6627209186553955,
      "logps/chosen": -34.94283676147461,
      "logps/rejected": -48.7259407043457,
      "loss": 0.4784,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1199615001678467,
      "rewards/margins": 0.6647182703018188,
      "rewards/rejected": -1.7846797704696655,
      "step": 740
    },
    {
      "epoch": 0.19,
      "learning_rate": 1.483201961539896e-07,
      "logits/chosen": -1.619032859802246,
      "logits/rejected": -1.669075608253479,
      "logps/chosen": -30.97340965270996,
      "logps/rejected": -61.18779754638672,
      "loss": 0.4338,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.950448215007782,
      "rewards/margins": 1.3586184978485107,
      "rewards/rejected": -2.3090667724609375,
      "step": 745
    },
    {
      "epoch": 0.19,
      "learning_rate": 1.4759473930370737e-07,
      "logits/chosen": -1.607339859008789,
      "logits/rejected": -1.595664620399475,
      "logps/chosen": -37.54076385498047,
      "logps/rejected": -58.147911071777344,
      "loss": 0.4578,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.8425636291503906,
      "rewards/margins": 1.2238950729370117,
      "rewards/rejected": -2.0664587020874023,
      "step": 750
    },
    {
      "epoch": 0.19,
      "learning_rate": 1.4686602941621615e-07,
      "logits/chosen": -1.6055225133895874,
      "logits/rejected": -1.6801090240478516,
      "logps/chosen": -39.49528884887695,
      "logps/rejected": -54.026329040527344,
      "loss": 0.4586,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2323989868164062,
      "rewards/margins": 0.6681613922119141,
      "rewards/rejected": -1.9005606174468994,
      "step": 755
    },
    {
      "epoch": 0.19,
      "learning_rate": 1.4613411629786877e-07,
      "logits/chosen": -1.5876009464263916,
      "logits/rejected": -1.6064348220825195,
      "logps/chosen": -35.21834182739258,
      "logps/rejected": -57.62762451171875,
      "loss": 0.4449,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0678521394729614,
      "rewards/margins": 1.2525171041488647,
      "rewards/rejected": -2.320369243621826,
      "step": 760
    },
    {
      "epoch": 0.19,
      "learning_rate": 1.4539904997395468e-07,
      "logits/chosen": -1.5777242183685303,
      "logits/rejected": -1.6244598627090454,
      "logps/chosen": -33.42559814453125,
      "logps/rejected": -52.55967330932617,
      "loss": 0.4599,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0470192432403564,
      "rewards/margins": 1.0231316089630127,
      "rewards/rejected": -2.070150852203369,
      "step": 765
    },
    {
      "epoch": 0.19,
      "learning_rate": 1.4466088068528067e-07,
      "logits/chosen": -1.5677874088287354,
      "logits/rejected": -1.563103199005127,
      "logps/chosen": -31.043560028076172,
      "logps/rejected": -52.094017028808594,
      "loss": 0.4656,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8798549771308899,
      "rewards/margins": 1.1949479579925537,
      "rewards/rejected": -2.074802875518799,
      "step": 770
    },
    {
      "epoch": 0.19,
      "learning_rate": 1.4391965888473702e-07,
      "logits/chosen": -1.5365984439849854,
      "logits/rejected": -1.5609190464019775,
      "logps/chosen": -38.239593505859375,
      "logps/rejected": -51.5609245300293,
      "loss": 0.4535,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2842613458633423,
      "rewards/margins": 0.8575951457023621,
      "rewards/rejected": -2.1418566703796387,
      "step": 775
    },
    {
      "epoch": 0.2,
      "learning_rate": 1.4317543523384928e-07,
      "logits/chosen": -1.522896647453308,
      "logits/rejected": -1.639601707458496,
      "logps/chosen": -32.27168273925781,
      "logps/rejected": -56.03171920776367,
      "loss": 0.4754,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.2065473794937134,
      "rewards/margins": 1.2756444215774536,
      "rewards/rejected": -2.482191562652588,
      "step": 780
    },
    {
      "epoch": 0.2,
      "learning_rate": 1.4242826059931536e-07,
      "logits/chosen": -1.6802387237548828,
      "logits/rejected": -1.691427230834961,
      "logps/chosen": -32.94379425048828,
      "logps/rejected": -50.2864875793457,
      "loss": 0.4789,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.9614617228507996,
      "rewards/margins": 1.0711755752563477,
      "rewards/rejected": -2.032637119293213,
      "step": 785
    },
    {
      "epoch": 0.2,
      "learning_rate": 1.4167818604952903e-07,
      "logits/chosen": -1.6119060516357422,
      "logits/rejected": -1.6578251123428345,
      "logps/chosen": -34.777610778808594,
      "logps/rejected": -49.634010314941406,
      "loss": 0.4639,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0565636157989502,
      "rewards/margins": 0.7960433959960938,
      "rewards/rejected": -1.852607011795044,
      "step": 790
    },
    {
      "epoch": 0.2,
      "learning_rate": 1.4092526285108939e-07,
      "logits/chosen": -1.712721824645996,
      "logits/rejected": -1.752832055091858,
      "logps/chosen": -37.20418930053711,
      "logps/rejected": -53.678688049316406,
      "loss": 0.4833,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.1638964414596558,
      "rewards/margins": 1.0863351821899414,
      "rewards/rejected": -2.2502317428588867,
      "step": 795
    },
    {
      "epoch": 0.2,
      "learning_rate": 1.4016954246529695e-07,
      "logits/chosen": -1.5309230089187622,
      "logits/rejected": -1.579245924949646,
      "logps/chosen": -35.23495864868164,
      "logps/rejected": -51.5868034362793,
      "loss": 0.4421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.0845030546188354,
      "rewards/margins": 1.109590768814087,
      "rewards/rejected": -2.194093704223633,
      "step": 800
    },
    {
      "epoch": 0.2,
      "eval_logits/chosen": -1.756993055343628,
      "eval_logits/rejected": -1.8224340677261353,
      "eval_logps/chosen": -38.5142707824707,
      "eval_logps/rejected": -55.59178924560547,
      "eval_loss": 0.46481600403785706,
      "eval_rewards/accuracies": 0.7643769979476929,
      "eval_rewards/chosen": -1.1238588094711304,
      "eval_rewards/margins": 1.046633005142212,
      "eval_rewards/rejected": -2.170491933822632,
      "eval_runtime": 658.2706,
      "eval_samples_per_second": 30.383,
      "eval_steps_per_second": 0.475,
      "step": 800
    },
    {
      "epoch": 0.2,
      "learning_rate": 1.3941107654463616e-07,
      "logits/chosen": -1.6531871557235718,
      "logits/rejected": -1.6673660278320312,
      "logps/chosen": -31.110912322998047,
      "logps/rejected": -48.3098030090332,
      "loss": 0.4441,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.9944551587104797,
      "rewards/margins": 1.0610458850860596,
      "rewards/rejected": -2.0555014610290527,
      "step": 805
    },
    {
      "epoch": 0.2,
      "learning_rate": 1.3864991692924522e-07,
      "logits/chosen": -1.5531246662139893,
      "logits/rejected": -1.653599500656128,
      "logps/chosen": -40.24319076538086,
      "logps/rejected": -53.15726852416992,
      "loss": 0.4374,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.204906702041626,
      "rewards/margins": 0.9076882600784302,
      "rewards/rejected": -2.1125950813293457,
      "step": 810
    },
    {
      "epoch": 0.2,
      "learning_rate": 1.3788611564337276e-07,
      "logits/chosen": -1.6369264125823975,
      "logits/rejected": -1.691960334777832,
      "logps/chosen": -35.385231018066406,
      "logps/rejected": -55.32622528076172,
      "loss": 0.4493,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.088341474533081,
      "rewards/margins": 1.0900700092315674,
      "rewards/rejected": -2.1784114837646484,
      "step": 815
    },
    {
      "epoch": 0.2,
      "learning_rate": 1.3711972489182207e-07,
      "logits/chosen": -1.6485437154769897,
      "logits/rejected": -1.6988540887832642,
      "logps/chosen": -32.4288215637207,
      "logps/rejected": -58.22150802612305,
      "loss": 0.4148,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.9628303647041321,
      "rewards/margins": 1.4620063304901123,
      "rewards/rejected": -2.4248366355895996,
      "step": 820
    },
    {
      "epoch": 0.21,
      "learning_rate": 1.3635079705638297e-07,
      "logits/chosen": -1.5908691883087158,
      "logits/rejected": -1.5887458324432373,
      "logps/chosen": -33.48944854736328,
      "logps/rejected": -47.57482147216797,
      "loss": 0.4581,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.048614740371704,
      "rewards/margins": 1.045222282409668,
      "rewards/rejected": -2.093837261199951,
      "step": 825
    },
    {
      "epoch": 0.21,
      "learning_rate": 1.3557938469225164e-07,
      "logits/chosen": -1.496455430984497,
      "logits/rejected": -1.565406084060669,
      "logps/chosen": -28.345775604248047,
      "logps/rejected": -48.28296661376953,
      "loss": 0.4276,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8644731640815735,
      "rewards/margins": 1.2932841777801514,
      "rewards/rejected": -2.157757520675659,
      "step": 830
    },
    {
      "epoch": 0.21,
      "learning_rate": 1.3480554052443843e-07,
      "logits/chosen": -1.6535274982452393,
      "logits/rejected": -1.6487712860107422,
      "logps/chosen": -37.31575012207031,
      "logps/rejected": -55.0163459777832,
      "loss": 0.4411,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.180275797843933,
      "rewards/margins": 0.9364711046218872,
      "rewards/rejected": -2.1167469024658203,
      "step": 835
    },
    {
      "epoch": 0.21,
      "learning_rate": 1.340293174441643e-07,
      "logits/chosen": -1.5797991752624512,
      "logits/rejected": -1.6624667644500732,
      "logps/chosen": -35.370601654052734,
      "logps/rejected": -52.5462646484375,
      "loss": 0.435,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.2066878080368042,
      "rewards/margins": 1.18440842628479,
      "rewards/rejected": -2.3910961151123047,
      "step": 840
    },
    {
      "epoch": 0.21,
      "learning_rate": 1.332507685052457e-07,
      "logits/chosen": -1.5771610736846924,
      "logits/rejected": -1.6233526468276978,
      "logps/chosen": -36.874610900878906,
      "logps/rejected": -50.1643180847168,
      "loss": 0.4601,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0964281558990479,
      "rewards/margins": 1.0329301357269287,
      "rewards/rejected": -2.1293582916259766,
      "step": 845
    },
    {
      "epoch": 0.21,
      "learning_rate": 1.3246994692046836e-07,
      "logits/chosen": -1.5860016345977783,
      "logits/rejected": -1.5693161487579346,
      "logps/chosen": -34.47091293334961,
      "logps/rejected": -58.84907913208008,
      "loss": 0.4253,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.114932656288147,
      "rewards/margins": 1.1820452213287354,
      "rewards/rejected": -2.2969777584075928,
      "step": 850
    },
    {
      "epoch": 0.21,
      "learning_rate": 1.3168690605795043e-07,
      "logits/chosen": -1.5049854516983032,
      "logits/rejected": -1.5469257831573486,
      "logps/chosen": -39.44264602661133,
      "logps/rejected": -49.3149299621582,
      "loss": 0.4473,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.0659205913543701,
      "rewards/margins": 1.319551944732666,
      "rewards/rejected": -2.3854727745056152,
      "step": 855
    },
    {
      "epoch": 0.21,
      "learning_rate": 1.3090169943749475e-07,
      "logits/chosen": -1.595726728439331,
      "logits/rejected": -1.6204969882965088,
      "logps/chosen": -34.34868621826172,
      "logps/rejected": -64.38568115234375,
      "loss": 0.4518,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0860964059829712,
      "rewards/margins": 1.4995462894439697,
      "rewards/rejected": -2.5856425762176514,
      "step": 860
    },
    {
      "epoch": 0.22,
      "learning_rate": 1.3011438072693074e-07,
      "logits/chosen": -1.602315902709961,
      "logits/rejected": -1.654049277305603,
      "logps/chosen": -33.55876541137695,
      "logps/rejected": -59.39331817626953,
      "loss": 0.4508,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.0192234516143799,
      "rewards/margins": 1.5562570095062256,
      "rewards/rejected": -2.5754806995391846,
      "step": 865
    },
    {
      "epoch": 0.22,
      "learning_rate": 1.2932500373844649e-07,
      "logits/chosen": -1.5844796895980835,
      "logits/rejected": -1.641312837600708,
      "logps/chosen": -35.43933868408203,
      "logps/rejected": -52.629310607910156,
      "loss": 0.4199,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.1855111122131348,
      "rewards/margins": 1.2338942289352417,
      "rewards/rejected": -2.419405460357666,
      "step": 870
    },
    {
      "epoch": 0.22,
      "learning_rate": 1.2853362242491051e-07,
      "logits/chosen": -1.554743766784668,
      "logits/rejected": -1.5775463581085205,
      "logps/chosen": -38.14448928833008,
      "logps/rejected": -57.41911697387695,
      "loss": 0.4269,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0879703760147095,
      "rewards/margins": 1.3935130834579468,
      "rewards/rejected": -2.481483221054077,
      "step": 875
    },
    {
      "epoch": 0.22,
      "learning_rate": 1.2774029087618446e-07,
      "logits/chosen": -1.5769081115722656,
      "logits/rejected": -1.6464605331420898,
      "logps/chosen": -40.778114318847656,
      "logps/rejected": -62.757667541503906,
      "loss": 0.4533,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.490722417831421,
      "rewards/margins": 1.171587347984314,
      "rewards/rejected": -2.6623096466064453,
      "step": 880
    },
    {
      "epoch": 0.22,
      "learning_rate": 1.2694506331542577e-07,
      "logits/chosen": -1.572746992111206,
      "logits/rejected": -1.6023595333099365,
      "logps/chosen": -36.60691833496094,
      "logps/rejected": -61.279563903808594,
      "loss": 0.4322,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.055189609527588,
      "rewards/margins": 1.3352783918380737,
      "rewards/rejected": -2.390468120574951,
      "step": 885
    },
    {
      "epoch": 0.22,
      "learning_rate": 1.2614799409538198e-07,
      "logits/chosen": -1.5945748090744019,
      "logits/rejected": -1.6221317052841187,
      "logps/chosen": -36.74169158935547,
      "logps/rejected": -55.456451416015625,
      "loss": 0.4427,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.1329044103622437,
      "rewards/margins": 1.147170066833496,
      "rewards/rejected": -2.2800745964050293,
      "step": 890
    },
    {
      "epoch": 0.22,
      "learning_rate": 1.253491376946754e-07,
      "logits/chosen": -1.650987982749939,
      "logits/rejected": -1.6859050989151,
      "logps/chosen": -35.98708724975586,
      "logps/rejected": -60.302879333496094,
      "loss": 0.4441,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.0836153030395508,
      "rewards/margins": 1.4782553911209106,
      "rewards/rejected": -2.561870574951172,
      "step": 895
    },
    {
      "epoch": 0.23,
      "learning_rate": 1.2454854871407992e-07,
      "logits/chosen": -1.6000125408172607,
      "logits/rejected": -1.6861261129379272,
      "logps/chosen": -37.93037796020508,
      "logps/rejected": -55.5538215637207,
      "loss": 0.4428,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2983626127243042,
      "rewards/margins": 1.157037615776062,
      "rewards/rejected": -2.455399990081787,
      "step": 900
    },
    {
      "epoch": 0.23,
      "eval_logits/chosen": -1.754516839981079,
      "eval_logits/rejected": -1.8192579746246338,
      "eval_logps/chosen": -39.144981384277344,
      "eval_logps/rejected": -57.34225082397461,
      "eval_loss": 0.4540169835090637,
      "eval_rewards/accuracies": 0.7651757001876831,
      "eval_rewards/chosen": -1.1869295835494995,
      "eval_rewards/margins": 1.1586089134216309,
      "eval_rewards/rejected": -2.34553861618042,
      "eval_runtime": 661.1199,
      "eval_samples_per_second": 30.252,
      "eval_steps_per_second": 0.473,
      "step": 900
    },
    {
      "epoch": 0.23,
      "learning_rate": 1.2374628187278885e-07,
      "logits/chosen": -1.6199579238891602,
      "logits/rejected": -1.6416032314300537,
      "logps/chosen": -36.601234436035156,
      "logps/rejected": -49.50715255737305,
      "loss": 0.4519,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.2911183834075928,
      "rewards/margins": 0.9798983335494995,
      "rewards/rejected": -2.2710165977478027,
      "step": 905
    },
    {
      "epoch": 0.23,
      "learning_rate": 1.2294239200467515e-07,
      "logits/chosen": -1.556757926940918,
      "logits/rejected": -1.586511254310608,
      "logps/chosen": -32.10668182373047,
      "logps/rejected": -58.393943786621094,
      "loss": 0.4559,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.089107871055603,
      "rewards/margins": 1.1830793619155884,
      "rewards/rejected": -2.2721872329711914,
      "step": 910
    },
    {
      "epoch": 0.23,
      "learning_rate": 1.2213693405454345e-07,
      "logits/chosen": -1.7255284786224365,
      "logits/rejected": -1.718605399131775,
      "logps/chosen": -32.51628875732422,
      "logps/rejected": -49.17650604248047,
      "loss": 0.4712,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.175525426864624,
      "rewards/margins": 1.0937503576278687,
      "rewards/rejected": -2.2692759037017822,
      "step": 915
    },
    {
      "epoch": 0.23,
      "learning_rate": 1.213299630743747e-07,
      "logits/chosen": -1.4932045936584473,
      "logits/rejected": -1.6324726343154907,
      "logps/chosen": -40.018192291259766,
      "logps/rejected": -57.08234786987305,
      "loss": 0.4608,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3543583154678345,
      "rewards/margins": 1.2706938982009888,
      "rewards/rejected": -2.625051975250244,
      "step": 920
    },
    {
      "epoch": 0.23,
      "learning_rate": 1.205215342195634e-07,
      "logits/chosen": -1.6383206844329834,
      "logits/rejected": -1.6871874332427979,
      "logps/chosen": -33.45853805541992,
      "logps/rejected": -57.266441345214844,
      "loss": 0.4555,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.0636919736862183,
      "rewards/margins": 1.4653905630111694,
      "rewards/rejected": -2.529082775115967,
      "step": 925
    },
    {
      "epoch": 0.23,
      "learning_rate": 1.1971170274514802e-07,
      "logits/chosen": -1.6617015600204468,
      "logits/rejected": -1.7073558568954468,
      "logps/chosen": -34.81584930419922,
      "logps/rejected": -50.07064437866211,
      "loss": 0.4288,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.9755508303642273,
      "rewards/margins": 1.349270224571228,
      "rewards/rejected": -2.3248209953308105,
      "step": 930
    },
    {
      "epoch": 0.23,
      "learning_rate": 1.1890052400203402e-07,
      "logits/chosen": -1.5495891571044922,
      "logits/rejected": -1.6336759328842163,
      "logps/chosen": -33.12566375732422,
      "logps/rejected": -51.78517532348633,
      "loss": 0.4653,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.1974151134490967,
      "rewards/margins": 1.0483092069625854,
      "rewards/rejected": -2.2457242012023926,
      "step": 935
    },
    {
      "epoch": 0.23,
      "learning_rate": 1.18088053433211e-07,
      "logits/chosen": -1.528899908065796,
      "logits/rejected": -1.6124309301376343,
      "logps/chosen": -39.89997100830078,
      "logps/rejected": -61.714500427246094,
      "loss": 0.4124,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9432302713394165,
      "rewards/margins": 1.5717684030532837,
      "rewards/rejected": -2.5149986743927,
      "step": 940
    },
    {
      "epoch": 0.24,
      "learning_rate": 1.1727434656996305e-07,
      "logits/chosen": -1.499908208847046,
      "logits/rejected": -1.5667937994003296,
      "logps/chosen": -35.76382827758789,
      "logps/rejected": -52.2210578918457,
      "loss": 0.437,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.2630939483642578,
      "rewards/margins": 0.9919009208679199,
      "rewards/rejected": -2.2549948692321777,
      "step": 945
    },
    {
      "epoch": 0.24,
      "learning_rate": 1.1645945902807339e-07,
      "logits/chosen": -1.4900109767913818,
      "logits/rejected": -1.5031189918518066,
      "logps/chosen": -37.20661926269531,
      "logps/rejected": -51.6440315246582,
      "loss": 0.4046,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.1591427326202393,
      "rewards/margins": 0.9798423051834106,
      "rewards/rejected": -2.1389849185943604,
      "step": 950
    },
    {
      "epoch": 0.24,
      "learning_rate": 1.1564344650402309e-07,
      "logits/chosen": -1.6302865743637085,
      "logits/rejected": -1.716436743736267,
      "logps/chosen": -39.310768127441406,
      "logps/rejected": -60.70337677001953,
      "loss": 0.4087,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.3054693937301636,
      "rewards/margins": 1.2337620258331299,
      "rewards/rejected": -2.539231538772583,
      "step": 955
    },
    {
      "epoch": 0.24,
      "learning_rate": 1.1482636477118419e-07,
      "logits/chosen": -1.5733920335769653,
      "logits/rejected": -1.6743791103363037,
      "logps/chosen": -32.88540267944336,
      "logps/rejected": -55.623626708984375,
      "loss": 0.4443,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.010662317276001,
      "rewards/margins": 1.533831000328064,
      "rewards/rejected": -2.5444931983947754,
      "step": 960
    },
    {
      "epoch": 0.24,
      "learning_rate": 1.1400826967600779e-07,
      "logits/chosen": -1.5256679058074951,
      "logits/rejected": -1.5824081897735596,
      "logps/chosen": -37.93513107299805,
      "logps/rejected": -60.231719970703125,
      "loss": 0.4454,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.357273817062378,
      "rewards/margins": 1.2650949954986572,
      "rewards/rejected": -2.6223690509796143,
      "step": 965
    },
    {
      "epoch": 0.24,
      "learning_rate": 1.131892171342069e-07,
      "logits/chosen": -1.614967703819275,
      "logits/rejected": -1.67970871925354,
      "logps/chosen": -37.4019889831543,
      "logps/rejected": -57.1629638671875,
      "loss": 0.4669,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.1967872381210327,
      "rewards/margins": 1.4661554098129272,
      "rewards/rejected": -2.662942409515381,
      "step": 970
    },
    {
      "epoch": 0.24,
      "learning_rate": 1.1236926312693478e-07,
      "logits/chosen": -1.6489760875701904,
      "logits/rejected": -1.6984256505966187,
      "logps/chosen": -37.733211517333984,
      "logps/rejected": -63.036476135253906,
      "loss": 0.4354,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.297151803970337,
      "rewards/margins": 1.3187490701675415,
      "rewards/rejected": -2.615900754928589,
      "step": 975
    },
    {
      "epoch": 0.24,
      "learning_rate": 1.1154846369695863e-07,
      "logits/chosen": -1.5639992952346802,
      "logits/rejected": -1.643088936805725,
      "logps/chosen": -41.88310241699219,
      "logps/rejected": -60.37211990356445,
      "loss": 0.4446,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3221181631088257,
      "rewards/margins": 1.4724633693695068,
      "rewards/rejected": -2.794581651687622,
      "step": 980
    },
    {
      "epoch": 0.25,
      "learning_rate": 1.1072687494482918e-07,
      "logits/chosen": -1.548208236694336,
      "logits/rejected": -1.5871665477752686,
      "logps/chosen": -47.56840133666992,
      "logps/rejected": -66.3207778930664,
      "loss": 0.4672,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.517787218093872,
      "rewards/margins": 1.2745654582977295,
      "rewards/rejected": -2.7923526763916016,
      "step": 985
    },
    {
      "epoch": 0.25,
      "learning_rate": 1.0990455302504628e-07,
      "logits/chosen": -1.5319437980651855,
      "logits/rejected": -1.6285107135772705,
      "logps/chosen": -42.794715881347656,
      "logps/rejected": -60.02912139892578,
      "loss": 0.4282,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3966975212097168,
      "rewards/margins": 1.354801893234253,
      "rewards/rejected": -2.7514994144439697,
      "step": 990
    },
    {
      "epoch": 0.25,
      "learning_rate": 1.0908155414222082e-07,
      "logits/chosen": -1.6465413570404053,
      "logits/rejected": -1.647565484046936,
      "logps/chosen": -38.375003814697266,
      "logps/rejected": -55.00554275512695,
      "loss": 0.4371,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3204407691955566,
      "rewards/margins": 0.98493492603302,
      "rewards/rejected": -2.305375576019287,
      "step": 995
    },
    {
      "epoch": 0.25,
      "learning_rate": 1.0825793454723325e-07,
      "logits/chosen": -1.5337989330291748,
      "logits/rejected": -1.6268227100372314,
      "logps/chosen": -31.162311553955078,
      "logps/rejected": -53.79045867919922,
      "loss": 0.4313,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.9945358037948608,
      "rewards/margins": 1.6616699695587158,
      "rewards/rejected": -2.656205654144287,
      "step": 1000
    },
    {
      "epoch": 0.25,
      "eval_logits/chosen": -1.7576247453689575,
      "eval_logits/rejected": -1.8220233917236328,
      "eval_logps/chosen": -39.66193389892578,
      "eval_logps/rejected": -58.60415267944336,
      "eval_loss": 0.44551411271095276,
      "eval_rewards/accuracies": 0.7715654969215393,
      "eval_rewards/chosen": -1.2386250495910645,
      "eval_rewards/margins": 1.2331026792526245,
      "eval_rewards/rejected": -2.4717278480529785,
      "eval_runtime": 664.3994,
      "eval_samples_per_second": 30.102,
      "eval_steps_per_second": 0.471,
      "step": 1000
    },
    {
      "epoch": 0.25,
      "learning_rate": 1.0743375053338877e-07,
      "logits/chosen": -1.5790529251098633,
      "logits/rejected": -1.589842438697815,
      "logps/chosen": -34.36865997314453,
      "logps/rejected": -59.8324089050293,
      "loss": 0.4741,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.165611982345581,
      "rewards/margins": 1.1656267642974854,
      "rewards/rejected": -2.331238269805908,
      "step": 1005
    },
    {
      "epoch": 0.25,
      "learning_rate": 1.0660905843256993e-07,
      "logits/chosen": -1.5421807765960693,
      "logits/rejected": -1.5461338758468628,
      "logps/chosen": -34.33051300048828,
      "logps/rejected": -53.014427185058594,
      "loss": 0.4437,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.215490460395813,
      "rewards/margins": 1.2017762660980225,
      "rewards/rejected": -2.417266368865967,
      "step": 1010
    },
    {
      "epoch": 0.25,
      "learning_rate": 1.057839146113864e-07,
      "logits/chosen": -1.5130884647369385,
      "logits/rejected": -1.6016590595245361,
      "logps/chosen": -39.093894958496094,
      "logps/rejected": -51.003597259521484,
      "loss": 0.4505,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.2024251222610474,
      "rewards/margins": 1.1259633302688599,
      "rewards/rejected": -2.3283886909484863,
      "step": 1015
    },
    {
      "epoch": 0.26,
      "learning_rate": 1.0495837546732223e-07,
      "logits/chosen": -1.665765404701233,
      "logits/rejected": -1.7019054889678955,
      "logps/chosen": -40.30259704589844,
      "logps/rejected": -61.30155563354492,
      "loss": 0.4353,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.3531296253204346,
      "rewards/margins": 1.1265552043914795,
      "rewards/rejected": -2.479685068130493,
      "step": 1020
    },
    {
      "epoch": 0.26,
      "learning_rate": 1.0413249742488131e-07,
      "logits/chosen": -1.6483551263809204,
      "logits/rejected": -1.6976540088653564,
      "logps/chosen": -30.37175941467285,
      "logps/rejected": -57.36388397216797,
      "loss": 0.4403,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.0984838008880615,
      "rewards/margins": 1.4797937870025635,
      "rewards/rejected": -2.578277349472046,
      "step": 1025
    },
    {
      "epoch": 0.26,
      "learning_rate": 1.033063369317308e-07,
      "logits/chosen": -1.6444858312606812,
      "logits/rejected": -1.7046945095062256,
      "logps/chosen": -39.15083312988281,
      "logps/rejected": -57.31549835205078,
      "loss": 0.4556,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.297633409500122,
      "rewards/margins": 1.1475791931152344,
      "rewards/rejected": -2.4452126026153564,
      "step": 1030
    },
    {
      "epoch": 0.26,
      "learning_rate": 1.0247995045484301e-07,
      "logits/chosen": -1.7556772232055664,
      "logits/rejected": -1.778540015220642,
      "logps/chosen": -32.11585998535156,
      "logps/rejected": -59.79661178588867,
      "loss": 0.4196,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0470155477523804,
      "rewards/margins": 1.7535333633422852,
      "rewards/rejected": -2.800549030303955,
      "step": 1035
    },
    {
      "epoch": 0.26,
      "learning_rate": 1.0165339447663586e-07,
      "logits/chosen": -1.621145248413086,
      "logits/rejected": -1.676640510559082,
      "logps/chosen": -41.72956466674805,
      "logps/rejected": -52.85761260986328,
      "loss": 0.4325,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.3959109783172607,
      "rewards/margins": 1.1459295749664307,
      "rewards/rejected": -2.5418403148651123,
      "step": 1040
    },
    {
      "epoch": 0.26,
      "learning_rate": 1.0082672549111248e-07,
      "logits/chosen": -1.5826036930084229,
      "logits/rejected": -1.5808308124542236,
      "logps/chosen": -42.364906311035156,
      "logps/rejected": -63.37822341918945,
      "loss": 0.4357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.297041654586792,
      "rewards/margins": 1.3919389247894287,
      "rewards/rejected": -2.6889805793762207,
      "step": 1045
    },
    {
      "epoch": 0.26,
      "learning_rate": 1e-07,
      "logits/chosen": -1.5667890310287476,
      "logits/rejected": -1.6108767986297607,
      "logps/chosen": -37.04503631591797,
      "logps/rejected": -70.06754302978516,
      "loss": 0.4323,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1212371587753296,
      "rewards/margins": 1.5776902437210083,
      "rewards/rejected": -2.698927402496338,
      "step": 1050
    },
    {
      "epoch": 0.26,
      "learning_rate": 9.917327450888751e-08,
      "logits/chosen": -1.7088003158569336,
      "logits/rejected": -1.7888380289077759,
      "logps/chosen": -33.05218505859375,
      "logps/rejected": -53.8926887512207,
      "loss": 0.3986,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.9519890546798706,
      "rewards/margins": 1.2312923669815063,
      "rewards/rejected": -2.183281183242798,
      "step": 1055
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.834660552336415e-08,
      "logits/chosen": -1.555781602859497,
      "logits/rejected": -1.5879541635513306,
      "logps/chosen": -35.17169189453125,
      "logps/rejected": -62.30256271362305,
      "loss": 0.4364,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.192533016204834,
      "rewards/margins": 1.6748558282852173,
      "rewards/rejected": -2.8673884868621826,
      "step": 1060
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.752004954515699e-08,
      "logits/chosen": -1.5017266273498535,
      "logits/rejected": -1.581365704536438,
      "logps/chosen": -38.57265853881836,
      "logps/rejected": -59.8960075378418,
      "loss": 0.3938,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.2197906970977783,
      "rewards/margins": 1.3951218128204346,
      "rewards/rejected": -2.614912509918213,
      "step": 1065
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.669366306826918e-08,
      "logits/chosen": -1.675785779953003,
      "logits/rejected": -1.7346951961517334,
      "logps/chosen": -33.21458053588867,
      "logps/rejected": -59.9798583984375,
      "loss": 0.4476,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.257973313331604,
      "rewards/margins": 1.5266462564468384,
      "rewards/rejected": -2.7846198081970215,
      "step": 1070
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.586750257511866e-08,
      "logits/chosen": -1.592603325843811,
      "logits/rejected": -1.630692720413208,
      "logps/chosen": -36.74489974975586,
      "logps/rejected": -55.419898986816406,
      "loss": 0.4328,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3241074085235596,
      "rewards/margins": 1.336132287979126,
      "rewards/rejected": -2.6602396965026855,
      "step": 1075
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.504162453267775e-08,
      "logits/chosen": -1.561734914779663,
      "logits/rejected": -1.6008018255233765,
      "logps/chosen": -33.03455352783203,
      "logps/rejected": -62.66986846923828,
      "loss": 0.4287,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.328235149383545,
      "rewards/margins": 1.3843605518341064,
      "rewards/rejected": -2.7125957012176514,
      "step": 1080
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.421608538861361e-08,
      "logits/chosen": -1.6418508291244507,
      "logits/rejected": -1.7292274236679077,
      "logps/chosen": -37.394859313964844,
      "logps/rejected": -52.148902893066406,
      "loss": 0.4368,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2655750513076782,
      "rewards/margins": 1.1081774234771729,
      "rewards/rejected": -2.3737523555755615,
      "step": 1085
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.339094156743006e-08,
      "logits/chosen": -1.6823419332504272,
      "logits/rejected": -1.7162498235702515,
      "logps/chosen": -33.99588394165039,
      "logps/rejected": -62.70172882080078,
      "loss": 0.3985,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.046633005142212,
      "rewards/margins": 1.7750266790390015,
      "rewards/rejected": -2.821659564971924,
      "step": 1090
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.256624946661125e-08,
      "logits/chosen": -1.5356955528259277,
      "logits/rejected": -1.5542399883270264,
      "logps/chosen": -36.15153884887695,
      "logps/rejected": -60.288482666015625,
      "loss": 0.4296,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.113905668258667,
      "rewards/margins": 1.3887674808502197,
      "rewards/rejected": -2.5026729106903076,
      "step": 1095
    },
    {
      "epoch": 0.28,
      "learning_rate": 9.174206545276677e-08,
      "logits/chosen": -1.6465669870376587,
      "logits/rejected": -1.6484692096710205,
      "logps/chosen": -39.20000076293945,
      "logps/rejected": -67.43548583984375,
      "loss": 0.3895,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.1534570455551147,
      "rewards/margins": 1.7428104877471924,
      "rewards/rejected": -2.8962674140930176,
      "step": 1100
    },
    {
      "epoch": 0.28,
      "eval_logits/chosen": -1.7623785734176636,
      "eval_logits/rejected": -1.827060341835022,
      "eval_logps/chosen": -40.319217681884766,
      "eval_logps/rejected": -59.921573638916016,
      "eval_loss": 0.43849822878837585,
      "eval_rewards/accuracies": 0.7719648480415344,
      "eval_rewards/chosen": -1.3043534755706787,
      "eval_rewards/margins": 1.2991164922714233,
      "eval_rewards/rejected": -2.6034700870513916,
      "eval_runtime": 670.9922,
      "eval_samples_per_second": 29.807,
      "eval_steps_per_second": 0.466,
      "step": 1100
    },
    {
      "epoch": 0.28,
      "learning_rate": 9.091844585777917e-08,
      "logits/chosen": -1.5940697193145752,
      "logits/rejected": -1.6277192831039429,
      "logps/chosen": -45.242149353027344,
      "logps/rejected": -59.017784118652344,
      "loss": 0.4473,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4398057460784912,
      "rewards/margins": 1.2071270942687988,
      "rewards/rejected": -2.646932601928711,
      "step": 1105
    },
    {
      "epoch": 0.28,
      "learning_rate": 9.009544697495372e-08,
      "logits/chosen": -1.5828858613967896,
      "logits/rejected": -1.636697769165039,
      "logps/chosen": -37.46663284301758,
      "logps/rejected": -55.285606384277344,
      "loss": 0.4278,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.2059729099273682,
      "rewards/margins": 1.2867709398269653,
      "rewards/rejected": -2.492743968963623,
      "step": 1110
    },
    {
      "epoch": 0.28,
      "learning_rate": 8.927312505517084e-08,
      "logits/chosen": -1.528216004371643,
      "logits/rejected": -1.5571744441986084,
      "logps/chosen": -35.96453857421875,
      "logps/rejected": -49.5126838684082,
      "loss": 0.4604,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.3253941535949707,
      "rewards/margins": 0.9808666110038757,
      "rewards/rejected": -2.3062610626220703,
      "step": 1115
    },
    {
      "epoch": 0.28,
      "learning_rate": 8.845153630304139e-08,
      "logits/chosen": -1.6100482940673828,
      "logits/rejected": -1.6352077722549438,
      "logps/chosen": -32.09465408325195,
      "logps/rejected": -59.7017822265625,
      "loss": 0.4167,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1275434494018555,
      "rewards/margins": 1.606652021408081,
      "rewards/rejected": -2.7341952323913574,
      "step": 1120
    },
    {
      "epoch": 0.28,
      "learning_rate": 8.763073687306523e-08,
      "logits/chosen": -1.5683681964874268,
      "logits/rejected": -1.5995924472808838,
      "logps/chosen": -35.26288604736328,
      "logps/rejected": -56.42836380004883,
      "loss": 0.4528,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.483015537261963,
      "rewards/margins": 1.1930134296417236,
      "rewards/rejected": -2.6760289669036865,
      "step": 1125
    },
    {
      "epoch": 0.28,
      "learning_rate": 8.68107828657931e-08,
      "logits/chosen": -1.515114426612854,
      "logits/rejected": -1.5763431787490845,
      "logps/chosen": -36.468536376953125,
      "logps/rejected": -65.7103042602539,
      "loss": 0.3842,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.0401891469955444,
      "rewards/margins": 1.8647140264511108,
      "rewards/rejected": -2.9049034118652344,
      "step": 1130
    },
    {
      "epoch": 0.28,
      "learning_rate": 8.59917303239922e-08,
      "logits/chosen": -1.6495635509490967,
      "logits/rejected": -1.6883957386016846,
      "logps/chosen": -36.199073791503906,
      "logps/rejected": -55.47711944580078,
      "loss": 0.3981,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.1502685546875,
      "rewards/margins": 1.4356176853179932,
      "rewards/rejected": -2.585886001586914,
      "step": 1135
    },
    {
      "epoch": 0.28,
      "learning_rate": 8.517363522881579e-08,
      "logits/chosen": -1.5667603015899658,
      "logits/rejected": -1.6185035705566406,
      "logps/chosen": -32.305355072021484,
      "logps/rejected": -60.237579345703125,
      "loss": 0.4161,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.119470238685608,
      "rewards/margins": 1.565954327583313,
      "rewards/rejected": -2.685424327850342,
      "step": 1140
    },
    {
      "epoch": 0.29,
      "learning_rate": 8.435655349597689e-08,
      "logits/chosen": -1.618090033531189,
      "logits/rejected": -1.671553373336792,
      "logps/chosen": -35.39004898071289,
      "logps/rejected": -64.5052261352539,
      "loss": 0.4255,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.4270261526107788,
      "rewards/margins": 1.423475742340088,
      "rewards/rejected": -2.8505020141601562,
      "step": 1145
    },
    {
      "epoch": 0.29,
      "learning_rate": 8.354054097192658e-08,
      "logits/chosen": -1.5533418655395508,
      "logits/rejected": -1.6151001453399658,
      "logps/chosen": -41.16820526123047,
      "logps/rejected": -57.746246337890625,
      "loss": 0.413,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.3642529249191284,
      "rewards/margins": 1.594090461730957,
      "rewards/rejected": -2.958343267440796,
      "step": 1150
    },
    {
      "epoch": 0.29,
      "learning_rate": 8.2725653430037e-08,
      "logits/chosen": -1.6192905902862549,
      "logits/rejected": -1.6287028789520264,
      "logps/chosen": -39.75355911254883,
      "logps/rejected": -64.3330078125,
      "loss": 0.4372,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.475280523300171,
      "rewards/margins": 1.366707444190979,
      "rewards/rejected": -2.8419878482818604,
      "step": 1155
    },
    {
      "epoch": 0.29,
      "learning_rate": 8.191194656678904e-08,
      "logits/chosen": -1.6061389446258545,
      "logits/rejected": -1.6677024364471436,
      "logps/chosen": -38.45769500732422,
      "logps/rejected": -53.96708297729492,
      "loss": 0.4248,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.3363707065582275,
      "rewards/margins": 1.3108900785446167,
      "rewards/rejected": -2.647261142730713,
      "step": 1160
    },
    {
      "epoch": 0.29,
      "learning_rate": 8.109947599796598e-08,
      "logits/chosen": -1.563370943069458,
      "logits/rejected": -1.6023693084716797,
      "logps/chosen": -35.938392639160156,
      "logps/rejected": -66.66423797607422,
      "loss": 0.3955,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.204688549041748,
      "rewards/margins": 1.7450326681137085,
      "rewards/rejected": -2.949721097946167,
      "step": 1165
    },
    {
      "epoch": 0.29,
      "learning_rate": 8.028829725485198e-08,
      "logits/chosen": -1.5231170654296875,
      "logits/rejected": -1.5746376514434814,
      "logps/chosen": -42.712867736816406,
      "logps/rejected": -58.072288513183594,
      "loss": 0.4561,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.479255199432373,
      "rewards/margins": 1.2070258855819702,
      "rewards/rejected": -2.686281204223633,
      "step": 1170
    },
    {
      "epoch": 0.29,
      "learning_rate": 7.947846578043658e-08,
      "logits/chosen": -1.6910524368286133,
      "logits/rejected": -1.7825229167938232,
      "logps/chosen": -35.068939208984375,
      "logps/rejected": -63.149635314941406,
      "loss": 0.4065,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.9734908938407898,
      "rewards/margins": 1.701172113418579,
      "rewards/rejected": -2.6746630668640137,
      "step": 1175
    },
    {
      "epoch": 0.29,
      "learning_rate": 7.867003692562532e-08,
      "logits/chosen": -1.6070430278778076,
      "logits/rejected": -1.647657036781311,
      "logps/chosen": -35.467254638671875,
      "logps/rejected": -51.75764846801758,
      "loss": 0.42,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4265575408935547,
      "rewards/margins": 1.182373285293579,
      "rewards/rejected": -2.608931064605713,
      "step": 1180
    },
    {
      "epoch": 0.3,
      "learning_rate": 7.786306594545656e-08,
      "logits/chosen": -1.543676733970642,
      "logits/rejected": -1.613452672958374,
      "logps/chosen": -34.50806427001953,
      "logps/rejected": -49.17207336425781,
      "loss": 0.4661,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3245508670806885,
      "rewards/margins": 1.224057912826538,
      "rewards/rejected": -2.5486085414886475,
      "step": 1185
    },
    {
      "epoch": 0.3,
      "learning_rate": 7.705760799532485e-08,
      "logits/chosen": -1.6542270183563232,
      "logits/rejected": -1.676404595375061,
      "logps/chosen": -38.976356506347656,
      "logps/rejected": -61.9526481628418,
      "loss": 0.4308,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.3091213703155518,
      "rewards/margins": 1.3484121561050415,
      "rewards/rejected": -2.6575334072113037,
      "step": 1190
    },
    {
      "epoch": 0.3,
      "learning_rate": 7.625371812721114e-08,
      "logits/chosen": -1.6496282815933228,
      "logits/rejected": -1.686861276626587,
      "logps/chosen": -43.114749908447266,
      "logps/rejected": -58.412376403808594,
      "loss": 0.4126,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4000921249389648,
      "rewards/margins": 1.3142130374908447,
      "rewards/rejected": -2.7143054008483887,
      "step": 1195
    },
    {
      "epoch": 0.3,
      "learning_rate": 7.545145128592009e-08,
      "logits/chosen": -1.6392017602920532,
      "logits/rejected": -1.659065842628479,
      "logps/chosen": -34.54826736450195,
      "logps/rejected": -56.08115768432617,
      "loss": 0.4386,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3348114490509033,
      "rewards/margins": 1.3222072124481201,
      "rewards/rejected": -2.6570186614990234,
      "step": 1200
    },
    {
      "epoch": 0.3,
      "eval_logits/chosen": -1.7654143571853638,
      "eval_logits/rejected": -1.8298496007919312,
      "eval_logps/chosen": -40.549720764160156,
      "eval_logps/rejected": -60.5917854309082,
      "eval_loss": 0.43321406841278076,
      "eval_rewards/accuracies": 0.7775558829307556,
      "eval_rewards/chosen": -1.3274036645889282,
      "eval_rewards/margins": 1.3430880308151245,
      "eval_rewards/rejected": -2.6704916954040527,
      "eval_runtime": 672.9903,
      "eval_samples_per_second": 29.718,
      "eval_steps_per_second": 0.465,
      "step": 1200
    },
    {
      "epoch": 0.3,
      "learning_rate": 7.465086230532459e-08,
      "logits/chosen": -1.7243528366088867,
      "logits/rejected": -1.745429277420044,
      "logps/chosen": -35.07295608520508,
      "logps/rejected": -57.85649871826172,
      "loss": 0.4251,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.1319130659103394,
      "rewards/margins": 1.6454994678497314,
      "rewards/rejected": -2.7774124145507812,
      "step": 1205
    },
    {
      "epoch": 0.3,
      "learning_rate": 7.385200590461802e-08,
      "logits/chosen": -1.577544927597046,
      "logits/rejected": -1.5696823596954346,
      "logps/chosen": -34.15651321411133,
      "logps/rejected": -66.87596130371094,
      "loss": 0.4143,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.263463020324707,
      "rewards/margins": 1.7731821537017822,
      "rewards/rejected": -3.0366451740264893,
      "step": 1210
    },
    {
      "epoch": 0.3,
      "learning_rate": 7.305493668457419e-08,
      "logits/chosen": -1.5483906269073486,
      "logits/rejected": -1.611096978187561,
      "logps/chosen": -34.1319580078125,
      "logps/rejected": -58.55057907104492,
      "loss": 0.4032,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.100731372833252,
      "rewards/margins": 1.511641263961792,
      "rewards/rejected": -2.612372636795044,
      "step": 1215
    },
    {
      "epoch": 0.3,
      "learning_rate": 7.225970912381556e-08,
      "logits/chosen": -1.5925017595291138,
      "logits/rejected": -1.6905170679092407,
      "logps/chosen": -35.592220306396484,
      "logps/rejected": -57.27701950073242,
      "loss": 0.3736,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.184050440788269,
      "rewards/margins": 1.6517807245254517,
      "rewards/rejected": -2.8358309268951416,
      "step": 1220
    },
    {
      "epoch": 0.31,
      "learning_rate": 7.146637757508949e-08,
      "logits/chosen": -1.5678200721740723,
      "logits/rejected": -1.6304035186767578,
      "logps/chosen": -33.75212860107422,
      "logps/rejected": -57.301422119140625,
      "loss": 0.4147,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.9995731115341187,
      "rewards/margins": 1.5843132734298706,
      "rewards/rejected": -2.5838866233825684,
      "step": 1225
    },
    {
      "epoch": 0.31,
      "learning_rate": 7.067499626155353e-08,
      "logits/chosen": -1.426163911819458,
      "logits/rejected": -1.4889883995056152,
      "logps/chosen": -40.3751106262207,
      "logps/rejected": -62.611167907714844,
      "loss": 0.4304,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.4146851301193237,
      "rewards/margins": 1.500084400177002,
      "rewards/rejected": -2.914769411087036,
      "step": 1230
    },
    {
      "epoch": 0.31,
      "learning_rate": 6.988561927306926e-08,
      "logits/chosen": -1.6196644306182861,
      "logits/rejected": -1.6350748538970947,
      "logps/chosen": -34.81641387939453,
      "logps/rejected": -60.31574630737305,
      "loss": 0.387,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.2771868705749512,
      "rewards/margins": 1.5419944524765015,
      "rewards/rejected": -2.819181203842163,
      "step": 1235
    },
    {
      "epoch": 0.31,
      "learning_rate": 6.909830056250527e-08,
      "logits/chosen": -1.574197769165039,
      "logits/rejected": -1.6305344104766846,
      "logps/chosen": -41.36061096191406,
      "logps/rejected": -59.025543212890625,
      "loss": 0.4207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.610036849975586,
      "rewards/margins": 1.2757810354232788,
      "rewards/rejected": -2.885817766189575,
      "step": 1240
    },
    {
      "epoch": 0.31,
      "learning_rate": 6.831309394204956e-08,
      "logits/chosen": -1.4915789365768433,
      "logits/rejected": -1.5080435276031494,
      "logps/chosen": -33.570152282714844,
      "logps/rejected": -60.77013397216797,
      "loss": 0.4148,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.416123390197754,
      "rewards/margins": 1.3803520202636719,
      "rewards/rejected": -2.7964751720428467,
      "step": 1245
    },
    {
      "epoch": 0.31,
      "learning_rate": 6.753005307953166e-08,
      "logits/chosen": -1.5550016164779663,
      "logits/rejected": -1.5937179327011108,
      "logps/chosen": -35.67682647705078,
      "logps/rejected": -50.56844711303711,
      "loss": 0.3835,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.2889750003814697,
      "rewards/margins": 1.1508995294570923,
      "rewards/rejected": -2.4398744106292725,
      "step": 1250
    },
    {
      "epoch": 0.31,
      "learning_rate": 6.674923149475432e-08,
      "logits/chosen": -1.6518977880477905,
      "logits/rejected": -1.7234160900115967,
      "logps/chosen": -34.21056365966797,
      "logps/rejected": -57.03264617919922,
      "loss": 0.4093,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.2470835447311401,
      "rewards/margins": 1.5973832607269287,
      "rewards/rejected": -2.8444666862487793,
      "step": 1255
    },
    {
      "epoch": 0.32,
      "learning_rate": 6.597068255583569e-08,
      "logits/chosen": -1.5592567920684814,
      "logits/rejected": -1.6327491998672485,
      "logps/chosen": -38.95235061645508,
      "logps/rejected": -53.28133010864258,
      "loss": 0.4692,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.4167959690093994,
      "rewards/margins": 1.2280877828598022,
      "rewards/rejected": -2.644883632659912,
      "step": 1260
    },
    {
      "epoch": 0.32,
      "learning_rate": 6.519445947556154e-08,
      "logits/chosen": -1.609717607498169,
      "logits/rejected": -1.6457746028900146,
      "logps/chosen": -37.1761360168457,
      "logps/rejected": -56.73893356323242,
      "loss": 0.4155,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.2005279064178467,
      "rewards/margins": 1.3979219198226929,
      "rewards/rejected": -2.59844970703125,
      "step": 1265
    },
    {
      "epoch": 0.32,
      "learning_rate": 6.442061530774834e-08,
      "logits/chosen": -1.547778844833374,
      "logits/rejected": -1.6019436120986938,
      "logps/chosen": -39.04016876220703,
      "logps/rejected": -57.7167854309082,
      "loss": 0.4295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.386942744255066,
      "rewards/margins": 1.4044148921966553,
      "rewards/rejected": -2.7913577556610107,
      "step": 1270
    },
    {
      "epoch": 0.32,
      "learning_rate": 6.3649202943617e-08,
      "logits/chosen": -1.541163444519043,
      "logits/rejected": -1.6102149486541748,
      "logps/chosen": -36.81908416748047,
      "logps/rejected": -59.110313415527344,
      "loss": 0.4437,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.454079031944275,
      "rewards/margins": 1.5748546123504639,
      "rewards/rejected": -3.0289340019226074,
      "step": 1275
    },
    {
      "epoch": 0.32,
      "learning_rate": 6.288027510817791e-08,
      "logits/chosen": -1.652465581893921,
      "logits/rejected": -1.6677935123443604,
      "logps/chosen": -33.92074966430664,
      "logps/rejected": -49.562801361083984,
      "loss": 0.4439,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2316993474960327,
      "rewards/margins": 1.157551884651184,
      "rewards/rejected": -2.389251232147217,
      "step": 1280
    },
    {
      "epoch": 0.32,
      "learning_rate": 6.211388435662721e-08,
      "logits/chosen": -1.627846121788025,
      "logits/rejected": -1.6586716175079346,
      "logps/chosen": -34.919586181640625,
      "logps/rejected": -57.35619354248047,
      "loss": 0.4321,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.1946351528167725,
      "rewards/margins": 1.3707973957061768,
      "rewards/rejected": -2.565432548522949,
      "step": 1285
    },
    {
      "epoch": 0.32,
      "learning_rate": 6.135008307075479e-08,
      "logits/chosen": -1.5772037506103516,
      "logits/rejected": -1.6464507579803467,
      "logps/chosen": -35.31485366821289,
      "logps/rejected": -54.949005126953125,
      "loss": 0.4616,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.2055904865264893,
      "rewards/margins": 1.443077802658081,
      "rewards/rejected": -2.6486685276031494,
      "step": 1290
    },
    {
      "epoch": 0.32,
      "learning_rate": 6.058892345536387e-08,
      "logits/chosen": -1.5377371311187744,
      "logits/rejected": -1.5599898099899292,
      "logps/chosen": -41.80292510986328,
      "logps/rejected": -53.32208251953125,
      "loss": 0.4572,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4467737674713135,
      "rewards/margins": 1.1260101795196533,
      "rewards/rejected": -2.5727837085723877,
      "step": 1295
    },
    {
      "epoch": 0.33,
      "learning_rate": 5.983045753470308e-08,
      "logits/chosen": -1.6316235065460205,
      "logits/rejected": -1.6944658756256104,
      "logps/chosen": -33.74870681762695,
      "logps/rejected": -61.187347412109375,
      "loss": 0.4035,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3623381853103638,
      "rewards/margins": 1.4390232563018799,
      "rewards/rejected": -2.801361560821533,
      "step": 1300
    },
    {
      "epoch": 0.33,
      "eval_logits/chosen": -1.7694017887115479,
      "eval_logits/rejected": -1.8335498571395874,
      "eval_logps/chosen": -40.61641311645508,
      "eval_logps/rejected": -60.981990814208984,
      "eval_loss": 0.4293850064277649,
      "eval_rewards/accuracies": 0.7811501622200012,
      "eval_rewards/chosen": -1.3340733051300049,
      "eval_rewards/margins": 1.3754388093948364,
      "eval_rewards/rejected": -2.7095119953155518,
      "eval_runtime": 677.4118,
      "eval_samples_per_second": 29.524,
      "eval_steps_per_second": 0.462,
      "step": 1300
    },
    {
      "epoch": 0.33,
      "learning_rate": 5.9074737148910606e-08,
      "logits/chosen": -1.6358497142791748,
      "logits/rejected": -1.6877262592315674,
      "logps/chosen": -33.53594207763672,
      "logps/rejected": -52.47662353515625,
      "loss": 0.4214,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.1719892024993896,
      "rewards/margins": 1.2264606952667236,
      "rewards/rejected": -2.398449659347534,
      "step": 1305
    },
    {
      "epoch": 0.33,
      "learning_rate": 5.832181395047098e-08,
      "logits/chosen": -1.5000022649765015,
      "logits/rejected": -1.5386083126068115,
      "logps/chosen": -34.92070770263672,
      "logps/rejected": -61.40190887451172,
      "loss": 0.4332,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2568442821502686,
      "rewards/margins": 1.5954145193099976,
      "rewards/rejected": -2.8522589206695557,
      "step": 1310
    },
    {
      "epoch": 0.33,
      "learning_rate": 5.7571739400684635e-08,
      "logits/chosen": -1.5450022220611572,
      "logits/rejected": -1.6439878940582275,
      "logps/chosen": -36.46127700805664,
      "logps/rejected": -57.535743713378906,
      "loss": 0.4263,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.1943490505218506,
      "rewards/margins": 1.5856481790542603,
      "rewards/rejected": -2.7799973487854004,
      "step": 1315
    },
    {
      "epoch": 0.33,
      "learning_rate": 5.682456476615072e-08,
      "logits/chosen": -1.5656230449676514,
      "logits/rejected": -1.6265404224395752,
      "logps/chosen": -37.38570022583008,
      "logps/rejected": -59.56365966796875,
      "loss": 0.4365,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4264448881149292,
      "rewards/margins": 1.2477037906646729,
      "rewards/rejected": -2.6741487979888916,
      "step": 1320
    },
    {
      "epoch": 0.33,
      "learning_rate": 5.6080341115262976e-08,
      "logits/chosen": -1.6068060398101807,
      "logits/rejected": -1.656057357788086,
      "logps/chosen": -39.901092529296875,
      "logps/rejected": -60.206336975097656,
      "loss": 0.4312,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5584208965301514,
      "rewards/margins": 1.1731221675872803,
      "rewards/rejected": -2.7315430641174316,
      "step": 1325
    },
    {
      "epoch": 0.33,
      "learning_rate": 5.533911931471935e-08,
      "logits/chosen": -1.607656478881836,
      "logits/rejected": -1.6602356433868408,
      "logps/chosen": -35.008094787597656,
      "logps/rejected": -58.70320510864258,
      "loss": 0.4371,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.375211477279663,
      "rewards/margins": 1.3568847179412842,
      "rewards/rejected": -2.7320961952209473,
      "step": 1330
    },
    {
      "epoch": 0.33,
      "learning_rate": 5.460095002604532e-08,
      "logits/chosen": -1.5863715410232544,
      "logits/rejected": -1.5841939449310303,
      "logps/chosen": -37.74492263793945,
      "logps/rejected": -61.49238967895508,
      "loss": 0.4058,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.1969599723815918,
      "rewards/margins": 1.476892352104187,
      "rewards/rejected": -2.6738524436950684,
      "step": 1335
    },
    {
      "epoch": 0.34,
      "learning_rate": 5.386588370213123e-08,
      "logits/chosen": -1.6613683700561523,
      "logits/rejected": -1.6925634145736694,
      "logps/chosen": -32.180824279785156,
      "logps/rejected": -64.28355407714844,
      "loss": 0.4017,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.0029981136322021,
      "rewards/margins": 1.6082184314727783,
      "rewards/rejected": -2.6112160682678223,
      "step": 1340
    },
    {
      "epoch": 0.34,
      "learning_rate": 5.313397058378386e-08,
      "logits/chosen": -1.5557955503463745,
      "logits/rejected": -1.6191835403442383,
      "logps/chosen": -44.55752182006836,
      "logps/rejected": -57.750877380371094,
      "loss": 0.4101,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.429595708847046,
      "rewards/margins": 1.3189971446990967,
      "rewards/rejected": -2.7485926151275635,
      "step": 1345
    },
    {
      "epoch": 0.34,
      "learning_rate": 5.240526069629264e-08,
      "logits/chosen": -1.6459201574325562,
      "logits/rejected": -1.6954996585845947,
      "logps/chosen": -30.61590576171875,
      "logps/rejected": -54.773284912109375,
      "loss": 0.4337,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0733121633529663,
      "rewards/margins": 1.3275502920150757,
      "rewards/rejected": -2.400862455368042,
      "step": 1350
    },
    {
      "epoch": 0.34,
      "learning_rate": 5.1679803846010403e-08,
      "logits/chosen": -1.6004453897476196,
      "logits/rejected": -1.636222243309021,
      "logps/chosen": -36.51213073730469,
      "logps/rejected": -56.715606689453125,
      "loss": 0.4328,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.2924304008483887,
      "rewards/margins": 1.5186452865600586,
      "rewards/rejected": -2.811075448989868,
      "step": 1355
    },
    {
      "epoch": 0.34,
      "learning_rate": 5.0957649616949215e-08,
      "logits/chosen": -1.6312261819839478,
      "logits/rejected": -1.640782117843628,
      "logps/chosen": -37.51857376098633,
      "logps/rejected": -70.64649200439453,
      "loss": 0.4218,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0793393850326538,
      "rewards/margins": 1.7332353591918945,
      "rewards/rejected": -2.812574863433838,
      "step": 1360
    },
    {
      "epoch": 0.34,
      "learning_rate": 5.0238847367391314e-08,
      "logits/chosen": -1.5880235433578491,
      "logits/rejected": -1.6467769145965576,
      "logps/chosen": -32.823516845703125,
      "logps/rejected": -61.8253288269043,
      "loss": 0.4022,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0444438457489014,
      "rewards/margins": 1.6926997900009155,
      "rewards/rejected": -2.7371437549591064,
      "step": 1365
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.952344622651565e-08,
      "logits/chosen": -1.6432594060897827,
      "logits/rejected": -1.6623003482818604,
      "logps/chosen": -32.66761779785156,
      "logps/rejected": -61.32550048828125,
      "loss": 0.4036,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.2330167293548584,
      "rewards/margins": 1.3832911252975464,
      "rewards/rejected": -2.6163077354431152,
      "step": 1370
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.8811495091039923e-08,
      "logits/chosen": -1.6371605396270752,
      "logits/rejected": -1.6313444375991821,
      "logps/chosen": -36.848670959472656,
      "logps/rejected": -58.194969177246094,
      "loss": 0.4266,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.425100564956665,
      "rewards/margins": 1.1800484657287598,
      "rewards/rejected": -2.6051487922668457,
      "step": 1375
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.810304262187851e-08,
      "logits/chosen": -1.5991828441619873,
      "logits/rejected": -1.6860853433609009,
      "logps/chosen": -41.5286865234375,
      "logps/rejected": -60.21526336669922,
      "loss": 0.402,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.278834581375122,
      "rewards/margins": 1.2690808773040771,
      "rewards/rejected": -2.547915458679199,
      "step": 1380
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.739813724081661e-08,
      "logits/chosen": -1.6154054403305054,
      "logits/rejected": -1.6397669315338135,
      "logps/chosen": -40.9693489074707,
      "logps/rejected": -64.9931640625,
      "loss": 0.3921,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.4517185688018799,
      "rewards/margins": 1.4309002161026,
      "rewards/rejected": -2.8826186656951904,
      "step": 1385
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.6696827127200644e-08,
      "logits/chosen": -1.7005449533462524,
      "logits/rejected": -1.7360703945159912,
      "logps/chosen": -36.8565559387207,
      "logps/rejected": -56.22419357299805,
      "loss": 0.4363,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3760679960250854,
      "rewards/margins": 1.0964031219482422,
      "rewards/rejected": -2.472470998764038,
      "step": 1390
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.599916021464531e-08,
      "logits/chosen": -1.5377979278564453,
      "logits/rejected": -1.6516033411026,
      "logps/chosen": -39.64506149291992,
      "logps/rejected": -54.380836486816406,
      "loss": 0.4367,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.8013836145401,
      "rewards/margins": 1.0361871719360352,
      "rewards/rejected": -2.8375706672668457,
      "step": 1395
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.530518418775733e-08,
      "logits/chosen": -1.5973961353302002,
      "logits/rejected": -1.6904951333999634,
      "logps/chosen": -30.8753604888916,
      "logps/rejected": -58.85967254638672,
      "loss": 0.435,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.0102612972259521,
      "rewards/margins": 1.7187198400497437,
      "rewards/rejected": -2.7289814949035645,
      "step": 1400
    },
    {
      "epoch": 0.35,
      "eval_logits/chosen": -1.7733038663864136,
      "eval_logits/rejected": -1.8373956680297852,
      "eval_logps/chosen": -40.727088928222656,
      "eval_logps/rejected": -61.374794006347656,
      "eval_loss": 0.4264953136444092,
      "eval_rewards/accuracies": 0.780351459980011,
      "eval_rewards/chosen": -1.3451405763626099,
      "eval_rewards/margins": 1.403651475906372,
      "eval_rewards/rejected": -2.7487919330596924,
      "eval_runtime": 676.5091,
      "eval_samples_per_second": 29.564,
      "eval_steps_per_second": 0.463,
      "step": 1400
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.4614946478876306e-08,
      "logits/chosen": -1.5811793804168701,
      "logits/rejected": -1.6037524938583374,
      "logps/chosen": -41.72180938720703,
      "logps/rejected": -56.50455856323242,
      "loss": 0.4415,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.3052692413330078,
      "rewards/margins": 1.4770625829696655,
      "rewards/rejected": -2.782331943511963,
      "step": 1405
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.392849426483274e-08,
      "logits/chosen": -1.5790523290634155,
      "logits/rejected": -1.6081444025039673,
      "logps/chosen": -36.94837188720703,
      "logps/rejected": -61.35870361328125,
      "loss": 0.432,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3901188373565674,
      "rewards/margins": 1.331687092781067,
      "rewards/rejected": -2.721806049346924,
      "step": 1410
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.324587446372364e-08,
      "logits/chosen": -1.6028099060058594,
      "logits/rejected": -1.6534366607666016,
      "logps/chosen": -39.56840515136719,
      "logps/rejected": -59.75825119018555,
      "loss": 0.4361,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.5059459209442139,
      "rewards/margins": 1.4793611764907837,
      "rewards/rejected": -2.985307216644287,
      "step": 1415
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.256713373170564e-08,
      "logits/chosen": -1.5526655912399292,
      "logits/rejected": -1.6214317083358765,
      "logps/chosen": -38.09294509887695,
      "logps/rejected": -58.60236358642578,
      "loss": 0.4803,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.434067726135254,
      "rewards/margins": 1.3282562494277954,
      "rewards/rejected": -2.762324094772339,
      "step": 1420
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.1892318459806175e-08,
      "logits/chosen": -1.5805480480194092,
      "logits/rejected": -1.6314042806625366,
      "logps/chosen": -37.16118240356445,
      "logps/rejected": -51.327178955078125,
      "loss": 0.424,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4472792148590088,
      "rewards/margins": 1.1773505210876465,
      "rewards/rejected": -2.6246297359466553,
      "step": 1425
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.1221474770752695e-08,
      "logits/chosen": -1.5926434993743896,
      "logits/rejected": -1.6545213460922241,
      "logps/chosen": -34.86847686767578,
      "logps/rejected": -57.7491455078125,
      "loss": 0.4286,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1985056400299072,
      "rewards/margins": 1.478855848312378,
      "rewards/rejected": -2.6773617267608643,
      "step": 1430
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.055464851582021e-08,
      "logits/chosen": -1.6083743572235107,
      "logits/rejected": -1.718679428100586,
      "logps/chosen": -32.2127685546875,
      "logps/rejected": -67.01476287841797,
      "loss": 0.3731,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.0988613367080688,
      "rewards/margins": 2.053067207336426,
      "rewards/rejected": -3.151928186416626,
      "step": 1435
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.989188527169749e-08,
      "logits/chosen": -1.5989800691604614,
      "logits/rejected": -1.6427767276763916,
      "logps/chosen": -36.41338348388672,
      "logps/rejected": -56.013519287109375,
      "loss": 0.409,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3234190940856934,
      "rewards/margins": 1.381638526916504,
      "rewards/rejected": -2.7050576210021973,
      "step": 1440
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.923323033737188e-08,
      "logits/chosen": -1.5973937511444092,
      "logits/rejected": -1.6154829263687134,
      "logps/chosen": -41.698753356933594,
      "logps/rejected": -56.54584503173828,
      "loss": 0.4451,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.5426132678985596,
      "rewards/margins": 1.18071711063385,
      "rewards/rejected": -2.723330497741699,
      "step": 1445
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.857872873103322e-08,
      "logits/chosen": -1.6224912405014038,
      "logits/rejected": -1.6204932928085327,
      "logps/chosen": -36.34685516357422,
      "logps/rejected": -48.54065704345703,
      "loss": 0.453,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.276078462600708,
      "rewards/margins": 1.1468660831451416,
      "rewards/rejected": -2.4229445457458496,
      "step": 1450
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.7928425186996883e-08,
      "logits/chosen": -1.628617525100708,
      "logits/rejected": -1.7156063318252563,
      "logps/chosen": -31.950937271118164,
      "logps/rejected": -52.45146560668945,
      "loss": 0.4136,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2364708185195923,
      "rewards/margins": 1.715372085571289,
      "rewards/rejected": -2.951842784881592,
      "step": 1455
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.7282364152646295e-08,
      "logits/chosen": -1.6578394174575806,
      "logits/rejected": -1.7061634063720703,
      "logps/chosen": -35.03657913208008,
      "logps/rejected": -70.58955383300781,
      "loss": 0.3766,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.331438422203064,
      "rewards/margins": 1.8967783451080322,
      "rewards/rejected": -3.2282166481018066,
      "step": 1460
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.664058978539495e-08,
      "logits/chosen": -1.5942935943603516,
      "logits/rejected": -1.6189035177230835,
      "logps/chosen": -42.353904724121094,
      "logps/rejected": -57.705413818359375,
      "loss": 0.4212,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5647684335708618,
      "rewards/margins": 1.509913682937622,
      "rewards/rejected": -3.0746822357177734,
      "step": 1465
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.600314594966833e-08,
      "logits/chosen": -1.6005092859268188,
      "logits/rejected": -1.6767311096191406,
      "logps/chosen": -43.09211349487305,
      "logps/rejected": -60.247520446777344,
      "loss": 0.4038,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.4567948579788208,
      "rewards/margins": 1.4423904418945312,
      "rewards/rejected": -2.8991851806640625,
      "step": 1470
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.53700762139059e-08,
      "logits/chosen": -1.6018587350845337,
      "logits/rejected": -1.6889028549194336,
      "logps/chosen": -37.60720443725586,
      "logps/rejected": -58.150054931640625,
      "loss": 0.3888,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.441217064857483,
      "rewards/margins": 1.420534372329712,
      "rewards/rejected": -2.8617515563964844,
      "step": 1475
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.474142384758313e-08,
      "logits/chosen": -1.6099767684936523,
      "logits/rejected": -1.699637770652771,
      "logps/chosen": -40.36447525024414,
      "logps/rejected": -59.49592971801758,
      "loss": 0.4391,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.593315839767456,
      "rewards/margins": 1.6204936504364014,
      "rewards/rejected": -3.2138094902038574,
      "step": 1480
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.41172318182542e-08,
      "logits/chosen": -1.634606957435608,
      "logits/rejected": -1.627526044845581,
      "logps/chosen": -34.031497955322266,
      "logps/rejected": -55.68294143676758,
      "loss": 0.4205,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2308284044265747,
      "rewards/margins": 1.326682448387146,
      "rewards/rejected": -2.5575108528137207,
      "step": 1485
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.349754278861516e-08,
      "logits/chosen": -1.647221326828003,
      "logits/rejected": -1.6850544214248657,
      "logps/chosen": -42.39990234375,
      "logps/rejected": -56.22304153442383,
      "loss": 0.4265,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4471819400787354,
      "rewards/margins": 1.2381715774536133,
      "rewards/rejected": -2.6853535175323486,
      "step": 1490
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.2882399113588066e-08,
      "logits/chosen": -1.6549670696258545,
      "logits/rejected": -1.636690378189087,
      "logps/chosen": -35.067543029785156,
      "logps/rejected": -57.54313278198242,
      "loss": 0.3982,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2351791858673096,
      "rewards/margins": 1.4587191343307495,
      "rewards/rejected": -2.6938984394073486,
      "step": 1495
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.227184283742591e-08,
      "logits/chosen": -1.566417932510376,
      "logits/rejected": -1.5962722301483154,
      "logps/chosen": -44.59258270263672,
      "logps/rejected": -54.915374755859375,
      "loss": 0.4202,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.6373296976089478,
      "rewards/margins": 1.1503968238830566,
      "rewards/rejected": -2.787726879119873,
      "step": 1500
    },
    {
      "epoch": 0.38,
      "eval_logits/chosen": -1.7744992971420288,
      "eval_logits/rejected": -1.8385426998138428,
      "eval_logps/chosen": -41.52806091308594,
      "eval_logps/rejected": -62.49430465698242,
      "eval_loss": 0.42402884364128113,
      "eval_rewards/accuracies": 0.7823482155799866,
      "eval_rewards/chosen": -1.425237774848938,
      "eval_rewards/margins": 1.4355047941207886,
      "eval_rewards/rejected": -2.8607423305511475,
      "eval_runtime": 682.38,
      "eval_samples_per_second": 29.309,
      "eval_steps_per_second": 0.459,
      "step": 1500
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.166591569083916e-08,
      "logits/chosen": -1.6541789770126343,
      "logits/rejected": -1.666421890258789,
      "logps/chosen": -51.190147399902344,
      "logps/rejected": -61.54121780395508,
      "loss": 0.4285,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.477967619895935,
      "rewards/margins": 1.5420701503753662,
      "rewards/rejected": -3.0200376510620117,
      "step": 1505
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.106465908814342e-08,
      "logits/chosen": -1.566292643547058,
      "logits/rejected": -1.586924433708191,
      "logps/chosen": -40.33735275268555,
      "logps/rejected": -58.9432258605957,
      "loss": 0.3928,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4586553573608398,
      "rewards/margins": 1.309951663017273,
      "rewards/rejected": -2.7686069011688232,
      "step": 1510
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.04681141244288e-08,
      "logits/chosen": -1.5833979845046997,
      "logits/rejected": -1.6595224142074585,
      "logps/chosen": -35.47198486328125,
      "logps/rejected": -64.98353576660156,
      "loss": 0.437,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.2309479713439941,
      "rewards/margins": 1.9191757440567017,
      "rewards/rejected": -3.1501235961914062,
      "step": 1515
    },
    {
      "epoch": 0.38,
      "learning_rate": 2.987632157275114e-08,
      "logits/chosen": -1.571750283241272,
      "logits/rejected": -1.6132491827011108,
      "logps/chosen": -44.85167694091797,
      "logps/rejected": -68.72865295410156,
      "loss": 0.433,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.527980923652649,
      "rewards/margins": 1.5129810571670532,
      "rewards/rejected": -3.0409622192382812,
      "step": 1520
    },
    {
      "epoch": 0.38,
      "learning_rate": 2.928932188134525e-08,
      "logits/chosen": -1.495047926902771,
      "logits/rejected": -1.5616710186004639,
      "logps/chosen": -41.87842559814453,
      "logps/rejected": -72.66767883300781,
      "loss": 0.4095,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.471101999282837,
      "rewards/margins": 1.7681621313095093,
      "rewards/rejected": -3.2392642498016357,
      "step": 1525
    },
    {
      "epoch": 0.38,
      "learning_rate": 2.8707155170860297e-08,
      "logits/chosen": -1.7271652221679688,
      "logits/rejected": -1.7250388860702515,
      "logps/chosen": -39.89320373535156,
      "logps/rejected": -60.249778747558594,
      "loss": 0.4112,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5881242752075195,
      "rewards/margins": 1.2204608917236328,
      "rewards/rejected": -2.8085849285125732,
      "step": 1530
    },
    {
      "epoch": 0.38,
      "learning_rate": 2.8129861231617612e-08,
      "logits/chosen": -1.5804831981658936,
      "logits/rejected": -1.6398807764053345,
      "logps/chosen": -43.65148162841797,
      "logps/rejected": -62.10638427734375,
      "loss": 0.45,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4383070468902588,
      "rewards/margins": 1.6000694036483765,
      "rewards/rejected": -3.038376569747925,
      "step": 1535
    },
    {
      "epoch": 0.39,
      "learning_rate": 2.7557479520891104e-08,
      "logits/chosen": -1.5704247951507568,
      "logits/rejected": -1.6201874017715454,
      "logps/chosen": -36.51436233520508,
      "logps/rejected": -67.26272583007812,
      "loss": 0.3921,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.2914024591445923,
      "rewards/margins": 1.8169571161270142,
      "rewards/rejected": -3.1083598136901855,
      "step": 1540
    },
    {
      "epoch": 0.39,
      "learning_rate": 2.699004916021038e-08,
      "logits/chosen": -1.6186180114746094,
      "logits/rejected": -1.647109031677246,
      "logps/chosen": -40.04192352294922,
      "logps/rejected": -61.596595764160156,
      "loss": 0.4176,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.2729463577270508,
      "rewards/margins": 1.463202714920044,
      "rewards/rejected": -2.7361490726470947,
      "step": 1545
    },
    {
      "epoch": 0.39,
      "learning_rate": 2.642760893268684e-08,
      "logits/chosen": -1.6283318996429443,
      "logits/rejected": -1.6734449863433838,
      "logps/chosen": -41.37560272216797,
      "logps/rejected": -61.95280838012695,
      "loss": 0.4289,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.6277186870574951,
      "rewards/margins": 1.0712679624557495,
      "rewards/rejected": -2.698986768722534,
      "step": 1550
    },
    {
      "epoch": 0.39,
      "learning_rate": 2.5870197280362915e-08,
      "logits/chosen": -1.5385441780090332,
      "logits/rejected": -1.5929187536239624,
      "logps/chosen": -36.092308044433594,
      "logps/rejected": -59.61371994018555,
      "loss": 0.4337,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.4656503200531006,
      "rewards/margins": 1.491877555847168,
      "rewards/rejected": -2.9575278759002686,
      "step": 1555
    },
    {
      "epoch": 0.39,
      "learning_rate": 2.5317852301584643e-08,
      "logits/chosen": -1.7016273736953735,
      "logits/rejected": -1.7216222286224365,
      "logps/chosen": -42.860923767089844,
      "logps/rejected": -68.04586791992188,
      "loss": 0.4334,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8822638988494873,
      "rewards/margins": 1.0386995077133179,
      "rewards/rejected": -2.9209635257720947,
      "step": 1560
    },
    {
      "epoch": 0.39,
      "learning_rate": 2.477061174839755e-08,
      "logits/chosen": -1.667069435119629,
      "logits/rejected": -1.686384916305542,
      "logps/chosen": -35.156497955322266,
      "logps/rejected": -59.22174835205078,
      "loss": 0.4266,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.3924586772918701,
      "rewards/margins": 1.4962780475616455,
      "rewards/rejected": -2.8887367248535156,
      "step": 1565
    },
    {
      "epoch": 0.39,
      "learning_rate": 2.4228513023966547e-08,
      "logits/chosen": -1.5280156135559082,
      "logits/rejected": -1.576804757118225,
      "logps/chosen": -38.35932159423828,
      "logps/rejected": -57.656890869140625,
      "loss": 0.4513,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.3916337490081787,
      "rewards/margins": 1.4954885244369507,
      "rewards/rejected": -2.887122392654419,
      "step": 1570
    },
    {
      "epoch": 0.39,
      "learning_rate": 2.3691593180019364e-08,
      "logits/chosen": -1.6398168802261353,
      "logits/rejected": -1.6756404638290405,
      "logps/chosen": -37.17683410644531,
      "logps/rejected": -66.59625244140625,
      "loss": 0.4156,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4407873153686523,
      "rewards/margins": 1.6703660488128662,
      "rewards/rejected": -3.1111538410186768,
      "step": 1575
    },
    {
      "epoch": 0.4,
      "learning_rate": 2.315988891431412e-08,
      "logits/chosen": -1.5830076932907104,
      "logits/rejected": -1.6350247859954834,
      "logps/chosen": -39.05982208251953,
      "logps/rejected": -58.4395751953125,
      "loss": 0.4463,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5787169933319092,
      "rewards/margins": 1.208711862564087,
      "rewards/rejected": -2.787428617477417,
      "step": 1580
    },
    {
      "epoch": 0.4,
      "learning_rate": 2.263343656813107e-08,
      "logits/chosen": -1.6600052118301392,
      "logits/rejected": -1.723974585533142,
      "logps/chosen": -38.07198715209961,
      "logps/rejected": -54.6322135925293,
      "loss": 0.391,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5236351490020752,
      "rewards/margins": 1.2169435024261475,
      "rewards/rejected": -2.7405786514282227,
      "step": 1585
    },
    {
      "epoch": 0.4,
      "learning_rate": 2.2112272123788767e-08,
      "logits/chosen": -1.5720638036727905,
      "logits/rejected": -1.6471821069717407,
      "logps/chosen": -33.8635139465332,
      "logps/rejected": -59.320289611816406,
      "loss": 0.4516,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.3420783281326294,
      "rewards/margins": 1.5687716007232666,
      "rewards/rejected": -2.9108500480651855,
      "step": 1590
    },
    {
      "epoch": 0.4,
      "learning_rate": 2.1596431202184705e-08,
      "logits/chosen": -1.6860291957855225,
      "logits/rejected": -1.775101900100708,
      "logps/chosen": -41.8366584777832,
      "logps/rejected": -57.60499954223633,
      "loss": 0.3887,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6872646808624268,
      "rewards/margins": 1.5663306713104248,
      "rewards/rejected": -3.2535953521728516,
      "step": 1595
    },
    {
      "epoch": 0.4,
      "learning_rate": 2.108594906036065e-08,
      "logits/chosen": -1.643472671508789,
      "logits/rejected": -1.6051247119903564,
      "logps/chosen": -34.84891891479492,
      "logps/rejected": -64.89028930664062,
      "loss": 0.4156,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.273041009902954,
      "rewards/margins": 1.3676272630691528,
      "rewards/rejected": -2.6406683921813965,
      "step": 1600
    },
    {
      "epoch": 0.4,
      "eval_logits/chosen": -1.7730110883712769,
      "eval_logits/rejected": -1.8370084762573242,
      "eval_logps/chosen": -41.216156005859375,
      "eval_logps/rejected": -62.23305892944336,
      "eval_loss": 0.4231519401073456,
      "eval_rewards/accuracies": 0.784744381904602,
      "eval_rewards/chosen": -1.3940469026565552,
      "eval_rewards/margins": 1.4405717849731445,
      "eval_rewards/rejected": -2.83461856842041,
      "eval_runtime": 682.593,
      "eval_samples_per_second": 29.3,
      "eval_steps_per_second": 0.459,
      "step": 1600
    },
    {
      "epoch": 0.4,
      "learning_rate": 2.0580860589092895e-08,
      "logits/chosen": -1.6066029071807861,
      "logits/rejected": -1.6168630123138428,
      "logps/chosen": -41.878971099853516,
      "logps/rejected": -57.808631896972656,
      "loss": 0.4474,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.3217754364013672,
      "rewards/margins": 1.4163072109222412,
      "rewards/rejected": -2.7380826473236084,
      "step": 1605
    },
    {
      "epoch": 0.4,
      "learning_rate": 2.008120031050753e-08,
      "logits/chosen": -1.6395975351333618,
      "logits/rejected": -1.647607445716858,
      "logps/chosen": -32.06978225708008,
      "logps/rejected": -61.098243713378906,
      "loss": 0.4582,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.065802812576294,
      "rewards/margins": 1.5604586601257324,
      "rewards/rejected": -2.6262614727020264,
      "step": 1610
    },
    {
      "epoch": 0.4,
      "learning_rate": 1.9587002375720862e-08,
      "logits/chosen": -1.6124694347381592,
      "logits/rejected": -1.6230475902557373,
      "logps/chosen": -37.16108703613281,
      "logps/rejected": -65.89389038085938,
      "loss": 0.4233,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3276442289352417,
      "rewards/margins": 1.7044506072998047,
      "rewards/rejected": -3.032094717025757,
      "step": 1615
    },
    {
      "epoch": 0.41,
      "learning_rate": 1.9098300562505266e-08,
      "logits/chosen": -1.4507148265838623,
      "logits/rejected": -1.503017783164978,
      "logps/chosen": -42.85657501220703,
      "logps/rejected": -58.918609619140625,
      "loss": 0.4174,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.3876417875289917,
      "rewards/margins": 1.4152082204818726,
      "rewards/rejected": -2.8028502464294434,
      "step": 1620
    },
    {
      "epoch": 0.41,
      "learning_rate": 1.8615128272980507e-08,
      "logits/chosen": -1.6832008361816406,
      "logits/rejected": -1.7357594966888428,
      "logps/chosen": -39.325965881347656,
      "logps/rejected": -56.73716354370117,
      "loss": 0.4169,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.447385549545288,
      "rewards/margins": 1.2441480159759521,
      "rewards/rejected": -2.6915335655212402,
      "step": 1625
    },
    {
      "epoch": 0.41,
      "learning_rate": 1.8137518531330763e-08,
      "logits/chosen": -1.6094249486923218,
      "logits/rejected": -1.6434955596923828,
      "logps/chosen": -48.951080322265625,
      "logps/rejected": -68.89022064208984,
      "loss": 0.396,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.743072748184204,
      "rewards/margins": 1.4508569240570068,
      "rewards/rejected": -3.193929672241211,
      "step": 1630
    },
    {
      "epoch": 0.41,
      "learning_rate": 1.7665503981547425e-08,
      "logits/chosen": -1.5964616537094116,
      "logits/rejected": -1.6168407201766968,
      "logps/chosen": -34.803096771240234,
      "logps/rejected": -55.64680862426758,
      "loss": 0.3922,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.1366093158721924,
      "rewards/margins": 1.37692129611969,
      "rewards/rejected": -2.513530731201172,
      "step": 1635
    },
    {
      "epoch": 0.41,
      "learning_rate": 1.7199116885197997e-08,
      "logits/chosen": -1.628090500831604,
      "logits/rejected": -1.6611076593399048,
      "logps/chosen": -37.68436813354492,
      "logps/rejected": -69.68168640136719,
      "loss": 0.4042,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.5950982570648193,
      "rewards/margins": 1.762877106666565,
      "rewards/rejected": -3.3579750061035156,
      "step": 1640
    },
    {
      "epoch": 0.41,
      "learning_rate": 1.6738389119220965e-08,
      "logits/chosen": -1.660617470741272,
      "logits/rejected": -1.707891821861267,
      "logps/chosen": -42.92943572998047,
      "logps/rejected": -61.7951774597168,
      "loss": 0.4284,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.3408089876174927,
      "rewards/margins": 1.690129280090332,
      "rewards/rejected": -3.0309383869171143,
      "step": 1645
    },
    {
      "epoch": 0.41,
      "learning_rate": 1.6283352173747144e-08,
      "logits/chosen": -1.6138479709625244,
      "logits/rejected": -1.6750342845916748,
      "logps/chosen": -46.45161437988281,
      "logps/rejected": -58.84626007080078,
      "loss": 0.4116,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.4797322750091553,
      "rewards/margins": 1.2433393001556396,
      "rewards/rejected": -2.723071575164795,
      "step": 1650
    },
    {
      "epoch": 0.41,
      "learning_rate": 1.5834037149947288e-08,
      "logits/chosen": -1.6072242259979248,
      "logits/rejected": -1.6416151523590088,
      "logps/chosen": -38.2723388671875,
      "logps/rejected": -57.70778274536133,
      "loss": 0.4209,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.4590721130371094,
      "rewards/margins": 1.4725803136825562,
      "rewards/rejected": -2.931652545928955,
      "step": 1655
    },
    {
      "epoch": 0.41,
      "learning_rate": 1.5390474757906446e-08,
      "logits/chosen": -1.5879796743392944,
      "logits/rejected": -1.6024706363677979,
      "logps/chosen": -39.99146270751953,
      "logps/rejected": -63.56482696533203,
      "loss": 0.3969,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.3184869289398193,
      "rewards/margins": 1.4366834163665771,
      "rewards/rejected": -2.7551703453063965,
      "step": 1660
    },
    {
      "epoch": 0.42,
      "learning_rate": 1.495269531452491e-08,
      "logits/chosen": -1.54337477684021,
      "logits/rejected": -1.6411644220352173,
      "logps/chosen": -41.12114334106445,
      "logps/rejected": -64.66667175292969,
      "loss": 0.4173,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.4270210266113281,
      "rewards/margins": 1.6593749523162842,
      "rewards/rejected": -3.0863962173461914,
      "step": 1665
    },
    {
      "epoch": 0.42,
      "learning_rate": 1.4520728741446087e-08,
      "logits/chosen": -1.515159010887146,
      "logits/rejected": -1.5698691606521606,
      "logps/chosen": -36.55290222167969,
      "logps/rejected": -53.955406188964844,
      "loss": 0.4138,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2876484394073486,
      "rewards/margins": 1.5186991691589355,
      "rewards/rejected": -2.806347608566284,
      "step": 1670
    },
    {
      "epoch": 0.42,
      "learning_rate": 1.409460456301147e-08,
      "logits/chosen": -1.5839471817016602,
      "logits/rejected": -1.6540180444717407,
      "logps/chosen": -40.58269500732422,
      "logps/rejected": -59.73209762573242,
      "loss": 0.3761,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4282195568084717,
      "rewards/margins": 1.590958595275879,
      "rewards/rejected": -3.0191779136657715,
      "step": 1675
    },
    {
      "epoch": 0.42,
      "learning_rate": 1.367435190424261e-08,
      "logits/chosen": -1.6678202152252197,
      "logits/rejected": -1.6910464763641357,
      "logps/chosen": -36.54034423828125,
      "logps/rejected": -64.41203308105469,
      "loss": 0.4144,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.320227026939392,
      "rewards/margins": 1.7249934673309326,
      "rewards/rejected": -3.0452206134796143,
      "step": 1680
    },
    {
      "epoch": 0.42,
      "learning_rate": 1.3259999488850471e-08,
      "logits/chosen": -1.6234756708145142,
      "logits/rejected": -1.6420822143554688,
      "logps/chosen": -42.137020111083984,
      "logps/rejected": -67.44734954833984,
      "loss": 0.4129,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.601574182510376,
      "rewards/margins": 1.8647511005401611,
      "rewards/rejected": -3.466325044631958,
      "step": 1685
    },
    {
      "epoch": 0.42,
      "learning_rate": 1.285157563727226e-08,
      "logits/chosen": -1.604988694190979,
      "logits/rejected": -1.6851269006729126,
      "logps/chosen": -33.69852828979492,
      "logps/rejected": -53.167816162109375,
      "loss": 0.4284,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3362572193145752,
      "rewards/margins": 1.3889684677124023,
      "rewards/rejected": -2.7252256870269775,
      "step": 1690
    },
    {
      "epoch": 0.42,
      "learning_rate": 1.244910826473572e-08,
      "logits/chosen": -1.6953544616699219,
      "logits/rejected": -1.7079284191131592,
      "logps/chosen": -42.12826156616211,
      "logps/rejected": -69.13725280761719,
      "loss": 0.4081,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.5907026529312134,
      "rewards/margins": 1.5819746255874634,
      "rewards/rejected": -3.1726772785186768,
      "step": 1695
    },
    {
      "epoch": 0.42,
      "learning_rate": 1.2052624879351104e-08,
      "logits/chosen": -1.5870174169540405,
      "logits/rejected": -1.6522458791732788,
      "logps/chosen": -43.6306266784668,
      "logps/rejected": -68.43583679199219,
      "loss": 0.4484,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.6367082595825195,
      "rewards/margins": 1.6760241985321045,
      "rewards/rejected": -3.312732696533203,
      "step": 1700
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": -1.77384352684021,
      "eval_logits/rejected": -1.83782160282135,
      "eval_logps/chosen": -41.399322509765625,
      "eval_logps/rejected": -62.5122184753418,
      "eval_loss": 0.42222440242767334,
      "eval_rewards/accuracies": 0.7835463285446167,
      "eval_rewards/chosen": -1.41236412525177,
      "eval_rewards/margins": 1.450170636177063,
      "eval_rewards/rejected": -2.862534761428833,
      "eval_runtime": 685.6658,
      "eval_samples_per_second": 29.169,
      "eval_steps_per_second": 0.456,
      "step": 1700
    },
    {
      "epoch": 0.43,
      "learning_rate": 1.1662152580231144e-08,
      "logits/chosen": -1.559308409690857,
      "logits/rejected": -1.614293098449707,
      "logps/chosen": -39.211181640625,
      "logps/rejected": -52.80438995361328,
      "loss": 0.4043,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.3606131076812744,
      "rewards/margins": 1.217910647392273,
      "rewards/rejected": -2.578523635864258,
      "step": 1705
    },
    {
      "epoch": 0.43,
      "learning_rate": 1.1277718055638818e-08,
      "logits/chosen": -1.60663640499115,
      "logits/rejected": -1.6318187713623047,
      "logps/chosen": -40.94214630126953,
      "logps/rejected": -59.09077835083008,
      "loss": 0.3994,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6311317682266235,
      "rewards/margins": 1.4645156860351562,
      "rewards/rejected": -3.095647096633911,
      "step": 1710
    },
    {
      "epoch": 0.43,
      "learning_rate": 1.089934758116322e-08,
      "logits/chosen": -1.6760724782943726,
      "logits/rejected": -1.7328195571899414,
      "logps/chosen": -39.17350387573242,
      "logps/rejected": -69.71881866455078,
      "loss": 0.4338,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.3581247329711914,
      "rewards/margins": 1.6831839084625244,
      "rewards/rejected": -3.041308879852295,
      "step": 1715
    },
    {
      "epoch": 0.43,
      "learning_rate": 1.0527067017923653e-08,
      "logits/chosen": -1.5283730030059814,
      "logits/rejected": -1.5773917436599731,
      "logps/chosen": -42.39814376831055,
      "logps/rejected": -66.12855529785156,
      "loss": 0.3815,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.6773475408554077,
      "rewards/margins": 1.4954215288162231,
      "rewards/rejected": -3.1727688312530518,
      "step": 1720
    },
    {
      "epoch": 0.43,
      "learning_rate": 1.0160901810802114e-08,
      "logits/chosen": -1.5901730060577393,
      "logits/rejected": -1.6413530111312866,
      "logps/chosen": -38.090606689453125,
      "logps/rejected": -59.455177307128906,
      "loss": 0.4193,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.486709713935852,
      "rewards/margins": 1.4121246337890625,
      "rewards/rejected": -2.898834228515625,
      "step": 1725
    },
    {
      "epoch": 0.43,
      "learning_rate": 9.800876986704109e-09,
      "logits/chosen": -1.6886398792266846,
      "logits/rejected": -1.7103347778320312,
      "logps/chosen": -35.96743392944336,
      "logps/rejected": -66.10350799560547,
      "loss": 0.4172,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.0819008350372314,
      "rewards/margins": 1.741093635559082,
      "rewards/rejected": -2.8229947090148926,
      "step": 1730
    },
    {
      "epoch": 0.43,
      "learning_rate": 9.447017152848125e-09,
      "logits/chosen": -1.570709228515625,
      "logits/rejected": -1.6483341455459595,
      "logps/chosen": -37.19647979736328,
      "logps/rejected": -62.676673889160156,
      "loss": 0.3874,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.3310636281967163,
      "rewards/margins": 1.7905714511871338,
      "rewards/rejected": -3.1216349601745605,
      "step": 1735
    },
    {
      "epoch": 0.43,
      "learning_rate": 9.099346495083749e-09,
      "logits/chosen": -1.547473669052124,
      "logits/rejected": -1.6466220617294312,
      "logps/chosen": -40.151947021484375,
      "logps/rejected": -69.61228942871094,
      "loss": 0.4246,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.591925024986267,
      "rewards/margins": 1.6890861988067627,
      "rewards/rejected": -3.2810111045837402,
      "step": 1740
    },
    {
      "epoch": 0.44,
      "learning_rate": 8.75788877623862e-09,
      "logits/chosen": -1.587485432624817,
      "logits/rejected": -1.6669256687164307,
      "logps/chosen": -40.81727981567383,
      "logps/rejected": -59.38299560546875,
      "loss": 0.3917,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.6642837524414062,
      "rewards/margins": 1.4963958263397217,
      "rewards/rejected": -3.160679340362549,
      "step": 1745
    },
    {
      "epoch": 0.44,
      "learning_rate": 8.422667334494249e-09,
      "logits/chosen": -1.6186378002166748,
      "logits/rejected": -1.655207633972168,
      "logps/chosen": -36.024803161621094,
      "logps/rejected": -52.768226623535156,
      "loss": 0.3974,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.1602214574813843,
      "rewards/margins": 1.2412422895431519,
      "rewards/rejected": -2.401463747024536,
      "step": 1750
    },
    {
      "epoch": 0.44,
      "learning_rate": 8.093705081790891e-09,
      "logits/chosen": -1.5935732126235962,
      "logits/rejected": -1.6794706583023071,
      "logps/chosen": -36.518035888671875,
      "logps/rejected": -56.438758850097656,
      "loss": 0.4131,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.366016149520874,
      "rewards/margins": 1.6523935794830322,
      "rewards/rejected": -3.0184097290039062,
      "step": 1755
    },
    {
      "epoch": 0.44,
      "learning_rate": 7.771024502261525e-09,
      "logits/chosen": -1.513396143913269,
      "logits/rejected": -1.6116784811019897,
      "logps/chosen": -40.99394607543945,
      "logps/rejected": -66.16812133789062,
      "loss": 0.4029,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.6409269571304321,
      "rewards/margins": 1.6607745885849,
      "rewards/rejected": -3.301701784133911,
      "step": 1760
    },
    {
      "epoch": 0.44,
      "learning_rate": 7.454647650695157e-09,
      "logits/chosen": -1.6389579772949219,
      "logits/rejected": -1.6486486196517944,
      "logps/chosen": -41.79307174682617,
      "logps/rejected": -59.66328811645508,
      "loss": 0.4002,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.5924742221832275,
      "rewards/margins": 1.3968980312347412,
      "rewards/rejected": -2.9893722534179688,
      "step": 1765
    },
    {
      "epoch": 0.44,
      "learning_rate": 7.144596151029303e-09,
      "logits/chosen": -1.5585296154022217,
      "logits/rejected": -1.5989525318145752,
      "logps/chosen": -40.13111114501953,
      "logps/rejected": -57.73792266845703,
      "loss": 0.4051,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3726533651351929,
      "rewards/margins": 1.4509780406951904,
      "rewards/rejected": -2.823631763458252,
      "step": 1770
    },
    {
      "epoch": 0.44,
      "learning_rate": 6.840891194872111e-09,
      "logits/chosen": -1.5726600885391235,
      "logits/rejected": -1.6179898977279663,
      "logps/chosen": -36.399269104003906,
      "logps/rejected": -57.10075759887695,
      "loss": 0.3965,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4039640426635742,
      "rewards/margins": 1.5349563360214233,
      "rewards/rejected": -2.938920259475708,
      "step": 1775
    },
    {
      "epoch": 0.45,
      "learning_rate": 6.5435535400539254e-09,
      "logits/chosen": -1.5845414400100708,
      "logits/rejected": -1.6111781597137451,
      "logps/chosen": -36.8482666015625,
      "logps/rejected": -65.9016342163086,
      "loss": 0.4049,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.438260555267334,
      "rewards/margins": 1.4630625247955322,
      "rewards/rejected": -2.901322841644287,
      "step": 1780
    },
    {
      "epoch": 0.45,
      "learning_rate": 6.252603509208465e-09,
      "logits/chosen": -1.6787540912628174,
      "logits/rejected": -1.7322124242782593,
      "logps/chosen": -36.04154586791992,
      "logps/rejected": -56.341712951660156,
      "loss": 0.4299,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.421565055847168,
      "rewards/margins": 1.3360097408294678,
      "rewards/rejected": -2.7575747966766357,
      "step": 1785
    },
    {
      "epoch": 0.45,
      "learning_rate": 5.9680609883838825e-09,
      "logits/chosen": -1.7003612518310547,
      "logits/rejected": -1.7145462036132812,
      "logps/chosen": -39.15507507324219,
      "logps/rejected": -60.2194709777832,
      "loss": 0.4203,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5533109903335571,
      "rewards/margins": 1.3238098621368408,
      "rewards/rejected": -2.8771207332611084,
      "step": 1790
    },
    {
      "epoch": 0.45,
      "learning_rate": 5.689945425683473e-09,
      "logits/chosen": -1.5450057983398438,
      "logits/rejected": -1.621883749961853,
      "logps/chosen": -33.54936981201172,
      "logps/rejected": -61.19047927856445,
      "loss": 0.4272,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.2975671291351318,
      "rewards/margins": 1.844926118850708,
      "rewards/rejected": -3.142493486404419,
      "step": 1795
    },
    {
      "epoch": 0.45,
      "learning_rate": 5.418275829936536e-09,
      "logits/chosen": -1.646265983581543,
      "logits/rejected": -1.673706293106079,
      "logps/chosen": -35.322303771972656,
      "logps/rejected": -54.55794143676758,
      "loss": 0.4042,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0582917928695679,
      "rewards/margins": 1.6028120517730713,
      "rewards/rejected": -2.6611039638519287,
      "step": 1800
    },
    {
      "epoch": 0.45,
      "eval_logits/chosen": -1.7735157012939453,
      "eval_logits/rejected": -1.837532639503479,
      "eval_logps/chosen": -41.474159240722656,
      "eval_logps/rejected": -62.62734603881836,
      "eval_loss": 0.4219669997692108,
      "eval_rewards/accuracies": 0.7867411971092224,
      "eval_rewards/chosen": -1.419848084449768,
      "eval_rewards/margins": 1.4541994333267212,
      "eval_rewards/rejected": -2.8740475177764893,
      "eval_runtime": 686.9405,
      "eval_samples_per_second": 29.115,
      "eval_steps_per_second": 0.456,
      "step": 1800
    },
    {
      "epoch": 0.45,
      "learning_rate": 5.15307076939906e-09,
      "logits/chosen": -1.6508762836456299,
      "logits/rejected": -1.6736085414886475,
      "logps/chosen": -34.982662200927734,
      "logps/rejected": -77.63450622558594,
      "loss": 0.4219,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.3135364055633545,
      "rewards/margins": 1.8769088983535767,
      "rewards/rejected": -3.1904451847076416,
      "step": 1805
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.8943483704846465e-09,
      "logits/chosen": -1.6302423477172852,
      "logits/rejected": -1.6854890584945679,
      "logps/chosen": -35.06315612792969,
      "logps/rejected": -58.57770538330078,
      "loss": 0.3989,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.2450635433197021,
      "rewards/margins": 1.4307886362075806,
      "rewards/rejected": -2.6758522987365723,
      "step": 1810
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.6421263165255855e-09,
      "logits/chosen": -1.6691267490386963,
      "logits/rejected": -1.645403265953064,
      "logps/chosen": -37.04888916015625,
      "logps/rejected": -57.073890686035156,
      "loss": 0.492,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.4926297664642334,
      "rewards/margins": 1.0904326438903809,
      "rewards/rejected": -2.5830626487731934,
      "step": 1815
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.396421846564236e-09,
      "logits/chosen": -1.6642389297485352,
      "logits/rejected": -1.6480493545532227,
      "logps/chosen": -38.11848449707031,
      "logps/rejected": -57.89258575439453,
      "loss": 0.4205,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4186296463012695,
      "rewards/margins": 1.2914459705352783,
      "rewards/rejected": -2.710075855255127,
      "step": 1820
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.157251754174729e-09,
      "logits/chosen": -1.5499961376190186,
      "logits/rejected": -1.5773283243179321,
      "logps/chosen": -33.74531936645508,
      "logps/rejected": -61.049644470214844,
      "loss": 0.4196,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.35075044631958,
      "rewards/margins": 1.5633623600006104,
      "rewards/rejected": -2.9141130447387695,
      "step": 1825
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.924632386315185e-09,
      "logits/chosen": -1.5876575708389282,
      "logits/rejected": -1.633788824081421,
      "logps/chosen": -38.55252456665039,
      "logps/rejected": -66.43755340576172,
      "loss": 0.4176,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.3177063465118408,
      "rewards/margins": 1.7444379329681396,
      "rewards/rejected": -3.0621442794799805,
      "step": 1830
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.6985796422103977e-09,
      "logits/chosen": -1.6687253713607788,
      "logits/rejected": -1.7130107879638672,
      "logps/chosen": -38.195884704589844,
      "logps/rejected": -60.389747619628906,
      "loss": 0.402,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4002857208251953,
      "rewards/margins": 1.495273232460022,
      "rewards/rejected": -2.895559072494507,
      "step": 1835
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.4791089722651433e-09,
      "logits/chosen": -1.565155029296875,
      "logits/rejected": -1.611336350440979,
      "logps/chosen": -37.587562561035156,
      "logps/rejected": -55.95196533203125,
      "loss": 0.3799,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.3682587146759033,
      "rewards/margins": 1.4426069259643555,
      "rewards/rejected": -2.810865640640259,
      "step": 1840
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.266235377008175e-09,
      "logits/chosen": -1.5435701608657837,
      "logits/rejected": -1.643436074256897,
      "logps/chosen": -35.90860366821289,
      "logps/rejected": -62.77077102661133,
      "loss": 0.4023,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4303227663040161,
      "rewards/margins": 1.7711102962493896,
      "rewards/rejected": -3.2014331817626953,
      "step": 1845
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.0599734060669624e-09,
      "logits/chosen": -1.5973424911499023,
      "logits/rejected": -1.6078002452850342,
      "logps/chosen": -35.39159393310547,
      "logps/rejected": -62.9890022277832,
      "loss": 0.4134,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6187604665756226,
      "rewards/margins": 1.505409836769104,
      "rewards/rejected": -3.1241705417633057,
      "step": 1850
    },
    {
      "epoch": 0.46,
      "learning_rate": 2.860337157173243e-09,
      "logits/chosen": -1.7700763940811157,
      "logits/rejected": -1.7940946817398071,
      "logps/chosen": -41.7716064453125,
      "logps/rejected": -68.30381774902344,
      "loss": 0.3871,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.6702451705932617,
      "rewards/margins": 1.6783167123794556,
      "rewards/rejected": -3.3485617637634277,
      "step": 1855
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.6673402751994255e-09,
      "logits/chosen": -1.5539026260375977,
      "logits/rejected": -1.6509206295013428,
      "logps/chosen": -32.936622619628906,
      "logps/rejected": -52.0973014831543,
      "loss": 0.4306,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.2196872234344482,
      "rewards/margins": 1.6633917093276978,
      "rewards/rejected": -2.8830788135528564,
      "step": 1860
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.480995951226028e-09,
      "logits/chosen": -1.6145340204238892,
      "logits/rejected": -1.6516563892364502,
      "logps/chosen": -38.63236618041992,
      "logps/rejected": -56.13676071166992,
      "loss": 0.4295,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.4363205432891846,
      "rewards/margins": 1.359151840209961,
      "rewards/rejected": -2.7954723834991455,
      "step": 1865
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.301316921640073e-09,
      "logits/chosen": -1.5776631832122803,
      "logits/rejected": -1.6412086486816406,
      "logps/chosen": -39.75293731689453,
      "logps/rejected": -53.77742385864258,
      "loss": 0.4327,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3652405738830566,
      "rewards/margins": 1.2693113088607788,
      "rewards/rejected": -2.634552001953125,
      "step": 1870
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.128315467264552e-09,
      "logits/chosen": -1.5176748037338257,
      "logits/rejected": -1.5679428577423096,
      "logps/chosen": -33.86006546020508,
      "logps/rejected": -54.69047164916992,
      "loss": 0.3959,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.3462402820587158,
      "rewards/margins": 1.469523310661316,
      "rewards/rejected": -2.815763473510742,
      "step": 1875
    },
    {
      "epoch": 0.47,
      "learning_rate": 1.962003412519064e-09,
      "logits/chosen": -1.62423574924469,
      "logits/rejected": -1.6784271001815796,
      "logps/chosen": -35.614990234375,
      "logps/rejected": -64.7601089477539,
      "loss": 0.4106,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.2779735326766968,
      "rewards/margins": 1.7429111003875732,
      "rewards/rejected": -3.0208849906921387,
      "step": 1880
    },
    {
      "epoch": 0.47,
      "learning_rate": 1.8023921246116402e-09,
      "logits/chosen": -1.5703450441360474,
      "logits/rejected": -1.6502907276153564,
      "logps/chosen": -41.66686248779297,
      "logps/rejected": -62.78812789916992,
      "loss": 0.4231,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.357747197151184,
      "rewards/margins": 1.6436717510223389,
      "rewards/rejected": -3.0014190673828125,
      "step": 1885
    },
    {
      "epoch": 0.47,
      "learning_rate": 1.6494925127617632e-09,
      "logits/chosen": -1.6496837139129639,
      "logits/rejected": -1.6998779773712158,
      "logps/chosen": -38.936195373535156,
      "logps/rejected": -61.56520462036133,
      "loss": 0.4303,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5239100456237793,
      "rewards/margins": 1.3634536266326904,
      "rewards/rejected": -2.887363910675049,
      "step": 1890
    },
    {
      "epoch": 0.47,
      "learning_rate": 1.5033150274548324e-09,
      "logits/chosen": -1.512958288192749,
      "logits/rejected": -1.5619103908538818,
      "logps/chosen": -38.248863220214844,
      "logps/rejected": -60.7032356262207,
      "loss": 0.4007,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4521785974502563,
      "rewards/margins": 1.4607205390930176,
      "rewards/rejected": -2.9128990173339844,
      "step": 1895
    },
    {
      "epoch": 0.47,
      "learning_rate": 1.3638696597277677e-09,
      "logits/chosen": -1.4637887477874756,
      "logits/rejected": -1.4906442165374756,
      "logps/chosen": -35.28557586669922,
      "logps/rejected": -61.87548065185547,
      "loss": 0.4279,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.39443039894104,
      "rewards/margins": 1.446938157081604,
      "rewards/rejected": -2.8413681983947754,
      "step": 1900
    },
    {
      "epoch": 0.47,
      "eval_logits/chosen": -1.7738286256790161,
      "eval_logits/rejected": -1.8378678560256958,
      "eval_logps/chosen": -41.48775863647461,
      "eval_logps/rejected": -62.66281509399414,
      "eval_loss": 0.42165884375572205,
      "eval_rewards/accuracies": 0.7863418459892273,
      "eval_rewards/chosen": -1.421207308769226,
      "eval_rewards/margins": 1.4563862085342407,
      "eval_rewards/rejected": -2.877593755722046,
      "eval_runtime": 691.2085,
      "eval_samples_per_second": 28.935,
      "eval_steps_per_second": 0.453,
      "step": 1900
    },
    {
      "epoch": 0.48,
      "learning_rate": 1.231165940486234e-09,
      "logits/chosen": -1.5569957494735718,
      "logits/rejected": -1.6261913776397705,
      "logps/chosen": -43.071205139160156,
      "logps/rejected": -69.6430435180664,
      "loss": 0.3962,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4279512166976929,
      "rewards/margins": 1.9218699932098389,
      "rewards/rejected": -3.349821090698242,
      "step": 1905
    },
    {
      "epoch": 0.48,
      "learning_rate": 1.1052129398531506e-09,
      "logits/chosen": -1.4935729503631592,
      "logits/rejected": -1.5457624197006226,
      "logps/chosen": -35.38841247558594,
      "logps/rejected": -55.37394332885742,
      "loss": 0.391,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.4084322452545166,
      "rewards/margins": 1.3835333585739136,
      "rewards/rejected": -2.7919657230377197,
      "step": 1910
    },
    {
      "epoch": 0.48,
      "learning_rate": 9.86019266548821e-10,
      "logits/chosen": -1.5539190769195557,
      "logits/rejected": -1.5932285785675049,
      "logps/chosen": -37.63361740112305,
      "logps/rejected": -64.95671081542969,
      "loss": 0.369,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5340850353240967,
      "rewards/margins": 1.674046516418457,
      "rewards/rejected": -3.2081313133239746,
      "step": 1915
    },
    {
      "epoch": 0.48,
      "learning_rate": 8.735930673024805e-10,
      "logits/chosen": -1.5568705797195435,
      "logits/rejected": -1.5749341249465942,
      "logps/chosen": -42.25778579711914,
      "logps/rejected": -62.059730529785156,
      "loss": 0.4085,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.544942855834961,
      "rewards/margins": 1.6132011413574219,
      "rewards/rejected": -3.1581437587738037,
      "step": 1920
    },
    {
      "epoch": 0.48,
      "learning_rate": 7.679420262954983e-10,
      "logits/chosen": -1.615049123764038,
      "logits/rejected": -1.6281379461288452,
      "logps/chosen": -40.41718673706055,
      "logps/rejected": -66.48967742919922,
      "loss": 0.3765,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.5013377666473389,
      "rewards/margins": 1.585729718208313,
      "rewards/rejected": -3.0870673656463623,
      "step": 1925
    },
    {
      "epoch": 0.48,
      "learning_rate": 6.690733646361856e-10,
      "logits/chosen": -1.5112344026565552,
      "logits/rejected": -1.5903434753417969,
      "logps/chosen": -43.65718078613281,
      "logps/rejected": -57.16192626953125,
      "loss": 0.3986,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4910286664962769,
      "rewards/margins": 1.6062946319580078,
      "rewards/rejected": -3.097323179244995,
      "step": 1930
    },
    {
      "epoch": 0.48,
      "learning_rate": 5.769938398662355e-10,
      "logits/chosen": -1.602304458618164,
      "logits/rejected": -1.6733729839324951,
      "logps/chosen": -36.098472595214844,
      "logps/rejected": -61.344871520996094,
      "loss": 0.4255,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.2141350507736206,
      "rewards/margins": 1.717765212059021,
      "rewards/rejected": -2.9318997859954834,
      "step": 1935
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.917097454988583e-10,
      "logits/chosen": -1.621604323387146,
      "logits/rejected": -1.5961339473724365,
      "logps/chosen": -38.64922332763672,
      "logps/rejected": -50.25775909423828,
      "loss": 0.4178,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6571447849273682,
      "rewards/margins": 0.9274013638496399,
      "rewards/rejected": -2.5845460891723633,
      "step": 1940
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.132269105886155e-10,
      "logits/chosen": -1.6364376544952393,
      "logits/rejected": -1.6746025085449219,
      "logps/chosen": -38.60770797729492,
      "logps/rejected": -63.7396125793457,
      "loss": 0.3967,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.5758907794952393,
      "rewards/margins": 1.5158182382583618,
      "rewards/rejected": -3.0917086601257324,
      "step": 1945
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.4155069933301526e-10,
      "logits/chosen": -1.5119609832763672,
      "logits/rejected": -1.5766661167144775,
      "logps/chosen": -35.88676071166992,
      "logps/rejected": -63.843101501464844,
      "loss": 0.3933,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.2250229120254517,
      "rewards/margins": 1.9286922216415405,
      "rewards/rejected": -3.153715133666992,
      "step": 1950
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.7668601070588436e-10,
      "logits/chosen": -1.5765451192855835,
      "logits/rejected": -1.5987513065338135,
      "logps/chosen": -36.32225799560547,
      "logps/rejected": -54.76746368408203,
      "loss": 0.3818,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.4643820524215698,
      "rewards/margins": 1.3365025520324707,
      "rewards/rejected": -2.80088472366333,
      "step": 1955
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.186372781225465e-10,
      "logits/chosen": -1.6033214330673218,
      "logits/rejected": -1.6094167232513428,
      "logps/chosen": -37.742759704589844,
      "logps/rejected": -62.64867401123047,
      "loss": 0.4155,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.5261662006378174,
      "rewards/margins": 1.1791956424713135,
      "rewards/rejected": -2.7053616046905518,
      "step": 1960
    },
    {
      "epoch": 0.49,
      "learning_rate": 1.6740846913674279e-10,
      "logits/chosen": -1.612987756729126,
      "logits/rejected": -1.6750417947769165,
      "logps/chosen": -37.47745895385742,
      "logps/rejected": -54.88645553588867,
      "loss": 0.3985,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.3979930877685547,
      "rewards/margins": 1.492209553718567,
      "rewards/rejected": -2.890202760696411,
      "step": 1965
    },
    {
      "epoch": 0.49,
      "learning_rate": 1.2300308516952628e-10,
      "logits/chosen": -1.6041619777679443,
      "logits/rejected": -1.6311432123184204,
      "logps/chosen": -38.45185852050781,
      "logps/rejected": -69.1260757446289,
      "loss": 0.4106,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.6295732259750366,
      "rewards/margins": 1.7004085779190063,
      "rewards/rejected": -3.329982042312622,
      "step": 1970
    },
    {
      "epoch": 0.49,
      "learning_rate": 8.542416126989804e-11,
      "logits/chosen": -1.5118248462677002,
      "logits/rejected": -1.565796971321106,
      "logps/chosen": -33.17974090576172,
      "logps/rejected": -60.13874435424805,
      "loss": 0.4105,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.143090009689331,
      "rewards/margins": 1.6752750873565674,
      "rewards/rejected": -2.8183650970458984,
      "step": 1975
    },
    {
      "epoch": 0.49,
      "learning_rate": 5.46742659073951e-11,
      "logits/chosen": -1.5557069778442383,
      "logits/rejected": -1.6500135660171509,
      "logps/chosen": -31.341394424438477,
      "logps/rejected": -57.10686492919922,
      "loss": 0.3918,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.228372573852539,
      "rewards/margins": 1.6340805292129517,
      "rewards/rejected": -2.862452983856201,
      "step": 1980
    },
    {
      "epoch": 0.5,
      "learning_rate": 3.0755500796531e-11,
      "logits/chosen": -1.477113962173462,
      "logits/rejected": -1.5181306600570679,
      "logps/chosen": -39.756988525390625,
      "logps/rejected": -61.148597717285156,
      "loss": 0.406,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2764074802398682,
      "rewards/margins": 1.4668867588043213,
      "rewards/rejected": -2.7432944774627686,
      "step": 1985
    },
    {
      "epoch": 0.5,
      "learning_rate": 1.3669500753099584e-11,
      "logits/chosen": -1.6041514873504639,
      "logits/rejected": -1.6731233596801758,
      "logps/chosen": -37.40638732910156,
      "logps/rejected": -59.667518615722656,
      "loss": 0.4275,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.3378410339355469,
      "rewards/margins": 1.7126833200454712,
      "rewards/rejected": -3.0505242347717285,
      "step": 1990
    },
    {
      "epoch": 0.5,
      "learning_rate": 3.417433582542095e-12,
      "logits/chosen": -1.5742517709732056,
      "logits/rejected": -1.5962722301483154,
      "logps/chosen": -32.036895751953125,
      "logps/rejected": -64.07173919677734,
      "loss": 0.4186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2192351818084717,
      "rewards/margins": 1.7447025775909424,
      "rewards/rejected": -2.963937520980835,
      "step": 1995
    },
    {
      "epoch": 0.5,
      "learning_rate": 0.0,
      "logits/chosen": -1.5938119888305664,
      "logits/rejected": -1.6400257349014282,
      "logps/chosen": -37.012535095214844,
      "logps/rejected": -61.68400955200195,
      "loss": 0.4215,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.2453371286392212,
      "rewards/margins": 1.8222535848617554,
      "rewards/rejected": -3.0675911903381348,
      "step": 2000
    },
    {
      "epoch": 0.5,
      "eval_logits/chosen": -1.7737666368484497,
      "eval_logits/rejected": -1.837766170501709,
      "eval_logps/chosen": -41.47916030883789,
      "eval_logps/rejected": -62.65677261352539,
      "eval_loss": 0.42165058851242065,
      "eval_rewards/accuracies": 0.7855431437492371,
      "eval_rewards/chosen": -1.4203476905822754,
      "eval_rewards/margins": 1.4566426277160645,
      "eval_rewards/rejected": -2.8769900798797607,
      "eval_runtime": 689.4497,
      "eval_samples_per_second": 29.009,
      "eval_steps_per_second": 0.454,
      "step": 2000
    }
  ],
  "logging_steps": 5,
  "max_steps": 2000,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}