{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 15284,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 3.270111183780249e-09,
      "logits/chosen": -2.634561777114868,
      "logits/rejected": -2.673060417175293,
      "logps/chosen": -207.5323944091797,
      "logps/rejected": -286.9266052246094,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0,
      "learning_rate": 3.270111183780249e-08,
      "logits/chosen": -2.217411756515503,
      "logits/rejected": -1.964966893196106,
      "logps/chosen": -185.95359802246094,
      "logps/rejected": -165.4095458984375,
      "loss": 0.6934,
      "rewards/accuracies": 0.2222222238779068,
      "rewards/chosen": -0.0003319006646052003,
      "rewards/margins": -0.0005709417164325714,
      "rewards/rejected": 0.00023904111003503203,
      "step": 10
    },
    {
      "epoch": 0.0,
      "learning_rate": 6.540222367560497e-08,
      "logits/chosen": -2.4321327209472656,
      "logits/rejected": -2.22309947013855,
      "logps/chosen": -232.46939086914062,
      "logps/rejected": -231.3448028564453,
      "loss": 0.6936,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00021246029064059258,
      "rewards/margins": -0.0009949529776349664,
      "rewards/rejected": 0.0007824926869943738,
      "step": 20
    },
    {
      "epoch": 0.0,
      "learning_rate": 9.810333551340746e-08,
      "logits/chosen": -2.2580509185791016,
      "logits/rejected": -2.162532091140747,
      "logps/chosen": -197.32386779785156,
      "logps/rejected": -219.0714111328125,
      "loss": 0.6927,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.0010027495445683599,
      "rewards/margins": 0.0008030139142647386,
      "rewards/rejected": 0.000199735484784469,
      "step": 30
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.3080444735120995e-07,
      "logits/chosen": -2.2124183177948,
      "logits/rejected": -2.2520499229431152,
      "logps/chosen": -276.0278015136719,
      "logps/rejected": -265.83056640625,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0004014628066215664,
      "rewards/margins": 0.00031038964516483247,
      "rewards/rejected": -0.0007118523353710771,
      "step": 40
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.6350555918901243e-07,
      "logits/chosen": -2.3488545417785645,
      "logits/rejected": -2.1415200233459473,
      "logps/chosen": -204.8636932373047,
      "logps/rejected": -184.67652893066406,
      "loss": 0.6935,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0005814622272737324,
      "rewards/margins": -0.0007665859302505851,
      "rewards/rejected": 0.0001851238775998354,
      "step": 50
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.9620667102681492e-07,
      "logits/chosen": -2.306964635848999,
      "logits/rejected": -2.0676581859588623,
      "logps/chosen": -209.7549285888672,
      "logps/rejected": -185.88958740234375,
      "loss": 0.6934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.000509511271957308,
      "rewards/margins": -0.0004293478559702635,
      "rewards/rejected": 0.0009388591279275715,
      "step": 60
    },
    {
      "epoch": 0.0,
      "learning_rate": 2.289077828646174e-07,
      "logits/chosen": -2.270014524459839,
      "logits/rejected": -2.157578229904175,
      "logps/chosen": -218.0652618408203,
      "logps/rejected": -207.98446655273438,
      "loss": 0.6932,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.0004429142572917044,
      "rewards/margins": -0.00011908942542504519,
      "rewards/rejected": 0.0005620036390610039,
      "step": 70
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.616088947024199e-07,
      "logits/chosen": -2.5093493461608887,
      "logits/rejected": -2.2287230491638184,
      "logps/chosen": -258.8407287597656,
      "logps/rejected": -213.6261444091797,
      "loss": 0.6934,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0010970671428367496,
      "rewards/margins": -0.0004738843417726457,
      "rewards/rejected": 0.0015709514264017344,
      "step": 80
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.943100065402224e-07,
      "logits/chosen": -2.258004665374756,
      "logits/rejected": -2.173494815826416,
      "logps/chosen": -184.66207885742188,
      "logps/rejected": -165.5071563720703,
      "loss": 0.6928,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.001227168133482337,
      "rewards/margins": 0.0007357962313108146,
      "rewards/rejected": 0.0004913717857562006,
      "step": 90
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.2701111837802487e-07,
      "logits/chosen": -2.4309122562408447,
      "logits/rejected": -2.425734758377075,
      "logps/chosen": -168.77853393554688,
      "logps/rejected": -183.8974609375,
      "loss": 0.6933,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0015637589385733008,
      "rewards/margins": -0.0003523774794302881,
      "rewards/rejected": 0.0019161365926265717,
      "step": 100
    },
    {
      "epoch": 0.01,
      "eval_logits/chosen": -2.349412441253662,
      "eval_logits/rejected": -2.1609179973602295,
      "eval_logps/chosen": -231.77978515625,
      "eval_logps/rejected": -211.4759979248047,
      "eval_loss": 0.6927058100700378,
      "eval_rewards/accuracies": 0.4950000047683716,
      "eval_rewards/chosen": 0.0022517528850585222,
      "eval_rewards/margins": 0.0008927026065066457,
      "eval_rewards/rejected": 0.0013590500457212329,
      "eval_runtime": 735.1102,
      "eval_samples_per_second": 2.721,
      "eval_steps_per_second": 1.36,
      "step": 100
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.5971223021582736e-07,
      "logits/chosen": -2.348449230194092,
      "logits/rejected": -2.0000503063201904,
      "logps/chosen": -222.5101776123047,
      "logps/rejected": -167.06593322753906,
      "loss": 0.6926,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0028537281323224306,
      "rewards/margins": 0.0011668555671349168,
      "rewards/rejected": 0.0016868729144334793,
      "step": 110
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.9241334205362984e-07,
      "logits/chosen": -2.3394782543182373,
      "logits/rejected": -2.2437832355499268,
      "logps/chosen": -224.00106811523438,
      "logps/rejected": -234.1225128173828,
      "loss": 0.6929,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.0029105721041560173,
      "rewards/margins": 0.0004235516826156527,
      "rewards/rejected": 0.002487020567059517,
      "step": 120
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.251144538914324e-07,
      "logits/chosen": -2.2624309062957764,
      "logits/rejected": -2.218439817428589,
      "logps/chosen": -149.37326049804688,
      "logps/rejected": -148.31546020507812,
      "loss": 0.6923,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.002726006554439664,
      "rewards/margins": 0.0016976374899968505,
      "rewards/rejected": 0.0010283689480274916,
      "step": 130
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.578155657292348e-07,
      "logits/chosen": -2.322082996368408,
      "logits/rejected": -2.2224292755126953,
      "logps/chosen": -225.61788940429688,
      "logps/rejected": -159.54702758789062,
      "loss": 0.6922,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0034105959348380566,
      "rewards/margins": 0.001969530712813139,
      "rewards/rejected": 0.0014410652220249176,
      "step": 140
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.905166775670374e-07,
      "logits/chosen": -2.366978406906128,
      "logits/rejected": -2.158670425415039,
      "logps/chosen": -231.0216827392578,
      "logps/rejected": -229.12841796875,
      "loss": 0.6916,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.003339612390846014,
      "rewards/margins": 0.0030463889706879854,
      "rewards/rejected": 0.0002932233619503677,
      "step": 150
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.232177894048398e-07,
      "logits/chosen": -2.2138314247131348,
      "logits/rejected": -2.2267518043518066,
      "logps/chosen": -260.34014892578125,
      "logps/rejected": -224.89797973632812,
      "loss": 0.691,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0052833338268101215,
      "rewards/margins": 0.004268099553883076,
      "rewards/rejected": 0.0010152345057576895,
      "step": 160
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.559189012426422e-07,
      "logits/chosen": -2.313904047012329,
      "logits/rejected": -2.036227226257324,
      "logps/chosen": -180.5023193359375,
      "logps/rejected": -156.74676513671875,
      "loss": 0.6921,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.003502651583403349,
      "rewards/margins": 0.002065155887976289,
      "rewards/rejected": 0.001437495811842382,
      "step": 170
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.886200130804448e-07,
      "logits/chosen": -2.397557258605957,
      "logits/rejected": -2.3370261192321777,
      "logps/chosen": -217.43923950195312,
      "logps/rejected": -198.5919647216797,
      "loss": 0.6903,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00789332203567028,
      "rewards/margins": 0.005768561270087957,
      "rewards/rejected": 0.002124760765582323,
      "step": 180
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.213211249182473e-07,
      "logits/chosen": -2.068686008453369,
      "logits/rejected": -2.1735379695892334,
      "logps/chosen": -190.86375427246094,
      "logps/rejected": -208.5029754638672,
      "loss": 0.6908,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.006324954330921173,
      "rewards/margins": 0.004828146193176508,
      "rewards/rejected": 0.001496807555668056,
      "step": 190
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.540222367560497e-07,
      "logits/chosen": -2.2759697437286377,
      "logits/rejected": -2.2394649982452393,
      "logps/chosen": -146.51278686523438,
      "logps/rejected": -177.26605224609375,
      "loss": 0.691,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00699248630553484,
      "rewards/margins": 0.0043672826141119,
      "rewards/rejected": 0.002625203924253583,
      "step": 200
    },
    {
      "epoch": 0.01,
      "eval_logits/chosen": -2.346784830093384,
      "eval_logits/rejected": -2.158590316772461,
      "eval_logps/chosen": -231.0670166015625,
      "eval_logps/rejected": -211.30332946777344,
      "eval_loss": 0.6900485754013062,
      "eval_rewards/accuracies": 0.5824999809265137,
      "eval_rewards/chosen": 0.009379291906952858,
      "eval_rewards/margins": 0.0062935808673501015,
      "eval_rewards/rejected": 0.003085711970925331,
      "eval_runtime": 732.1877,
      "eval_samples_per_second": 2.732,
      "eval_steps_per_second": 1.366,
      "step": 200
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.867233485938523e-07,
      "logits/chosen": -2.4248156547546387,
      "logits/rejected": -2.2009010314941406,
      "logps/chosen": -218.31198120117188,
      "logps/rejected": -187.88546752929688,
      "loss": 0.6885,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.012215192429721355,
      "rewards/margins": 0.009410833939909935,
      "rewards/rejected": 0.002804358024150133,
      "step": 210
    },
    {
      "epoch": 0.01,
      "learning_rate": 7.194244604316547e-07,
      "logits/chosen": -2.211219072341919,
      "logits/rejected": -2.0395007133483887,
      "logps/chosen": -182.6593475341797,
      "logps/rejected": -174.7406768798828,
      "loss": 0.6908,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.006770218722522259,
      "rewards/margins": 0.004744062200188637,
      "rewards/rejected": 0.0020261567551642656,
      "step": 220
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.521255722694571e-07,
      "logits/chosen": -2.418362855911255,
      "logits/rejected": -2.0232901573181152,
      "logps/chosen": -278.5413513183594,
      "logps/rejected": -183.809814453125,
      "loss": 0.6896,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012379782274365425,
      "rewards/margins": 0.007237741257995367,
      "rewards/rejected": 0.005142040550708771,
      "step": 230
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.848266841072597e-07,
      "logits/chosen": -2.197833299636841,
      "logits/rejected": -2.1625285148620605,
      "logps/chosen": -214.4978485107422,
      "logps/rejected": -206.1327667236328,
      "loss": 0.6872,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013530202209949493,
      "rewards/margins": 0.012023130431771278,
      "rewards/rejected": 0.0015070696827024221,
      "step": 240
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.175277959450622e-07,
      "logits/chosen": -2.164132595062256,
      "logits/rejected": -2.3230721950531006,
      "logps/chosen": -218.02047729492188,
      "logps/rejected": -220.7309112548828,
      "loss": 0.6874,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02405674383044243,
      "rewards/margins": 0.011705279350280762,
      "rewards/rejected": 0.012351466342806816,
      "step": 250
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.502289077828648e-07,
      "logits/chosen": -2.500403642654419,
      "logits/rejected": -2.140150547027588,
      "logps/chosen": -253.9658203125,
      "logps/rejected": -189.07437133789062,
      "loss": 0.686,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03189177066087723,
      "rewards/margins": 0.014671159908175468,
      "rewards/rejected": 0.01722061261534691,
      "step": 260
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.829300196206672e-07,
      "logits/chosen": -2.4172310829162598,
      "logits/rejected": -2.1231706142425537,
      "logps/chosen": -246.13687133789062,
      "logps/rejected": -230.90869140625,
      "loss": 0.6834,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.0267398152500391,
      "rewards/margins": 0.020012779161334038,
      "rewards/rejected": 0.006727036088705063,
      "step": 270
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.156311314584696e-07,
      "logits/chosen": -2.30179500579834,
      "logits/rejected": -2.1886558532714844,
      "logps/chosen": -159.56088256835938,
      "logps/rejected": -146.3377227783203,
      "loss": 0.6871,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.02627565525472164,
      "rewards/margins": 0.012460910715162754,
      "rewards/rejected": 0.013814744539558887,
      "step": 280
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.483322432962722e-07,
      "logits/chosen": -2.547403335571289,
      "logits/rejected": -2.1543164253234863,
      "logps/chosen": -281.8327941894531,
      "logps/rejected": -225.8657684326172,
      "loss": 0.6858,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.033460211008787155,
      "rewards/margins": 0.015266304835677147,
      "rewards/rejected": 0.01819390431046486,
      "step": 290
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.810333551340747e-07,
      "logits/chosen": -2.336268901824951,
      "logits/rejected": -2.1922969818115234,
      "logps/chosen": -264.8150634765625,
      "logps/rejected": -238.97476196289062,
      "loss": 0.6796,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03844591975212097,
      "rewards/margins": 0.02809033915400505,
      "rewards/rejected": 0.01035558246076107,
      "step": 300
    },
    {
      "epoch": 0.02,
      "eval_logits/chosen": -2.3478732109069824,
      "eval_logits/rejected": -2.159766674041748,
      "eval_logps/chosen": -228.3676300048828,
      "eval_logps/rejected": -210.05612182617188,
      "eval_loss": 0.683213472366333,
      "eval_rewards/accuracies": 0.578499972820282,
      "eval_rewards/chosen": 0.036373071372509,
      "eval_rewards/margins": 0.02081523835659027,
      "eval_rewards/rejected": 0.015557833015918732,
      "eval_runtime": 734.681,
      "eval_samples_per_second": 2.722,
      "eval_steps_per_second": 1.361,
      "step": 300
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0137344669718771e-06,
      "logits/chosen": -2.345863103866577,
      "logits/rejected": -2.3596839904785156,
      "logps/chosen": -167.11477661132812,
      "logps/rejected": -156.09837341308594,
      "loss": 0.6902,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.023755531758069992,
      "rewards/margins": 0.006341875996440649,
      "rewards/rejected": 0.017413653433322906,
      "step": 310
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0464355788096796e-06,
      "logits/chosen": -2.451446056365967,
      "logits/rejected": -2.0573291778564453,
      "logps/chosen": -221.1422119140625,
      "logps/rejected": -192.30712890625,
      "loss": 0.6841,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.040228985249996185,
      "rewards/margins": 0.019158074632287025,
      "rewards/rejected": 0.02107091248035431,
      "step": 320
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.079136690647482e-06,
      "logits/chosen": -2.431704044342041,
      "logits/rejected": -2.1997568607330322,
      "logps/chosen": -202.93020629882812,
      "logps/rejected": -176.2235870361328,
      "loss": 0.6766,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.054810572415590286,
      "rewards/margins": 0.034405697137117386,
      "rewards/rejected": 0.02040487341582775,
      "step": 330
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.1118378024852844e-06,
      "logits/chosen": -2.1808204650878906,
      "logits/rejected": -2.346876621246338,
      "logps/chosen": -150.7911834716797,
      "logps/rejected": -178.24876403808594,
      "loss": 0.6939,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.02923217974603176,
      "rewards/margins": 0.00014748350076843053,
      "rewards/rejected": 0.029084693640470505,
      "step": 340
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.144538914323087e-06,
      "logits/chosen": -2.4173805713653564,
      "logits/rejected": -1.9806461334228516,
      "logps/chosen": -317.50555419921875,
      "logps/rejected": -248.4846649169922,
      "loss": 0.6748,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04605143517255783,
      "rewards/margins": 0.03825680539011955,
      "rewards/rejected": 0.007794632576406002,
      "step": 350
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.1772400261608895e-06,
      "logits/chosen": -2.4809823036193848,
      "logits/rejected": -2.18863582611084,
      "logps/chosen": -219.45248413085938,
      "logps/rejected": -193.30197143554688,
      "loss": 0.6742,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06027550250291824,
      "rewards/margins": 0.040139321237802505,
      "rewards/rejected": 0.02013617567718029,
      "step": 360
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.2099411379986922e-06,
      "logits/chosen": -2.163193464279175,
      "logits/rejected": -2.23931884765625,
      "logps/chosen": -191.78585815429688,
      "logps/rejected": -206.739013671875,
      "loss": 0.6833,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.05710885673761368,
      "rewards/margins": 0.021854441612958908,
      "rewards/rejected": 0.03525441139936447,
      "step": 370
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.2426422498364946e-06,
      "logits/chosen": -2.3214056491851807,
      "logits/rejected": -2.0346055030822754,
      "logps/chosen": -215.81735229492188,
      "logps/rejected": -163.44287109375,
      "loss": 0.6685,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.058308642357587814,
      "rewards/margins": 0.05202876403927803,
      "rewards/rejected": 0.0062798731960356236,
      "step": 380
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2753433616742968e-06,
      "logits/chosen": -2.319272041320801,
      "logits/rejected": -2.2392351627349854,
      "logps/chosen": -181.83956909179688,
      "logps/rejected": -245.88583374023438,
      "loss": 0.6778,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03461431339383125,
      "rewards/margins": 0.0336410291492939,
      "rewards/rejected": 0.0009732857579365373,
      "step": 390
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3080444735120995e-06,
      "logits/chosen": -2.477436065673828,
      "logits/rejected": -2.1102676391601562,
      "logps/chosen": -220.322265625,
      "logps/rejected": -183.0004119873047,
      "loss": 0.6558,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.05170411616563797,
      "rewards/margins": 0.07928358018398285,
      "rewards/rejected": -0.027579456567764282,
      "step": 400
    },
    {
      "epoch": 0.03,
      "eval_logits/chosen": -2.3431003093719482,
      "eval_logits/rejected": -2.1555795669555664,
      "eval_logps/chosen": -228.5252685546875,
      "eval_logps/rejected": -213.00389099121094,
      "eval_loss": 0.6709386110305786,
      "eval_rewards/accuracies": 0.6029999852180481,
      "eval_rewards/chosen": 0.03479691967368126,
      "eval_rewards/margins": 0.0487169548869133,
      "eval_rewards/rejected": -0.013920034281909466,
      "eval_runtime": 730.9676,
      "eval_samples_per_second": 2.736,
      "eval_steps_per_second": 1.368,
      "step": 400
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3407455853499021e-06,
      "logits/chosen": -2.4622159004211426,
      "logits/rejected": -2.3118896484375,
      "logps/chosen": -258.96173095703125,
      "logps/rejected": -228.4738311767578,
      "loss": 0.6662,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0339154414832592,
      "rewards/margins": 0.05908499285578728,
      "rewards/rejected": -0.025169555097818375,
      "step": 410
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3734466971877046e-06,
      "logits/chosen": -2.288273811340332,
      "logits/rejected": -2.189143657684326,
      "logps/chosen": -178.7388153076172,
      "logps/rejected": -177.989013671875,
      "loss": 0.6625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03180820122361183,
      "rewards/margins": 0.06718931347131729,
      "rewards/rejected": -0.03538110852241516,
      "step": 420
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.406147809025507e-06,
      "logits/chosen": -2.280384063720703,
      "logits/rejected": -2.0939345359802246,
      "logps/chosen": -212.7125244140625,
      "logps/rejected": -187.62887573242188,
      "loss": 0.6598,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.024347607046365738,
      "rewards/margins": 0.07415790855884552,
      "rewards/rejected": -0.04981030896306038,
      "step": 430
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.4388489208633094e-06,
      "logits/chosen": -2.3787431716918945,
      "logits/rejected": -2.0949296951293945,
      "logps/chosen": -253.77340698242188,
      "logps/rejected": -229.16744995117188,
      "loss": 0.6698,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007238357327878475,
      "rewards/margins": 0.05305751413106918,
      "rewards/rejected": -0.04581916332244873,
      "step": 440
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.471550032701112e-06,
      "logits/chosen": -2.3988170623779297,
      "logits/rejected": -2.199057102203369,
      "logps/chosen": -188.98382568359375,
      "logps/rejected": -200.36709594726562,
      "loss": 0.6693,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0066403718665242195,
      "rewards/margins": 0.05581866577267647,
      "rewards/rejected": -0.049178291112184525,
      "step": 450
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5042511445389143e-06,
      "logits/chosen": -2.1432154178619385,
      "logits/rejected": -2.191903591156006,
      "logps/chosen": -154.96592712402344,
      "logps/rejected": -223.0820770263672,
      "loss": 0.6729,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04025929421186447,
      "rewards/margins": 0.04690135270357132,
      "rewards/rejected": -0.0871606320142746,
      "step": 460
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.536952256376717e-06,
      "logits/chosen": -2.03416109085083,
      "logits/rejected": -2.096924304962158,
      "logps/chosen": -215.8236846923828,
      "logps/rejected": -269.8562316894531,
      "loss": 0.6576,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14893050491809845,
      "rewards/margins": 0.08468537777662277,
      "rewards/rejected": -0.233615905046463,
      "step": 470
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5696533682145194e-06,
      "logits/chosen": -2.428750514984131,
      "logits/rejected": -2.1963045597076416,
      "logps/chosen": -199.69581604003906,
      "logps/rejected": -173.8529815673828,
      "loss": 0.6704,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.15449048578739166,
      "rewards/margins": 0.05215887352824211,
      "rewards/rejected": -0.20664937794208527,
      "step": 480
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.602354480052322e-06,
      "logits/chosen": -2.404479503631592,
      "logits/rejected": -2.315060615539551,
      "logps/chosen": -278.8482666015625,
      "logps/rejected": -232.26596069335938,
      "loss": 0.6554,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12208490073680878,
      "rewards/margins": 0.08369401842355728,
      "rewards/rejected": -0.20577891170978546,
      "step": 490
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.6350555918901245e-06,
      "logits/chosen": -2.1991093158721924,
      "logits/rejected": -1.8723251819610596,
      "logps/chosen": -228.2333984375,
      "logps/rejected": -223.07217407226562,
      "loss": 0.6509,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1016424298286438,
      "rewards/margins": 0.0938231348991394,
      "rewards/rejected": -0.1954655647277832,
      "step": 500
    },
    {
      "epoch": 0.03,
      "eval_logits/chosen": -2.339709520339966,
      "eval_logits/rejected": -2.152255058288574,
      "eval_logps/chosen": -238.85260009765625,
      "eval_logps/rejected": -228.26219177246094,
      "eval_loss": 0.6524506211280823,
      "eval_rewards/accuracies": 0.6060000061988831,
      "eval_rewards/chosen": -0.06847651302814484,
      "eval_rewards/margins": 0.09802625328302383,
      "eval_rewards/rejected": -0.16650275886058807,
      "eval_runtime": 732.3658,
      "eval_samples_per_second": 2.731,
      "eval_steps_per_second": 1.365,
      "step": 500
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.6677567037279269e-06,
      "logits/chosen": -2.4804911613464355,
      "logits/rejected": -2.132293701171875,
      "logps/chosen": -297.90289306640625,
      "logps/rejected": -274.1810302734375,
      "loss": 0.6225,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.016392212361097336,
      "rewards/margins": 0.16663143038749695,
      "rewards/rejected": -0.18302366137504578,
      "step": 510
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.7004578155657295e-06,
      "logits/chosen": -2.290311098098755,
      "logits/rejected": -2.3183419704437256,
      "logps/chosen": -209.4591064453125,
      "logps/rejected": -201.08511352539062,
      "loss": 0.6408,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.027397459372878075,
      "rewards/margins": 0.12046666443347931,
      "rewards/rejected": -0.14786411821842194,
      "step": 520
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.7331589274035318e-06,
      "logits/chosen": -2.177031993865967,
      "logits/rejected": -2.0574111938476562,
      "logps/chosen": -192.4308319091797,
      "logps/rejected": -207.4688720703125,
      "loss": 0.6297,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08504272997379303,
      "rewards/margins": 0.15592262148857117,
      "rewards/rejected": -0.2409653663635254,
      "step": 530
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.7658600392413344e-06,
      "logits/chosen": -2.390712022781372,
      "logits/rejected": -2.231996536254883,
      "logps/chosen": -214.75546264648438,
      "logps/rejected": -217.5111846923828,
      "loss": 0.6082,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11900047212839127,
      "rewards/margins": 0.20125310122966766,
      "rewards/rejected": -0.3202535808086395,
      "step": 540
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.7985611510791368e-06,
      "logits/chosen": -2.3818039894104004,
      "logits/rejected": -1.9254570007324219,
      "logps/chosen": -279.2803649902344,
      "logps/rejected": -284.9852294921875,
      "loss": 0.6098,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15730682015419006,
      "rewards/margins": 0.22093351185321808,
      "rewards/rejected": -0.37824031710624695,
      "step": 550
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8312622629169393e-06,
      "logits/chosen": -2.308039903640747,
      "logits/rejected": -2.169069766998291,
      "logps/chosen": -288.8381042480469,
      "logps/rejected": -263.2790222167969,
      "loss": 0.6716,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.25383657217025757,
      "rewards/margins": 0.08961613476276398,
      "rewards/rejected": -0.34345269203186035,
      "step": 560
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8639633747547417e-06,
      "logits/chosen": -2.396559000015259,
      "logits/rejected": -2.2422266006469727,
      "logps/chosen": -238.0849151611328,
      "logps/rejected": -224.30810546875,
      "loss": 0.6036,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08939802646636963,
      "rewards/margins": 0.21860051155090332,
      "rewards/rejected": -0.30799856781959534,
      "step": 570
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8966644865925443e-06,
      "logits/chosen": -2.2427303791046143,
      "logits/rejected": -2.270328998565674,
      "logps/chosen": -265.56524658203125,
      "logps/rejected": -288.98272705078125,
      "loss": 0.6622,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.26356470584869385,
      "rewards/margins": 0.12023051083087921,
      "rewards/rejected": -0.38379520177841187,
      "step": 580
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9293655984303466e-06,
      "logits/chosen": -2.6671719551086426,
      "logits/rejected": -2.251417875289917,
      "logps/chosen": -302.67108154296875,
      "logps/rejected": -241.68118286132812,
      "loss": 0.6275,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.19704832136631012,
      "rewards/margins": 0.22014763951301575,
      "rewards/rejected": -0.4171959459781647,
      "step": 590
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9620667102681494e-06,
      "logits/chosen": -2.4273905754089355,
      "logits/rejected": -2.434626817703247,
      "logps/chosen": -206.70437622070312,
      "logps/rejected": -203.82342529296875,
      "loss": 0.6521,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.17451904714107513,
      "rewards/margins": 0.11439670622348785,
      "rewards/rejected": -0.288915753364563,
      "step": 600
    },
    {
      "epoch": 0.04,
      "eval_logits/chosen": -2.395611047744751,
      "eval_logits/rejected": -2.204347610473633,
      "eval_logps/chosen": -246.47792053222656,
      "eval_logps/rejected": -243.2220001220703,
      "eval_loss": 0.6305904388427734,
      "eval_rewards/accuracies": 0.6010000109672546,
      "eval_rewards/chosen": -0.14472965896129608,
      "eval_rewards/margins": 0.1713714748620987,
      "eval_rewards/rejected": -0.3161011338233948,
      "eval_runtime": 732.687,
      "eval_samples_per_second": 2.73,
      "eval_steps_per_second": 1.365,
      "step": 600
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.994767822105952e-06,
      "logits/chosen": -2.361133098602295,
      "logits/rejected": -2.2484490871429443,
      "logps/chosen": -193.17393493652344,
      "logps/rejected": -212.9612274169922,
      "loss": 0.6562,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.17970283329486847,
      "rewards/margins": 0.1335226595401764,
      "rewards/rejected": -0.31322550773620605,
      "step": 610
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0274689339437543e-06,
      "logits/chosen": -2.2839503288269043,
      "logits/rejected": -2.068014621734619,
      "logps/chosen": -286.80120849609375,
      "logps/rejected": -266.9192199707031,
      "loss": 0.6457,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08111458271741867,
      "rewards/margins": 0.15472231805324554,
      "rewards/rejected": -0.2358369082212448,
      "step": 620
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0601700457815567e-06,
      "logits/chosen": -2.422149658203125,
      "logits/rejected": -2.0538346767425537,
      "logps/chosen": -281.3658142089844,
      "logps/rejected": -246.43167114257812,
      "loss": 0.6705,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.22011089324951172,
      "rewards/margins": 0.08579105138778687,
      "rewards/rejected": -0.3059019446372986,
      "step": 630
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.092871157619359e-06,
      "logits/chosen": -2.465773105621338,
      "logits/rejected": -2.275972366333008,
      "logps/chosen": -199.1258087158203,
      "logps/rejected": -225.9133758544922,
      "loss": 0.6435,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.22463631629943848,
      "rewards/margins": 0.13076074421405792,
      "rewards/rejected": -0.3553970456123352,
      "step": 640
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.1255722694571616e-06,
      "logits/chosen": -2.4932544231414795,
      "logits/rejected": -2.0854344367980957,
      "logps/chosen": -312.80609130859375,
      "logps/rejected": -245.77651977539062,
      "loss": 0.6011,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.33911433815956116,
      "rewards/margins": 0.2593802511692047,
      "rewards/rejected": -0.5984946489334106,
      "step": 650
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.158273381294964e-06,
      "logits/chosen": -2.4111924171447754,
      "logits/rejected": -2.306203603744507,
      "logps/chosen": -266.17041015625,
      "logps/rejected": -277.33294677734375,
      "loss": 0.6803,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.5374342799186707,
      "rewards/margins": 0.05651041865348816,
      "rewards/rejected": -0.5939447283744812,
      "step": 660
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.190974493132767e-06,
      "logits/chosen": -2.3987817764282227,
      "logits/rejected": -2.085740566253662,
      "logps/chosen": -292.57562255859375,
      "logps/rejected": -249.35812377929688,
      "loss": 0.6427,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5489651560783386,
      "rewards/margins": 0.13039323687553406,
      "rewards/rejected": -0.6793584227561951,
      "step": 670
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.223675604970569e-06,
      "logits/chosen": -2.4640307426452637,
      "logits/rejected": -2.2579846382141113,
      "logps/chosen": -271.7763977050781,
      "logps/rejected": -270.57086181640625,
      "loss": 0.634,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6104215383529663,
      "rewards/margins": 0.16124722361564636,
      "rewards/rejected": -0.7716687917709351,
      "step": 680
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.2563767168083718e-06,
      "logits/chosen": -2.558981418609619,
      "logits/rejected": -2.049004077911377,
      "logps/chosen": -307.95623779296875,
      "logps/rejected": -265.10205078125,
      "loss": 0.6143,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5070446729660034,
      "rewards/margins": 0.2017807513475418,
      "rewards/rejected": -0.7088254690170288,
      "step": 690
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.289077828646174e-06,
      "logits/chosen": -2.3857240676879883,
      "logits/rejected": -2.15393328666687,
      "logps/chosen": -295.8628845214844,
      "logps/rejected": -278.58331298828125,
      "loss": 0.6828,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5680192708969116,
      "rewards/margins": 0.07425983995199203,
      "rewards/rejected": -0.6422791481018066,
      "step": 700
    },
    {
      "epoch": 0.05,
      "eval_logits/chosen": -2.4134950637817383,
      "eval_logits/rejected": -2.220484733581543,
      "eval_logps/chosen": -279.9759521484375,
      "eval_logps/rejected": -274.9947204589844,
      "eval_loss": 0.6355435848236084,
      "eval_rewards/accuracies": 0.5995000004768372,
      "eval_rewards/chosen": -0.4797098934650421,
      "eval_rewards/margins": 0.15411828458309174,
      "eval_rewards/rejected": -0.6338281631469727,
      "eval_runtime": 735.4698,
      "eval_samples_per_second": 2.719,
      "eval_steps_per_second": 1.36,
      "step": 700
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.3217789404839766e-06,
      "logits/chosen": -2.271242141723633,
      "logits/rejected": -2.3510873317718506,
      "logps/chosen": -203.15206909179688,
      "logps/rejected": -256.40960693359375,
      "loss": 0.6759,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.46320945024490356,
      "rewards/margins": 0.06314848363399506,
      "rewards/rejected": -0.526357889175415,
      "step": 710
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.354480052321779e-06,
      "logits/chosen": -2.6204721927642822,
      "logits/rejected": -2.177229404449463,
      "logps/chosen": -301.5008544921875,
      "logps/rejected": -259.2355041503906,
      "loss": 0.6273,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4311140179634094,
      "rewards/margins": 0.17375989258289337,
      "rewards/rejected": -0.6048739552497864,
      "step": 720
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.3871811641595815e-06,
      "logits/chosen": -2.4361162185668945,
      "logits/rejected": -2.244396686553955,
      "logps/chosen": -279.8239440917969,
      "logps/rejected": -245.77609252929688,
      "loss": 0.5919,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.330314964056015,
      "rewards/margins": 0.246080681681633,
      "rewards/rejected": -0.5763957500457764,
      "step": 730
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4198822759973843e-06,
      "logits/chosen": -2.274948835372925,
      "logits/rejected": -2.289696216583252,
      "logps/chosen": -227.0866241455078,
      "logps/rejected": -250.60888671875,
      "loss": 0.6702,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.36480557918548584,
      "rewards/margins": 0.09589744359254837,
      "rewards/rejected": -0.4607029855251312,
      "step": 740
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4525833878351864e-06,
      "logits/chosen": -2.552551746368408,
      "logits/rejected": -2.348984479904175,
      "logps/chosen": -281.94403076171875,
      "logps/rejected": -228.70938110351562,
      "loss": 0.656,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2914310395717621,
      "rewards/margins": 0.13045336306095123,
      "rewards/rejected": -0.4218843877315521,
      "step": 750
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4852844996729892e-06,
      "logits/chosen": -2.296828031539917,
      "logits/rejected": -2.2030820846557617,
      "logps/chosen": -268.18157958984375,
      "logps/rejected": -303.1499328613281,
      "loss": 0.5807,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2905727028846741,
      "rewards/margins": 0.33510199189186096,
      "rewards/rejected": -0.6256746053695679,
      "step": 760
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5179856115107916e-06,
      "logits/chosen": -2.457963228225708,
      "logits/rejected": -2.1134114265441895,
      "logps/chosen": -301.0068359375,
      "logps/rejected": -267.90350341796875,
      "loss": 0.6252,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.27510613203048706,
      "rewards/margins": 0.20844988524913788,
      "rewards/rejected": -0.48355597257614136,
      "step": 770
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5506867233485937e-06,
      "logits/chosen": -2.5407819747924805,
      "logits/rejected": -2.106503486633301,
      "logps/chosen": -283.8204650878906,
      "logps/rejected": -266.38104248046875,
      "loss": 0.5409,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18124490976333618,
      "rewards/margins": 0.400592565536499,
      "rewards/rejected": -0.5818374752998352,
      "step": 780
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5833878351863965e-06,
      "logits/chosen": -2.586670398712158,
      "logits/rejected": -2.5163044929504395,
      "logps/chosen": -272.54180908203125,
      "logps/rejected": -296.7222900390625,
      "loss": 0.6022,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.20543861389160156,
      "rewards/margins": 0.26800888776779175,
      "rewards/rejected": -0.4734474718570709,
      "step": 790
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.616088947024199e-06,
      "logits/chosen": -2.2410073280334473,
      "logits/rejected": -2.4264373779296875,
      "logps/chosen": -227.9165496826172,
      "logps/rejected": -270.60357666015625,
      "loss": 0.6578,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.31427091360092163,
      "rewards/margins": 0.18160662055015564,
      "rewards/rejected": -0.4958775043487549,
      "step": 800
    },
    {
      "epoch": 0.05,
      "eval_logits/chosen": -2.4512248039245605,
      "eval_logits/rejected": -2.2566721439361572,
      "eval_logps/chosen": -273.8340759277344,
      "eval_logps/rejected": -281.5426940917969,
      "eval_loss": 0.6069934964179993,
      "eval_rewards/accuracies": 0.6050000190734863,
      "eval_rewards/chosen": -0.41829144954681396,
      "eval_rewards/margins": 0.2810164988040924,
      "eval_rewards/rejected": -0.6993079781532288,
      "eval_runtime": 732.0392,
      "eval_samples_per_second": 2.732,
      "eval_steps_per_second": 1.366,
      "step": 800
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6487900588620014e-06,
      "logits/chosen": -2.2319188117980957,
      "logits/rejected": -1.9784733057022095,
      "logps/chosen": -248.84750366210938,
      "logps/rejected": -223.2877960205078,
      "loss": 0.6512,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.4873189926147461,
      "rewards/margins": 0.16692177951335907,
      "rewards/rejected": -0.6542407870292664,
      "step": 810
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6814911706998042e-06,
      "logits/chosen": -2.381350040435791,
      "logits/rejected": -2.1919376850128174,
      "logps/chosen": -273.8369140625,
      "logps/rejected": -258.0627136230469,
      "loss": 0.6554,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.4849766194820404,
      "rewards/margins": 0.14979836344718933,
      "rewards/rejected": -0.6347750425338745,
      "step": 820
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.7141922825376067e-06,
      "logits/chosen": -2.2474639415740967,
      "logits/rejected": -2.2449734210968018,
      "logps/chosen": -300.6055908203125,
      "logps/rejected": -345.5708923339844,
      "loss": 0.5734,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.42681440711021423,
      "rewards/margins": 0.35034364461898804,
      "rewards/rejected": -0.7771580219268799,
      "step": 830
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.746893394375409e-06,
      "logits/chosen": -2.365583896636963,
      "logits/rejected": -2.2663304805755615,
      "logps/chosen": -271.7221984863281,
      "logps/rejected": -299.95050048828125,
      "loss": 0.5875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.37327784299850464,
      "rewards/margins": 0.30009618401527405,
      "rewards/rejected": -0.6733739972114563,
      "step": 840
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.779594506213211e-06,
      "logits/chosen": -2.282376766204834,
      "logits/rejected": -2.0392627716064453,
      "logps/chosen": -245.7705535888672,
      "logps/rejected": -261.686767578125,
      "loss": 0.6677,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.5591166615486145,
      "rewards/margins": 0.1622948795557022,
      "rewards/rejected": -0.7214115262031555,
      "step": 850
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.812295618051014e-06,
      "logits/chosen": -2.442470073699951,
      "logits/rejected": -2.2847018241882324,
      "logps/chosen": -313.7275695800781,
      "logps/rejected": -283.5700378417969,
      "loss": 0.5871,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.32766368985176086,
      "rewards/margins": 0.31569644808769226,
      "rewards/rejected": -0.6433601975440979,
      "step": 860
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.8449967298888164e-06,
      "logits/chosen": -2.3573145866394043,
      "logits/rejected": -2.129387855529785,
      "logps/chosen": -213.77999877929688,
      "logps/rejected": -211.1344757080078,
      "loss": 0.597,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3695598244667053,
      "rewards/margins": 0.2541285753250122,
      "rewards/rejected": -0.6236883401870728,
      "step": 870
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.877697841726619e-06,
      "logits/chosen": -2.4069507122039795,
      "logits/rejected": -2.30299711227417,
      "logps/chosen": -253.3367462158203,
      "logps/rejected": -255.14401245117188,
      "loss": 0.622,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3450094759464264,
      "rewards/margins": 0.1862207055091858,
      "rewards/rejected": -0.531230092048645,
      "step": 880
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.9103989535644217e-06,
      "logits/chosen": -2.2997517585754395,
      "logits/rejected": -2.4225234985351562,
      "logps/chosen": -243.65475463867188,
      "logps/rejected": -294.9205017089844,
      "loss": 0.6317,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.28171128034591675,
      "rewards/margins": 0.20735308527946472,
      "rewards/rejected": -0.48906439542770386,
      "step": 890
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.943100065402224e-06,
      "logits/chosen": -2.344895362854004,
      "logits/rejected": -2.197391986846924,
      "logps/chosen": -329.97265625,
      "logps/rejected": -335.56512451171875,
      "loss": 0.6272,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3732944428920746,
      "rewards/margins": 0.20909173786640167,
      "rewards/rejected": -0.5823861956596375,
      "step": 900
    },
    {
      "epoch": 0.06,
      "eval_logits/chosen": -2.318371534347534,
      "eval_logits/rejected": -2.133222818374634,
      "eval_logps/chosen": -259.9873962402344,
      "eval_logps/rejected": -263.5772399902344,
      "eval_loss": 0.6149184107780457,
      "eval_rewards/accuracies": 0.6060000061988831,
      "eval_rewards/chosen": -0.27982431650161743,
      "eval_rewards/margins": 0.23982904851436615,
      "eval_rewards/rejected": -0.5196533799171448,
      "eval_runtime": 732.3751,
      "eval_samples_per_second": 2.731,
      "eval_steps_per_second": 1.365,
      "step": 900
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.9758011772400266e-06,
      "logits/chosen": -2.2377166748046875,
      "logits/rejected": -2.2609286308288574,
      "logps/chosen": -286.5142517089844,
      "logps/rejected": -315.3929443359375,
      "loss": 0.6139,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.29205089807510376,
      "rewards/margins": 0.21504080295562744,
      "rewards/rejected": -0.5070916414260864,
      "step": 910
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.0085022890778286e-06,
      "logits/chosen": -2.2762789726257324,
      "logits/rejected": -2.0410525798797607,
      "logps/chosen": -217.5778350830078,
      "logps/rejected": -217.69351196289062,
      "loss": 0.6061,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.41838282346725464,
      "rewards/margins": 0.27039122581481934,
      "rewards/rejected": -0.6887739896774292,
      "step": 920
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.0412034009156314e-06,
      "logits/chosen": -2.2033088207244873,
      "logits/rejected": -2.3684961795806885,
      "logps/chosen": -298.0721740722656,
      "logps/rejected": -318.66436767578125,
      "loss": 0.6334,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.6238594055175781,
      "rewards/margins": 0.2485378533601761,
      "rewards/rejected": -0.872397243976593,
      "step": 930
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.073904512753434e-06,
      "logits/chosen": -2.3155341148376465,
      "logits/rejected": -1.962912917137146,
      "logps/chosen": -297.7289123535156,
      "logps/rejected": -312.3240661621094,
      "loss": 0.5093,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.680803120136261,
      "rewards/margins": 0.49523410201072693,
      "rewards/rejected": -1.1760371923446655,
      "step": 940
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1066056245912363e-06,
      "logits/chosen": -2.161400556564331,
      "logits/rejected": -2.205284595489502,
      "logps/chosen": -340.3791809082031,
      "logps/rejected": -353.7043762207031,
      "loss": 0.5633,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.0939515829086304,
      "rewards/margins": 0.4528425335884094,
      "rewards/rejected": -1.546794056892395,
      "step": 950
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1393067364290387e-06,
      "logits/chosen": -2.1303658485412598,
      "logits/rejected": -1.9281947612762451,
      "logps/chosen": -333.31866455078125,
      "logps/rejected": -334.27972412109375,
      "loss": 0.6545,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.0819190740585327,
      "rewards/margins": 0.39546647667884827,
      "rewards/rejected": -1.4773855209350586,
      "step": 960
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1720078482668416e-06,
      "logits/chosen": -2.1484124660491943,
      "logits/rejected": -1.9389692544937134,
      "logps/chosen": -317.99359130859375,
      "logps/rejected": -306.39434814453125,
      "loss": 0.5974,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.0642293691635132,
      "rewards/margins": 0.35507652163505554,
      "rewards/rejected": -1.4193060398101807,
      "step": 970
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.204708960104644e-06,
      "logits/chosen": -2.2778122425079346,
      "logits/rejected": -1.9579265117645264,
      "logps/chosen": -306.3919372558594,
      "logps/rejected": -315.81353759765625,
      "loss": 0.5629,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8898921012878418,
      "rewards/margins": 0.4200926423072815,
      "rewards/rejected": -1.3099846839904785,
      "step": 980
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.237410071942446e-06,
      "logits/chosen": -2.2833445072174072,
      "logits/rejected": -2.02951979637146,
      "logps/chosen": -295.0680236816406,
      "logps/rejected": -271.5545349121094,
      "loss": 0.6827,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8870692253112793,
      "rewards/margins": 0.10050360858440399,
      "rewards/rejected": -0.9875728487968445,
      "step": 990
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.270111183780249e-06,
      "logits/chosen": -2.214215040206909,
      "logits/rejected": -2.0094611644744873,
      "logps/chosen": -289.36053466796875,
      "logps/rejected": -294.6500549316406,
      "loss": 0.6772,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.8933695554733276,
      "rewards/margins": 0.2329256534576416,
      "rewards/rejected": -1.1262950897216797,
      "step": 1000
    },
    {
      "epoch": 0.07,
      "eval_logits/chosen": -2.273111343383789,
      "eval_logits/rejected": -2.0915205478668213,
      "eval_logps/chosen": -291.50830078125,
      "eval_logps/rejected": -301.5699157714844,
      "eval_loss": 0.5978599786758423,
      "eval_rewards/accuracies": 0.612500011920929,
      "eval_rewards/chosen": -0.5950338244438171,
      "eval_rewards/margins": 0.30454614758491516,
      "eval_rewards/rejected": -0.8995799422264099,
      "eval_runtime": 730.9914,
      "eval_samples_per_second": 2.736,
      "eval_steps_per_second": 1.368,
      "step": 1000
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.3028122956180513e-06,
      "logits/chosen": -2.149437427520752,
      "logits/rejected": -2.1762747764587402,
      "logps/chosen": -259.1993103027344,
      "logps/rejected": -311.66009521484375,
      "loss": 0.6148,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.47101396322250366,
      "rewards/margins": 0.24991849064826965,
      "rewards/rejected": -0.7209324836730957,
      "step": 1010
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.3355134074558538e-06,
      "logits/chosen": -2.1731173992156982,
      "logits/rejected": -2.0621697902679443,
      "logps/chosen": -268.55120849609375,
      "logps/rejected": -267.37139892578125,
      "loss": 0.6092,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4781845211982727,
      "rewards/margins": 0.24407725036144257,
      "rewards/rejected": -0.7222617268562317,
      "step": 1020
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.368214519293656e-06,
      "logits/chosen": -2.1015899181365967,
      "logits/rejected": -1.9048049449920654,
      "logps/chosen": -242.2707061767578,
      "logps/rejected": -249.1923828125,
      "loss": 0.589,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.43300414085388184,
      "rewards/margins": 0.27820152044296265,
      "rewards/rejected": -0.7112056612968445,
      "step": 1030
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.400915631131459e-06,
      "logits/chosen": -2.0095393657684326,
      "logits/rejected": -2.0408594608306885,
      "logps/chosen": -254.6264190673828,
      "logps/rejected": -327.074951171875,
      "loss": 0.5662,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5665396451950073,
      "rewards/margins": 0.43797236680984497,
      "rewards/rejected": -1.0045119524002075,
      "step": 1040
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.4336167429692615e-06,
      "logits/chosen": -2.2427849769592285,
      "logits/rejected": -2.028717517852783,
      "logps/chosen": -272.2727966308594,
      "logps/rejected": -288.40185546875,
      "loss": 0.589,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5772241353988647,
      "rewards/margins": 0.32382819056510925,
      "rewards/rejected": -0.9010523557662964,
      "step": 1050
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.4663178548070635e-06,
      "logits/chosen": -2.0555243492126465,
      "logits/rejected": -2.1193196773529053,
      "logps/chosen": -274.7303771972656,
      "logps/rejected": -302.765869140625,
      "loss": 0.5504,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5977641344070435,
      "rewards/margins": 0.44477134943008423,
      "rewards/rejected": -1.0425355434417725,
      "step": 1060
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.499018966644866e-06,
      "logits/chosen": -2.1316325664520264,
      "logits/rejected": -1.962110161781311,
      "logps/chosen": -294.1905517578125,
      "logps/rejected": -319.0716552734375,
      "loss": 0.618,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0236551761627197,
      "rewards/margins": 0.3169350028038025,
      "rewards/rejected": -1.3405901193618774,
      "step": 1070
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.531720078482669e-06,
      "logits/chosen": -2.026108980178833,
      "logits/rejected": -1.7189273834228516,
      "logps/chosen": -338.9638671875,
      "logps/rejected": -356.0903015136719,
      "loss": 0.6328,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.130941390991211,
      "rewards/margins": 0.2835771441459656,
      "rewards/rejected": -1.4145182371139526,
      "step": 1080
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.5644211903204712e-06,
      "logits/chosen": -2.0485777854919434,
      "logits/rejected": -2.090273380279541,
      "logps/chosen": -281.7182312011719,
      "logps/rejected": -323.7969665527344,
      "loss": 0.5626,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8283576965332031,
      "rewards/margins": 0.39265236258506775,
      "rewards/rejected": -1.2210099697113037,
      "step": 1090
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.5971223021582737e-06,
      "logits/chosen": -2.1952884197235107,
      "logits/rejected": -2.0548617839813232,
      "logps/chosen": -351.4318542480469,
      "logps/rejected": -323.7904968261719,
      "loss": 0.629,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.9563105702400208,
      "rewards/margins": 0.2947392463684082,
      "rewards/rejected": -1.2510497570037842,
      "step": 1100
    },
    {
      "epoch": 0.07,
      "eval_logits/chosen": -2.064201831817627,
      "eval_logits/rejected": -1.8958628177642822,
      "eval_logps/chosen": -348.63909912109375,
      "eval_logps/rejected": -370.0741882324219,
      "eval_loss": 0.5842385292053223,
      "eval_rewards/accuracies": 0.6255000233650208,
      "eval_rewards/chosen": -1.1663414239883423,
      "eval_rewards/margins": 0.4182808995246887,
      "eval_rewards/rejected": -1.5846223831176758,
      "eval_runtime": 731.6823,
      "eval_samples_per_second": 2.733,
      "eval_steps_per_second": 1.367,
      "step": 1100
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.6298234139960765e-06,
      "logits/chosen": -2.1259961128234863,
      "logits/rejected": -1.803380012512207,
      "logps/chosen": -338.1694641113281,
      "logps/rejected": -317.0995788574219,
      "loss": 0.6346,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.2040038108825684,
      "rewards/margins": 0.28601497411727905,
      "rewards/rejected": -1.4900187253952026,
      "step": 1110
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.6625245258338785e-06,
      "logits/chosen": -1.8661701679229736,
      "logits/rejected": -1.800474762916565,
      "logps/chosen": -327.68890380859375,
      "logps/rejected": -447.7711486816406,
      "loss": 0.5602,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.889104962348938,
      "rewards/margins": 0.43621620535850525,
      "rewards/rejected": -1.3253211975097656,
      "step": 1120
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.695225637671681e-06,
      "logits/chosen": -2.0691475868225098,
      "logits/rejected": -1.8956810235977173,
      "logps/chosen": -290.79425048828125,
      "logps/rejected": -287.4905700683594,
      "loss": 0.5978,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8351759910583496,
      "rewards/margins": 0.2977235019207001,
      "rewards/rejected": -1.132899522781372,
      "step": 1130
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.7279267495094834e-06,
      "logits/chosen": -2.0039424896240234,
      "logits/rejected": -1.8878767490386963,
      "logps/chosen": -243.4080810546875,
      "logps/rejected": -295.2784118652344,
      "loss": 0.552,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.829811692237854,
      "rewards/margins": 0.4174509048461914,
      "rewards/rejected": -1.2472625970840454,
      "step": 1140
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.7606278613472863e-06,
      "logits/chosen": -2.032029628753662,
      "logits/rejected": -1.7188432216644287,
      "logps/chosen": -389.14288330078125,
      "logps/rejected": -346.46759033203125,
      "loss": 0.6473,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0687159299850464,
      "rewards/margins": 0.3489280641078949,
      "rewards/rejected": -1.4176441431045532,
      "step": 1150
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.7933289731850887e-06,
      "logits/chosen": -1.8771930932998657,
      "logits/rejected": -1.678802490234375,
      "logps/chosen": -305.0873718261719,
      "logps/rejected": -325.5848693847656,
      "loss": 0.5391,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.7652674913406372,
      "rewards/margins": 0.48755812644958496,
      "rewards/rejected": -1.2528256177902222,
      "step": 1160
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.826030085022891e-06,
      "logits/chosen": -2.0287744998931885,
      "logits/rejected": -1.8205569982528687,
      "logps/chosen": -321.78076171875,
      "logps/rejected": -327.90667724609375,
      "loss": 0.5623,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.7413099408149719,
      "rewards/margins": 0.42277926206588745,
      "rewards/rejected": -1.1640890836715698,
      "step": 1170
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.858731196860693e-06,
      "logits/chosen": -2.0594873428344727,
      "logits/rejected": -1.65718674659729,
      "logps/chosen": -323.6650695800781,
      "logps/rejected": -304.764892578125,
      "loss": 0.6431,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.7283161878585815,
      "rewards/margins": 0.23335444927215576,
      "rewards/rejected": -0.9616705179214478,
      "step": 1180
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.891432308698496e-06,
      "logits/chosen": -1.7721526622772217,
      "logits/rejected": -1.7358468770980835,
      "logps/chosen": -270.286865234375,
      "logps/rejected": -338.5904235839844,
      "loss": 0.5986,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.7505649328231812,
      "rewards/margins": 0.4305826723575592,
      "rewards/rejected": -1.1811474561691284,
      "step": 1190
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.924133420536299e-06,
      "logits/chosen": -1.846959114074707,
      "logits/rejected": -1.6761360168457031,
      "logps/chosen": -326.3203125,
      "logps/rejected": -327.2926940917969,
      "loss": 0.6763,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4417558908462524,
      "rewards/margins": 0.3437504768371582,
      "rewards/rejected": -1.785506248474121,
      "step": 1200
    },
    {
      "epoch": 0.08,
      "eval_logits/chosen": -1.9452943801879883,
      "eval_logits/rejected": -1.778209924697876,
      "eval_logps/chosen": -354.6231384277344,
      "eval_logps/rejected": -379.3278503417969,
      "eval_loss": 0.5800026059150696,
      "eval_rewards/accuracies": 0.625,
      "eval_rewards/chosen": -1.2261821031570435,
      "eval_rewards/margins": 0.45097729563713074,
      "eval_rewards/rejected": -1.6771594285964966,
      "eval_runtime": 732.3829,
      "eval_samples_per_second": 2.731,
      "eval_steps_per_second": 1.365,
      "step": 1200
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.956834532374101e-06,
      "logits/chosen": -2.0891950130462646,
      "logits/rejected": -1.746002197265625,
      "logps/chosen": -308.41717529296875,
      "logps/rejected": -328.99591064453125,
      "loss": 0.4712,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.0315450429916382,
      "rewards/margins": 0.6453005075454712,
      "rewards/rejected": -1.6768455505371094,
      "step": 1210
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.989535644211904e-06,
      "logits/chosen": -1.9306423664093018,
      "logits/rejected": -1.703242301940918,
      "logps/chosen": -316.7760009765625,
      "logps/rejected": -343.18414306640625,
      "loss": 0.5702,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9964868426322937,
      "rewards/margins": 0.4902920722961426,
      "rewards/rejected": -1.486778974533081,
      "step": 1220
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.022236756049706e-06,
      "logits/chosen": -2.1357779502868652,
      "logits/rejected": -1.7851722240447998,
      "logps/chosen": -356.35040283203125,
      "logps/rejected": -368.52886962890625,
      "loss": 0.5604,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7921323776245117,
      "rewards/margins": 0.44653981924057007,
      "rewards/rejected": -1.238672137260437,
      "step": 1230
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.054937867887509e-06,
      "logits/chosen": -1.9934505224227905,
      "logits/rejected": -1.5923570394515991,
      "logps/chosen": -299.227783203125,
      "logps/rejected": -345.69647216796875,
      "loss": 0.5117,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7496100664138794,
      "rewards/margins": 0.57257080078125,
      "rewards/rejected": -1.3221808671951294,
      "step": 1240
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.087638979725311e-06,
      "logits/chosen": -1.8767526149749756,
      "logits/rejected": -1.9476865530014038,
      "logps/chosen": -310.9009704589844,
      "logps/rejected": -339.83123779296875,
      "loss": 0.6662,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.830611526966095,
      "rewards/margins": 0.27475300431251526,
      "rewards/rejected": -1.1053645610809326,
      "step": 1250
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.1203400915631135e-06,
      "logits/chosen": -1.716407060623169,
      "logits/rejected": -1.6570746898651123,
      "logps/chosen": -339.90679931640625,
      "logps/rejected": -354.4397277832031,
      "loss": 0.5875,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.0257585048675537,
      "rewards/margins": 0.42813658714294434,
      "rewards/rejected": -1.453895092010498,
      "step": 1260
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.153041203400916e-06,
      "logits/chosen": -1.982994794845581,
      "logits/rejected": -1.8656692504882812,
      "logps/chosen": -388.2208251953125,
      "logps/rejected": -401.22601318359375,
      "loss": 0.6485,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2379850149154663,
      "rewards/margins": 0.37639838457107544,
      "rewards/rejected": -1.614383339881897,
      "step": 1270
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.185742315238718e-06,
      "logits/chosen": -1.8478248119354248,
      "logits/rejected": -1.6495792865753174,
      "logps/chosen": -339.2015075683594,
      "logps/rejected": -353.11297607421875,
      "loss": 0.6318,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4109991788864136,
      "rewards/margins": 0.25778403878211975,
      "rewards/rejected": -1.668783187866211,
      "step": 1280
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.218443427076521e-06,
      "logits/chosen": -1.9099308252334595,
      "logits/rejected": -1.7717653512954712,
      "logps/chosen": -330.18585205078125,
      "logps/rejected": -346.7439880371094,
      "loss": 0.6937,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.4742728471755981,
      "rewards/margins": 0.15087108314037323,
      "rewards/rejected": -1.6251437664031982,
      "step": 1290
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.251144538914323e-06,
      "logits/chosen": -1.8934119939804077,
      "logits/rejected": -1.759953498840332,
      "logps/chosen": -387.2998962402344,
      "logps/rejected": -393.1944274902344,
      "loss": 0.6468,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.6670963764190674,
      "rewards/margins": 0.19090215861797333,
      "rewards/rejected": -1.8579986095428467,
      "step": 1300
    },
    {
      "epoch": 0.09,
      "eval_logits/chosen": -2.003377676010132,
      "eval_logits/rejected": -1.8354687690734863,
      "eval_logps/chosen": -375.2355651855469,
      "eval_logps/rejected": -384.9623718261719,
      "eval_loss": 0.5959275960922241,
      "eval_rewards/accuracies": 0.6265000104904175,
      "eval_rewards/chosen": -1.4323062896728516,
      "eval_rewards/margins": 0.3011987507343292,
      "eval_rewards/rejected": -1.7335046529769897,
      "eval_runtime": 733.6396,
      "eval_samples_per_second": 2.726,
      "eval_steps_per_second": 1.363,
      "step": 1300
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.283845650752126e-06,
      "logits/chosen": -2.0106637477874756,
      "logits/rejected": -1.8190252780914307,
      "logps/chosen": -429.07745361328125,
      "logps/rejected": -397.7805480957031,
      "loss": 0.5875,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.428654670715332,
      "rewards/margins": 0.3289453685283661,
      "rewards/rejected": -1.7576000690460205,
      "step": 1310
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.316546762589928e-06,
      "logits/chosen": -1.9939743280410767,
      "logits/rejected": -1.9128013849258423,
      "logps/chosen": -340.85955810546875,
      "logps/rejected": -335.62188720703125,
      "loss": 0.5983,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.3293203115463257,
      "rewards/margins": 0.29060477018356323,
      "rewards/rejected": -1.6199251413345337,
      "step": 1320
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.349247874427731e-06,
      "logits/chosen": -1.9885427951812744,
      "logits/rejected": -1.6930125951766968,
      "logps/chosen": -343.95068359375,
      "logps/rejected": -436.5520935058594,
      "loss": 0.5516,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.1741664409637451,
      "rewards/margins": 0.5251883268356323,
      "rewards/rejected": -1.6993547677993774,
      "step": 1330
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.381948986265534e-06,
      "logits/chosen": -2.123256206512451,
      "logits/rejected": -1.8905670642852783,
      "logps/chosen": -385.8031921386719,
      "logps/rejected": -422.24237060546875,
      "loss": 0.5249,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.2295849323272705,
      "rewards/margins": 0.49446210265159607,
      "rewards/rejected": -1.724047303199768,
      "step": 1340
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.414650098103336e-06,
      "logits/chosen": -2.1973934173583984,
      "logits/rejected": -1.9671694040298462,
      "logps/chosen": -410.22314453125,
      "logps/rejected": -429.04931640625,
      "loss": 0.593,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.482706069946289,
      "rewards/margins": 0.4125031530857086,
      "rewards/rejected": -1.8952090740203857,
      "step": 1350
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.447351209941138e-06,
      "logits/chosen": -1.8963654041290283,
      "logits/rejected": -1.6682153940200806,
      "logps/chosen": -404.86920166015625,
      "logps/rejected": -426.9701232910156,
      "loss": 0.5797,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9520457983016968,
      "rewards/margins": 0.40242427587509155,
      "rewards/rejected": -2.3544700145721436,
      "step": 1360
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.480052321778941e-06,
      "logits/chosen": -1.9020938873291016,
      "logits/rejected": -1.937169075012207,
      "logps/chosen": -372.9541320800781,
      "logps/rejected": -379.32916259765625,
      "loss": 0.6732,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.4604378938674927,
      "rewards/margins": 0.27071020007133484,
      "rewards/rejected": -1.73114812374115,
      "step": 1370
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.5127534336167435e-06,
      "logits/chosen": -2.0027899742126465,
      "logits/rejected": -1.7514774799346924,
      "logps/chosen": -382.690673828125,
      "logps/rejected": -369.9425048828125,
      "loss": 0.6172,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.3551666736602783,
      "rewards/margins": 0.309119313955307,
      "rewards/rejected": -1.6642862558364868,
      "step": 1380
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.5454545454545455e-06,
      "logits/chosen": -1.9697223901748657,
      "logits/rejected": -1.9489448070526123,
      "logps/chosen": -309.4676818847656,
      "logps/rejected": -308.8170471191406,
      "loss": 0.6996,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.4000898599624634,
      "rewards/margins": 0.09558413177728653,
      "rewards/rejected": -1.4956738948822021,
      "step": 1390
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.578155657292348e-06,
      "logits/chosen": -2.0561442375183105,
      "logits/rejected": -2.0292887687683105,
      "logps/chosen": -351.74664306640625,
      "logps/rejected": -418.7784118652344,
      "loss": 0.5302,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9634095430374146,
      "rewards/margins": 0.5810994505882263,
      "rewards/rejected": -1.544508934020996,
      "step": 1400
    },
    {
      "epoch": 0.09,
      "eval_logits/chosen": -2.039621353149414,
      "eval_logits/rejected": -1.8706350326538086,
      "eval_logps/chosen": -334.2268371582031,
      "eval_logps/rejected": -353.9125671386719,
      "eval_loss": 0.5790063738822937,
      "eval_rewards/accuracies": 0.6370000243186951,
      "eval_rewards/chosen": -1.0222188234329224,
      "eval_rewards/margins": 0.4007880985736847,
      "eval_rewards/rejected": -1.4230070114135742,
      "eval_runtime": 733.4141,
      "eval_samples_per_second": 2.727,
      "eval_steps_per_second": 1.363,
      "step": 1400
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.610856769130151e-06,
      "logits/chosen": -2.1047465801239014,
      "logits/rejected": -1.9882681369781494,
      "logps/chosen": -331.709228515625,
      "logps/rejected": -346.4400939941406,
      "loss": 0.5547,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9256499409675598,
      "rewards/margins": 0.44232434034347534,
      "rewards/rejected": -1.3679742813110352,
      "step": 1410
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.643557880967953e-06,
      "logits/chosen": -2.0810694694519043,
      "logits/rejected": -1.894683837890625,
      "logps/chosen": -282.4119873046875,
      "logps/rejected": -318.69873046875,
      "loss": 0.5953,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.8361316919326782,
      "rewards/margins": 0.38786545395851135,
      "rewards/rejected": -1.2239971160888672,
      "step": 1420
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.676258992805755e-06,
      "logits/chosen": -2.0333473682403564,
      "logits/rejected": -1.7796134948730469,
      "logps/chosen": -363.6483459472656,
      "logps/rejected": -386.46905517578125,
      "loss": 0.5182,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9402838945388794,
      "rewards/margins": 0.6496909260749817,
      "rewards/rejected": -1.5899746417999268,
      "step": 1430
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.708960104643558e-06,
      "logits/chosen": -1.9459807872772217,
      "logits/rejected": -1.925270438194275,
      "logps/chosen": -426.68353271484375,
      "logps/rejected": -433.5457458496094,
      "loss": 0.6221,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.2047381401062012,
      "rewards/margins": 0.4362231194972992,
      "rewards/rejected": -1.6409610509872437,
      "step": 1440
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.741661216481361e-06,
      "logits/chosen": -1.8569351434707642,
      "logits/rejected": -1.829811453819275,
      "logps/chosen": -342.13995361328125,
      "logps/rejected": -398.506103515625,
      "loss": 0.5758,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0579442977905273,
      "rewards/margins": 0.4511980414390564,
      "rewards/rejected": -1.5091423988342285,
      "step": 1450
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.774362328319163e-06,
      "logits/chosen": -1.8313878774642944,
      "logits/rejected": -1.699540376663208,
      "logps/chosen": -297.5150146484375,
      "logps/rejected": -293.62945556640625,
      "loss": 0.6199,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.8172494173049927,
      "rewards/margins": 0.26063865423202515,
      "rewards/rejected": -1.077888011932373,
      "step": 1460
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.807063440156966e-06,
      "logits/chosen": -1.8282153606414795,
      "logits/rejected": -1.6260473728179932,
      "logps/chosen": -334.1802673339844,
      "logps/rejected": -323.95416259765625,
      "loss": 0.5395,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.708707869052887,
      "rewards/margins": 0.5114428400993347,
      "rewards/rejected": -1.2201507091522217,
      "step": 1470
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.839764551994769e-06,
      "logits/chosen": -1.8896303176879883,
      "logits/rejected": -1.6281471252441406,
      "logps/chosen": -348.66851806640625,
      "logps/rejected": -361.35980224609375,
      "loss": 0.5617,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9996001124382019,
      "rewards/margins": 0.5042487382888794,
      "rewards/rejected": -1.503848910331726,
      "step": 1480
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.872465663832571e-06,
      "logits/chosen": -1.7976363897323608,
      "logits/rejected": -1.6468493938446045,
      "logps/chosen": -278.28057861328125,
      "logps/rejected": -313.6100769042969,
      "loss": 0.5226,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7052208185195923,
      "rewards/margins": 0.5578960180282593,
      "rewards/rejected": -1.2631168365478516,
      "step": 1490
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.905166775670373e-06,
      "logits/chosen": -1.8637592792510986,
      "logits/rejected": -1.6277945041656494,
      "logps/chosen": -289.73187255859375,
      "logps/rejected": -310.935546875,
      "loss": 0.5512,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.679719090461731,
      "rewards/margins": 0.48597225546836853,
      "rewards/rejected": -1.1656912565231323,
      "step": 1500
    },
    {
      "epoch": 0.1,
      "eval_logits/chosen": -1.829469919204712,
      "eval_logits/rejected": -1.6728575229644775,
      "eval_logps/chosen": -315.8946228027344,
      "eval_logps/rejected": -349.49725341796875,
      "eval_loss": 0.5626789331436157,
      "eval_rewards/accuracies": 0.6370000243186951,
      "eval_rewards/chosen": -0.8388968706130981,
      "eval_rewards/margins": 0.5399565100669861,
      "eval_rewards/rejected": -1.3788533210754395,
      "eval_runtime": 736.2443,
      "eval_samples_per_second": 2.716,
      "eval_steps_per_second": 1.358,
      "step": 1500
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.9378678875081756e-06,
      "logits/chosen": -1.9396146535873413,
      "logits/rejected": -1.689638376235962,
      "logps/chosen": -305.39373779296875,
      "logps/rejected": -330.161865234375,
      "loss": 0.5266,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9065009951591492,
      "rewards/margins": 0.5577627420425415,
      "rewards/rejected": -1.464263677597046,
      "step": 1510
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.9705689993459784e-06,
      "logits/chosen": -1.8057773113250732,
      "logits/rejected": -1.4475606679916382,
      "logps/chosen": -376.16839599609375,
      "logps/rejected": -374.25018310546875,
      "loss": 0.5563,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7124303579330444,
      "rewards/margins": 0.5579492449760437,
      "rewards/rejected": -2.2703795433044434,
      "step": 1520
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999999934793849e-06,
      "logits/chosen": -1.771942377090454,
      "logits/rejected": -1.6722147464752197,
      "logps/chosen": -446.30889892578125,
      "logps/rejected": -480.4701232910156,
      "loss": 0.5871,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.983555793762207,
      "rewards/margins": 0.6357766389846802,
      "rewards/rejected": -2.6193320751190186,
      "step": 1530
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999992110059814e-06,
      "logits/chosen": -1.6630691289901733,
      "logits/rejected": -1.6542637348175049,
      "logps/chosen": -494.4562072753906,
      "logps/rejected": -510.5018615722656,
      "loss": 0.6301,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.167686939239502,
      "rewards/margins": 0.3615902066230774,
      "rewards/rejected": -2.5292773246765137,
      "step": 1540
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999971244142299e-06,
      "logits/chosen": -1.7780059576034546,
      "logits/rejected": -1.522735357284546,
      "logps/chosen": -485.382080078125,
      "logps/rejected": -506.91070556640625,
      "loss": 0.5534,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1192898750305176,
      "rewards/margins": 0.6077495217323303,
      "rewards/rejected": -2.727038860321045,
      "step": 1550
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999937337150149e-06,
      "logits/chosen": -1.4890727996826172,
      "logits/rejected": -1.411381721496582,
      "logps/chosen": -386.9769287109375,
      "logps/rejected": -438.4666442871094,
      "loss": 0.5325,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.5246206521987915,
      "rewards/margins": 0.6687048673629761,
      "rewards/rejected": -2.1933255195617676,
      "step": 1560
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.99989038926024e-06,
      "logits/chosen": -1.4228270053863525,
      "logits/rejected": -1.507315993309021,
      "logps/chosen": -337.02008056640625,
      "logps/rejected": -390.37542724609375,
      "loss": 0.5921,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.3859201669692993,
      "rewards/margins": 0.5115332007408142,
      "rewards/rejected": -1.8974533081054688,
      "step": 1570
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999830400717476e-06,
      "logits/chosen": -1.558600664138794,
      "logits/rejected": -1.4721577167510986,
      "logps/chosen": -425.5896911621094,
      "logps/rejected": -467.05419921875,
      "loss": 0.5434,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3820263147354126,
      "rewards/margins": 0.5773999094963074,
      "rewards/rejected": -1.9594262838363647,
      "step": 1580
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999757371834787e-06,
      "logits/chosen": -1.4094642400741577,
      "logits/rejected": -1.4305822849273682,
      "logps/chosen": -366.4520263671875,
      "logps/rejected": -418.969482421875,
      "loss": 0.5355,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.2663311958312988,
      "rewards/margins": 0.6803295016288757,
      "rewards/rejected": -1.9466607570648193,
      "step": 1590
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999671302993125e-06,
      "logits/chosen": -1.4092227220535278,
      "logits/rejected": -1.3802907466888428,
      "logps/chosen": -370.32659912109375,
      "logps/rejected": -419.1243591308594,
      "loss": 0.6386,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.2174619436264038,
      "rewards/margins": 0.3340964615345001,
      "rewards/rejected": -1.5515583753585815,
      "step": 1600
    },
    {
      "epoch": 0.1,
      "eval_logits/chosen": -1.6499887704849243,
      "eval_logits/rejected": -1.501013994216919,
      "eval_logps/chosen": -314.13006591796875,
      "eval_logps/rejected": -340.3790283203125,
      "eval_loss": 0.5758016705513,
      "eval_rewards/accuracies": 0.6244999766349792,
      "eval_rewards/chosen": -0.8212510943412781,
      "eval_rewards/margins": 0.4664204716682434,
      "eval_rewards/rejected": -1.2876713275909424,
      "eval_runtime": 731.6603,
      "eval_samples_per_second": 2.734,
      "eval_steps_per_second": 1.367,
      "step": 1600
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999572194641471e-06,
      "logits/chosen": -1.6551891565322876,
      "logits/rejected": -1.4926649332046509,
      "logps/chosen": -364.64666748046875,
      "logps/rejected": -372.8708801269531,
      "loss": 0.5305,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8912836313247681,
      "rewards/margins": 0.606941819190979,
      "rewards/rejected": -1.4982253313064575,
      "step": 1610
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999460047296819e-06,
      "logits/chosen": -1.593339204788208,
      "logits/rejected": -1.4939100742340088,
      "logps/chosen": -325.66314697265625,
      "logps/rejected": -356.7687683105469,
      "loss": 0.5458,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1005899906158447,
      "rewards/margins": 0.5201475620269775,
      "rewards/rejected": -1.6207374334335327,
      "step": 1620
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999334861544186e-06,
      "logits/chosen": -1.6162506341934204,
      "logits/rejected": -1.380138635635376,
      "logps/chosen": -368.8914489746094,
      "logps/rejected": -376.7461242675781,
      "loss": 0.548,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.3943818807601929,
      "rewards/margins": 0.6385797262191772,
      "rewards/rejected": -2.03296160697937,
      "step": 1630
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999196638036604e-06,
      "logits/chosen": -1.5795015096664429,
      "logits/rejected": -1.4061323404312134,
      "logps/chosen": -481.0608825683594,
      "logps/rejected": -469.82843017578125,
      "loss": 0.6616,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.9364579916000366,
      "rewards/margins": 0.24003927409648895,
      "rewards/rejected": -2.176497459411621,
      "step": 1640
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999045377495111e-06,
      "logits/chosen": -1.1616097688674927,
      "logits/rejected": -1.364842176437378,
      "logps/chosen": -354.5358581542969,
      "logps/rejected": -512.4725341796875,
      "loss": 0.5295,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8020122051239014,
      "rewards/margins": 0.701541006565094,
      "rewards/rejected": -2.5035529136657715,
      "step": 1650
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998881080708759e-06,
      "logits/chosen": -1.1293063163757324,
      "logits/rejected": -1.1073631048202515,
      "logps/chosen": -438.1863708496094,
      "logps/rejected": -431.01885986328125,
      "loss": 0.6636,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.9786484241485596,
      "rewards/margins": 0.23777297139167786,
      "rewards/rejected": -2.216421365737915,
      "step": 1660
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998703748534599e-06,
      "logits/chosen": -0.9836093187332153,
      "logits/rejected": -0.8401772379875183,
      "logps/chosen": -413.32708740234375,
      "logps/rejected": -433.08270263671875,
      "loss": 0.5362,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7269728183746338,
      "rewards/margins": 0.7044621109962463,
      "rewards/rejected": -2.4314348697662354,
      "step": 1670
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998513381897683e-06,
      "logits/chosen": -1.4278907775878906,
      "logits/rejected": -1.1960400342941284,
      "logps/chosen": -346.00494384765625,
      "logps/rejected": -325.62591552734375,
      "loss": 0.6458,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.0676120519638062,
      "rewards/margins": 0.3661525547504425,
      "rewards/rejected": -1.4337646961212158,
      "step": 1680
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.9983099817910565e-06,
      "logits/chosen": -1.4905847311019897,
      "logits/rejected": -1.3855290412902832,
      "logps/chosen": -337.5746154785156,
      "logps/rejected": -381.63043212890625,
      "loss": 0.5957,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9231271743774414,
      "rewards/margins": 0.4853379726409912,
      "rewards/rejected": -1.408465027809143,
      "step": 1690
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998093549275754e-06,
      "logits/chosen": -1.5985859632492065,
      "logits/rejected": -1.6034762859344482,
      "logps/chosen": -314.97607421875,
      "logps/rejected": -382.0792541503906,
      "loss": 0.5515,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5464205741882324,
      "rewards/margins": 0.4531020224094391,
      "rewards/rejected": -0.9995225667953491,
      "step": 1700
    },
    {
      "epoch": 0.11,
      "eval_logits/chosen": -1.6102489233016968,
      "eval_logits/rejected": -1.4650778770446777,
      "eval_logps/chosen": -293.721435546875,
      "eval_logps/rejected": -316.3880615234375,
      "eval_loss": 0.5788753628730774,
      "eval_rewards/accuracies": 0.6154999732971191,
      "eval_rewards/chosen": -0.6171649694442749,
      "eval_rewards/margins": 0.43059632182121277,
      "eval_rewards/rejected": -1.04776132106781,
      "eval_runtime": 732.9705,
      "eval_samples_per_second": 2.729,
      "eval_steps_per_second": 1.364,
      "step": 1700
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997864085480794e-06,
      "logits/chosen": -1.642925500869751,
      "logits/rejected": -1.489255666732788,
      "logps/chosen": -360.6557922363281,
      "logps/rejected": -397.36541748046875,
      "loss": 0.555,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9430961608886719,
      "rewards/margins": 0.5524653196334839,
      "rewards/rejected": -1.4955614805221558,
      "step": 1710
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997621591603171e-06,
      "logits/chosen": -1.5001728534698486,
      "logits/rejected": -1.3734763860702515,
      "logps/chosen": -254.2823486328125,
      "logps/rejected": -298.25885009765625,
      "loss": 0.6253,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.9624954462051392,
      "rewards/margins": 0.44661301374435425,
      "rewards/rejected": -1.4091084003448486,
      "step": 1720
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997366068907853e-06,
      "logits/chosen": -1.6290733814239502,
      "logits/rejected": -1.5759193897247314,
      "logps/chosen": -319.1598205566406,
      "logps/rejected": -339.3824768066406,
      "loss": 0.5875,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6075009107589722,
      "rewards/margins": 0.4009774327278137,
      "rewards/rejected": -1.0084784030914307,
      "step": 1730
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997097518727771e-06,
      "logits/chosen": -1.764134168624878,
      "logits/rejected": -1.5227423906326294,
      "logps/chosen": -287.41790771484375,
      "logps/rejected": -303.2652893066406,
      "loss": 0.5696,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.6446150541305542,
      "rewards/margins": 0.48969754576683044,
      "rewards/rejected": -1.134312391281128,
      "step": 1740
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.9968159424638155e-06,
      "logits/chosen": -1.5703229904174805,
      "logits/rejected": -1.6936891078948975,
      "logps/chosen": -315.3788757324219,
      "logps/rejected": -401.99713134765625,
      "loss": 0.6687,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9777250289916992,
      "rewards/margins": 0.3477313220500946,
      "rewards/rejected": -1.3254563808441162,
      "step": 1750
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9965213415848235e-06,
      "logits/chosen": -1.5106548070907593,
      "logits/rejected": -1.2070127725601196,
      "logps/chosen": -344.24603271484375,
      "logps/rejected": -365.4908447265625,
      "loss": 0.5699,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.1584149599075317,
      "rewards/margins": 0.5810474753379822,
      "rewards/rejected": -1.7394624948501587,
      "step": 1760
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9962137176275805e-06,
      "logits/chosen": -1.5711520910263062,
      "logits/rejected": -1.3876466751098633,
      "logps/chosen": -352.61053466796875,
      "logps/rejected": -397.92626953125,
      "loss": 0.5467,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1831510066986084,
      "rewards/margins": 0.458737313747406,
      "rewards/rejected": -1.6418883800506592,
      "step": 1770
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9958930721968015e-06,
      "logits/chosen": -1.467863917350769,
      "logits/rejected": -1.585448980331421,
      "logps/chosen": -384.6903076171875,
      "logps/rejected": -428.30322265625,
      "loss": 0.6408,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7265514135360718,
      "rewards/margins": 0.32111164927482605,
      "rewards/rejected": -2.0476629734039307,
      "step": 1780
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995559406965132e-06,
      "logits/chosen": -1.654897928237915,
      "logits/rejected": -1.418758749961853,
      "logps/chosen": -423.865478515625,
      "logps/rejected": -449.65704345703125,
      "loss": 0.6173,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9195009469985962,
      "rewards/margins": 0.5540726184844971,
      "rewards/rejected": -2.473573923110962,
      "step": 1790
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995212723673131e-06,
      "logits/chosen": -1.7198116779327393,
      "logits/rejected": -1.5079858303070068,
      "logps/chosen": -350.39337158203125,
      "logps/rejected": -338.1083068847656,
      "loss": 0.5693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2019023895263672,
      "rewards/margins": 0.37697863578796387,
      "rewards/rejected": -1.5788812637329102,
      "step": 1800
    },
    {
      "epoch": 0.12,
      "eval_logits/chosen": -1.7295727729797363,
      "eval_logits/rejected": -1.5710750818252563,
      "eval_logps/chosen": -323.40234375,
      "eval_logps/rejected": -346.465087890625,
      "eval_loss": 0.5636827349662781,
      "eval_rewards/accuracies": 0.6434999704360962,
      "eval_rewards/chosen": -0.9139736890792847,
      "eval_rewards/margins": 0.43455827236175537,
      "eval_rewards/rejected": -1.34853196144104,
      "eval_runtime": 732.2426,
      "eval_samples_per_second": 2.731,
      "eval_steps_per_second": 1.366,
      "step": 1800
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.99485302412927e-06,
      "logits/chosen": -1.3826578855514526,
      "logits/rejected": -1.382262110710144,
      "logps/chosen": -298.8088684082031,
      "logps/rejected": -364.51837158203125,
      "loss": 0.4857,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9079780578613281,
      "rewards/margins": 0.6501679420471191,
      "rewards/rejected": -1.5581459999084473,
      "step": 1810
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.994480310209918e-06,
      "logits/chosen": -1.5383714437484741,
      "logits/rejected": -1.622907280921936,
      "logps/chosen": -388.2715759277344,
      "logps/rejected": -460.01556396484375,
      "loss": 0.5395,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.462885856628418,
      "rewards/margins": 0.5798969864845276,
      "rewards/rejected": -2.042782783508301,
      "step": 1820
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.994094583859332e-06,
      "logits/chosen": -1.383913516998291,
      "logits/rejected": -1.2517662048339844,
      "logps/chosen": -339.15753173828125,
      "logps/rejected": -435.96826171875,
      "loss": 0.5427,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.742448091506958,
      "rewards/margins": 0.6478117108345032,
      "rewards/rejected": -2.3902597427368164,
      "step": 1830
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9936958470896525e-06,
      "logits/chosen": -1.315035104751587,
      "logits/rejected": -1.1094564199447632,
      "logps/chosen": -418.587158203125,
      "logps/rejected": -475.2015686035156,
      "loss": 0.42,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.9484198093414307,
      "rewards/margins": 0.9951755404472351,
      "rewards/rejected": -2.9435954093933105,
      "step": 1840
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.993284101980883e-06,
      "logits/chosen": -1.1157817840576172,
      "logits/rejected": -1.0088772773742676,
      "logps/chosen": -598.3966064453125,
      "logps/rejected": -658.9550170898438,
      "loss": 0.504,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -3.5590121746063232,
      "rewards/margins": 1.0019128322601318,
      "rewards/rejected": -4.560925483703613,
      "step": 1850
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9928593506808885e-06,
      "logits/chosen": -1.3301093578338623,
      "logits/rejected": -1.1362839937210083,
      "logps/chosen": -572.7133178710938,
      "logps/rejected": -599.54150390625,
      "loss": 0.6122,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -3.158989191055298,
      "rewards/margins": 0.6380549669265747,
      "rewards/rejected": -3.797044038772583,
      "step": 1860
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.992421595405381e-06,
      "logits/chosen": -1.4222590923309326,
      "logits/rejected": -1.1619681119918823,
      "logps/chosen": -456.4795837402344,
      "logps/rejected": -450.6554260253906,
      "loss": 0.6322,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.282871723175049,
      "rewards/margins": 0.650227427482605,
      "rewards/rejected": -2.9330992698669434,
      "step": 1870
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.991970838437905e-06,
      "logits/chosen": -1.4716317653656006,
      "logits/rejected": -1.406322956085205,
      "logps/chosen": -450.2798767089844,
      "logps/rejected": -538.4368896484375,
      "loss": 0.5038,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.2495033740997314,
      "rewards/margins": 0.6005233526229858,
      "rewards/rejected": -2.8500266075134277,
      "step": 1880
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9915070821298294e-06,
      "logits/chosen": -1.499276876449585,
      "logits/rejected": -1.278843879699707,
      "logps/chosen": -382.90435791015625,
      "logps/rejected": -406.1724548339844,
      "loss": 0.6374,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.1593451499938965,
      "rewards/margins": 0.32356566190719604,
      "rewards/rejected": -2.4829111099243164,
      "step": 1890
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.991030328900336e-06,
      "logits/chosen": -1.4514752626419067,
      "logits/rejected": -1.212705373764038,
      "logps/chosen": -459.39501953125,
      "logps/rejected": -476.75286865234375,
      "loss": 0.4312,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7482576370239258,
      "rewards/margins": 0.8739405870437622,
      "rewards/rejected": -2.6221985816955566,
      "step": 1900
    },
    {
      "epoch": 0.12,
      "eval_logits/chosen": -1.4911623001098633,
      "eval_logits/rejected": -1.3446063995361328,
      "eval_logps/chosen": -395.8936462402344,
      "eval_logps/rejected": -431.74383544921875,
      "eval_loss": 0.5713217854499817,
      "eval_rewards/accuracies": 0.6299999952316284,
      "eval_rewards/chosen": -1.6388871669769287,
      "eval_rewards/margins": 0.5624324679374695,
      "eval_rewards/rejected": -2.201319694519043,
      "eval_runtime": 732.3148,
      "eval_samples_per_second": 2.731,
      "eval_steps_per_second": 1.366,
      "step": 1900
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9905405812364014e-06,
      "logits/chosen": -1.4818050861358643,
      "logits/rejected": -1.4596812725067139,
      "logps/chosen": -368.3716125488281,
      "logps/rejected": -425.0055236816406,
      "loss": 0.551,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.660753607749939,
      "rewards/margins": 0.5416861772537231,
      "rewards/rejected": -2.202439785003662,
      "step": 1910
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990037841692791e-06,
      "logits/chosen": -1.4347656965255737,
      "logits/rejected": -1.2928135395050049,
      "logps/chosen": -370.4066467285156,
      "logps/rejected": -383.96466064453125,
      "loss": 0.5827,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.6436573266983032,
      "rewards/margins": 0.5684258341789246,
      "rewards/rejected": -2.212083339691162,
      "step": 1920
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.989522112892039e-06,
      "logits/chosen": -1.4713729619979858,
      "logits/rejected": -1.4406095743179321,
      "logps/chosen": -374.61846923828125,
      "logps/rejected": -415.1582946777344,
      "loss": 0.5761,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.769890546798706,
      "rewards/margins": 0.4142988622188568,
      "rewards/rejected": -2.184189558029175,
      "step": 1930
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.98899339752444e-06,
      "logits/chosen": -1.5092216730117798,
      "logits/rejected": -1.2862017154693604,
      "logps/chosen": -367.26861572265625,
      "logps/rejected": -412.562744140625,
      "loss": 0.5197,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3766629695892334,
      "rewards/margins": 0.74237459897995,
      "rewards/rejected": -2.119037628173828,
      "step": 1940
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.988451698348033e-06,
      "logits/chosen": -1.470228672027588,
      "logits/rejected": -1.5228326320648193,
      "logps/chosen": -328.7142333984375,
      "logps/rejected": -400.4761962890625,
      "loss": 0.5722,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.5038667917251587,
      "rewards/margins": 0.5149255990982056,
      "rewards/rejected": -2.0187923908233643,
      "step": 1950
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987897018188585e-06,
      "logits/chosen": -1.424580693244934,
      "logits/rejected": -1.2031456232070923,
      "logps/chosen": -408.1636047363281,
      "logps/rejected": -384.49310302734375,
      "loss": 0.7038,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.8554420471191406,
      "rewards/margins": 0.33005625009536743,
      "rewards/rejected": -2.1854987144470215,
      "step": 1960
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9873293599395814e-06,
      "logits/chosen": -1.628864049911499,
      "logits/rejected": -1.5045515298843384,
      "logps/chosen": -337.1258850097656,
      "logps/rejected": -382.0345458984375,
      "loss": 0.5581,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4369412660598755,
      "rewards/margins": 0.48892107605934143,
      "rewards/rejected": -1.9258620738983154,
      "step": 1970
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.986748726562203e-06,
      "logits/chosen": -1.6803842782974243,
      "logits/rejected": -1.5705522298812866,
      "logps/chosen": -337.2610778808594,
      "logps/rejected": -366.073486328125,
      "loss": 0.5317,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2140662670135498,
      "rewards/margins": 0.5370473861694336,
      "rewards/rejected": -1.7511136531829834,
      "step": 1980
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.98615512108532e-06,
      "logits/chosen": -1.6891971826553345,
      "logits/rejected": -1.6144077777862549,
      "logps/chosen": -342.71551513671875,
      "logps/rejected": -398.6066589355469,
      "loss": 0.5161,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.2120475769042969,
      "rewards/margins": 0.5410778522491455,
      "rewards/rejected": -1.753125548362732,
      "step": 1990
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.985548546605469e-06,
      "logits/chosen": -1.2585963010787964,
      "logits/rejected": -1.388717532157898,
      "logps/chosen": -449.9461975097656,
      "logps/rejected": -512.4083251953125,
      "loss": 0.6104,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.368403911590576,
      "rewards/margins": 0.4482155740261078,
      "rewards/rejected": -2.816619396209717,
      "step": 2000
    },
    {
      "epoch": 0.13,
      "eval_logits/chosen": -1.3215383291244507,
      "eval_logits/rejected": -1.1864279508590698,
      "eval_logps/chosen": -490.3330993652344,
      "eval_logps/rejected": -524.0951538085938,
      "eval_loss": 0.5691915154457092,
      "eval_rewards/accuracies": 0.6294999718666077,
      "eval_rewards/chosen": -2.5832810401916504,
      "eval_rewards/margins": 0.5415511131286621,
      "eval_rewards/rejected": -3.1248319149017334,
      "eval_runtime": 735.4501,
      "eval_samples_per_second": 2.719,
      "eval_steps_per_second": 1.36,
      "step": 2000
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.984929006286838e-06,
      "logits/chosen": -1.1429121494293213,
      "logits/rejected": -1.0560420751571655,
      "logps/chosen": -457.6143493652344,
      "logps/rejected": -486.137939453125,
      "loss": 0.7274,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.483003616333008,
      "rewards/margins": 0.250672847032547,
      "rewards/rejected": -2.7336764335632324,
      "step": 2010
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.984296503361256e-06,
      "logits/chosen": -1.2648851871490479,
      "logits/rejected": -1.080210566520691,
      "logps/chosen": -455.17901611328125,
      "logps/rejected": -455.192626953125,
      "loss": 0.5704,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.5325915813446045,
      "rewards/margins": 0.4201744496822357,
      "rewards/rejected": -2.952765941619873,
      "step": 2020
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9836510411281645e-06,
      "logits/chosen": -1.1588724851608276,
      "logits/rejected": -1.0907584428787231,
      "logps/chosen": -537.9210815429688,
      "logps/rejected": -580.5594482421875,
      "loss": 0.4492,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.593892812728882,
      "rewards/margins": 0.8204075694084167,
      "rewards/rejected": -3.4143004417419434,
      "step": 2030
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.982992622954613e-06,
      "logits/chosen": -1.459470510482788,
      "logits/rejected": -1.2297568321228027,
      "logps/chosen": -494.07342529296875,
      "logps/rejected": -436.62872314453125,
      "loss": 0.5452,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1054527759552,
      "rewards/margins": 0.5672376155853271,
      "rewards/rejected": -2.6726901531219482,
      "step": 2040
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9823212522752325e-06,
      "logits/chosen": -1.6404403448104858,
      "logits/rejected": -1.4471931457519531,
      "logps/chosen": -450.13055419921875,
      "logps/rejected": -485.72357177734375,
      "loss": 0.5531,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.723168134689331,
      "rewards/margins": 0.696290135383606,
      "rewards/rejected": -2.4194586277008057,
      "step": 2050
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.981636932592222e-06,
      "logits/chosen": -1.469670057296753,
      "logits/rejected": -1.3739349842071533,
      "logps/chosen": -332.8818054199219,
      "logps/rejected": -400.9057312011719,
      "loss": 0.5048,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.1996879577636719,
      "rewards/margins": 0.720448911190033,
      "rewards/rejected": -1.92013680934906,
      "step": 2060
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.980939667475328e-06,
      "logits/chosen": -1.6971523761749268,
      "logits/rejected": -1.40958833694458,
      "logps/chosen": -414.5674743652344,
      "logps/rejected": -404.3069152832031,
      "loss": 0.5837,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.403923511505127,
      "rewards/margins": 0.5033141374588013,
      "rewards/rejected": -1.9072374105453491,
      "step": 2070
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.980229460561826e-06,
      "logits/chosen": -1.5918484926223755,
      "logits/rejected": -1.5012832880020142,
      "logps/chosen": -341.13470458984375,
      "logps/rejected": -422.94451904296875,
      "loss": 0.4334,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.230533480644226,
      "rewards/margins": 0.9711629152297974,
      "rewards/rejected": -2.2016961574554443,
      "step": 2080
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.979506315556503e-06,
      "logits/chosen": -1.5313241481781006,
      "logits/rejected": -1.2765053510665894,
      "logps/chosen": -427.59906005859375,
      "logps/rejected": -468.064208984375,
      "loss": 0.4989,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4110695123672485,
      "rewards/margins": 0.9145315885543823,
      "rewards/rejected": -2.325601100921631,
      "step": 2090
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9787702362316395e-06,
      "logits/chosen": -1.6753005981445312,
      "logits/rejected": -1.8308302164077759,
      "logps/chosen": -316.5,
      "logps/rejected": -395.70782470703125,
      "loss": 0.589,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.2567278146743774,
      "rewards/margins": 0.5369380116462708,
      "rewards/rejected": -1.7936656475067139,
      "step": 2100
    },
    {
      "epoch": 0.14,
      "eval_logits/chosen": -1.6258045434951782,
      "eval_logits/rejected": -1.468246340751648,
      "eval_logps/chosen": -352.625732421875,
      "eval_logps/rejected": -400.03143310546875,
      "eval_loss": 0.554843008518219,
      "eval_rewards/accuracies": 0.6355000138282776,
      "eval_rewards/chosen": -1.2062077522277832,
      "eval_rewards/margins": 0.6779874563217163,
      "eval_rewards/rejected": -1.88419508934021,
      "eval_runtime": 735.5271,
      "eval_samples_per_second": 2.719,
      "eval_steps_per_second": 1.36,
      "step": 2100
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9780212264269835e-06,
      "logits/chosen": -1.4886839389801025,
      "logits/rejected": -1.3000301122665405,
      "logps/chosen": -330.2353210449219,
      "logps/rejected": -356.14984130859375,
      "loss": 0.6792,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.45416259765625,
      "rewards/margins": 0.43050187826156616,
      "rewards/rejected": -1.88466477394104,
      "step": 2110
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.977259290049739e-06,
      "logits/chosen": -1.6070921421051025,
      "logits/rejected": -1.1900372505187988,
      "logps/chosen": -418.1002502441406,
      "logps/rejected": -486.99951171875,
      "loss": 0.3545,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.3115779161453247,
      "rewards/margins": 1.3841772079467773,
      "rewards/rejected": -2.6957552433013916,
      "step": 2120
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.976484431074538e-06,
      "logits/chosen": -1.1471256017684937,
      "logits/rejected": -1.1589750051498413,
      "logps/chosen": -361.8846130371094,
      "logps/rejected": -428.1923828125,
      "loss": 0.5364,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6807912588119507,
      "rewards/margins": 0.8391014337539673,
      "rewards/rejected": -2.5198922157287598,
      "step": 2130
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.975696653543425e-06,
      "logits/chosen": -1.278565764427185,
      "logits/rejected": -1.1102664470672607,
      "logps/chosen": -451.95819091796875,
      "logps/rejected": -545.0523071289062,
      "loss": 0.4936,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0230939388275146,
      "rewards/margins": 1.0566326379776,
      "rewards/rejected": -3.079726457595825,
      "step": 2140
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.974895961565835e-06,
      "logits/chosen": -1.0780308246612549,
      "logits/rejected": -0.8999558687210083,
      "logps/chosen": -351.4781188964844,
      "logps/rejected": -439.71697998046875,
      "loss": 0.5233,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7622718811035156,
      "rewards/margins": 0.8301091194152832,
      "rewards/rejected": -2.592381000518799,
      "step": 2150
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.974082359318566e-06,
      "logits/chosen": -1.0910061597824097,
      "logits/rejected": -1.0057199001312256,
      "logps/chosen": -397.6961669921875,
      "logps/rejected": -429.198974609375,
      "loss": 0.4968,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4404613971710205,
      "rewards/margins": 0.7698196172714233,
      "rewards/rejected": -2.2102808952331543,
      "step": 2160
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.973255851045769e-06,
      "logits/chosen": -1.1554844379425049,
      "logits/rejected": -1.1484893560409546,
      "logps/chosen": -364.1314392089844,
      "logps/rejected": -411.07427978515625,
      "loss": 0.5358,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4677320718765259,
      "rewards/margins": 0.83690345287323,
      "rewards/rejected": -2.3046352863311768,
      "step": 2170
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.972416441058915e-06,
      "logits/chosen": -0.9132987260818481,
      "logits/rejected": -0.8012576103210449,
      "logps/chosen": -434.9048767089844,
      "logps/rejected": -511.2308044433594,
      "loss": 0.4525,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.0514910221099854,
      "rewards/margins": 1.0296775102615356,
      "rewards/rejected": -3.0811686515808105,
      "step": 2180
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.971564133736777e-06,
      "logits/chosen": -0.8563289642333984,
      "logits/rejected": -0.6923235654830933,
      "logps/chosen": -347.26776123046875,
      "logps/rejected": -463.20440673828125,
      "loss": 0.4265,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6926708221435547,
      "rewards/margins": 1.1095556020736694,
      "rewards/rejected": -2.8022265434265137,
      "step": 2190
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.970698933525409e-06,
      "logits/chosen": -1.288993000984192,
      "logits/rejected": -1.045488953590393,
      "logps/chosen": -470.0204162597656,
      "logps/rejected": -507.50372314453125,
      "loss": 0.632,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9067342281341553,
      "rewards/margins": 0.7715495228767395,
      "rewards/rejected": -2.67828369140625,
      "step": 2200
    },
    {
      "epoch": 0.14,
      "eval_logits/chosen": -1.086236834526062,
      "eval_logits/rejected": -0.9608862400054932,
      "eval_logps/chosen": -404.18316650390625,
      "eval_logps/rejected": -461.1840515136719,
      "eval_loss": 0.5549870729446411,
      "eval_rewards/accuracies": 0.6340000033378601,
      "eval_rewards/chosen": -1.7217819690704346,
      "eval_rewards/margins": 0.7739396095275879,
      "eval_rewards/rejected": -2.4957213401794434,
      "eval_runtime": 733.4615,
      "eval_samples_per_second": 2.727,
      "eval_steps_per_second": 1.363,
      "step": 2200
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.969820844938118e-06,
      "logits/chosen": -1.1498113870620728,
      "logits/rejected": -0.9385069012641907,
      "logps/chosen": -386.1590881347656,
      "logps/rejected": -409.1542053222656,
      "loss": 0.5245,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6355583667755127,
      "rewards/margins": 0.7689772844314575,
      "rewards/rejected": -2.4045357704162598,
      "step": 2210
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.968929872555444e-06,
      "logits/chosen": -0.7596795558929443,
      "logits/rejected": -0.6876972913742065,
      "logps/chosen": -450.9970703125,
      "logps/rejected": -542.7660522460938,
      "loss": 0.5959,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.3386266231536865,
      "rewards/margins": 0.5610593557357788,
      "rewards/rejected": -2.899685859680176,
      "step": 2220
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.968026021025137e-06,
      "logits/chosen": -1.260619878768921,
      "logits/rejected": -1.0677745342254639,
      "logps/chosen": -378.0556335449219,
      "logps/rejected": -438.29425048828125,
      "loss": 0.4583,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7649071216583252,
      "rewards/margins": 1.003040075302124,
      "rewards/rejected": -2.76794695854187,
      "step": 2230
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.967109295062128e-06,
      "logits/chosen": -1.1158151626586914,
      "logits/rejected": -1.0317364931106567,
      "logps/chosen": -394.54669189453125,
      "logps/rejected": -475.63983154296875,
      "loss": 0.5701,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.695452094078064,
      "rewards/margins": 0.6017216444015503,
      "rewards/rejected": -2.2971739768981934,
      "step": 2240
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.966179699448509e-06,
      "logits/chosen": -1.0546188354492188,
      "logits/rejected": -0.8983756303787231,
      "logps/chosen": -342.80010986328125,
      "logps/rejected": -383.49835205078125,
      "loss": 0.5724,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.5204514265060425,
      "rewards/margins": 0.5852440595626831,
      "rewards/rejected": -2.1056954860687256,
      "step": 2250
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.965237239033506e-06,
      "logits/chosen": -1.2999796867370605,
      "logits/rejected": -1.1046022176742554,
      "logps/chosen": -478.2933044433594,
      "logps/rejected": -566.3145141601562,
      "loss": 0.4039,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.886345624923706,
      "rewards/margins": 1.3280307054519653,
      "rewards/rejected": -3.214376449584961,
      "step": 2260
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.964281918733453e-06,
      "logits/chosen": -1.1871646642684937,
      "logits/rejected": -1.0754516124725342,
      "logps/chosen": -340.6067810058594,
      "logps/rejected": -444.7334899902344,
      "loss": 0.4687,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5749232769012451,
      "rewards/margins": 1.0858345031738281,
      "rewards/rejected": -2.6607577800750732,
      "step": 2270
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9633137435317715e-06,
      "logits/chosen": -1.2950812578201294,
      "logits/rejected": -0.8824887275695801,
      "logps/chosen": -359.2813720703125,
      "logps/rejected": -409.513916015625,
      "loss": 0.4435,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3638633489608765,
      "rewards/margins": 1.1801769733428955,
      "rewards/rejected": -2.5440402030944824,
      "step": 2280
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9623327184789355e-06,
      "logits/chosen": -1.4587196111679077,
      "logits/rejected": -1.4261622428894043,
      "logps/chosen": -321.40521240234375,
      "logps/rejected": -359.53515625,
      "loss": 0.6161,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.0366475582122803,
      "rewards/margins": 0.42956238985061646,
      "rewards/rejected": -1.4662100076675415,
      "step": 2290
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9613388486924525e-06,
      "logits/chosen": -1.0783612728118896,
      "logits/rejected": -1.239875316619873,
      "logps/chosen": -268.2691650390625,
      "logps/rejected": -347.37420654296875,
      "loss": 0.5211,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.881351113319397,
      "rewards/margins": 0.6172493696212769,
      "rewards/rejected": -1.498600721359253,
      "step": 2300
    },
    {
      "epoch": 0.15,
      "eval_logits/chosen": -1.415626049041748,
      "eval_logits/rejected": -1.2698040008544922,
      "eval_logps/chosen": -328.3125915527344,
      "eval_logps/rejected": -375.56829833984375,
      "eval_loss": 0.5417385697364807,
      "eval_rewards/accuracies": 0.637499988079071,
      "eval_rewards/chosen": -0.9630764126777649,
      "eval_rewards/margins": 0.6764876842498779,
      "eval_rewards/rejected": -1.6395639181137085,
      "eval_runtime": 731.6375,
      "eval_samples_per_second": 2.734,
      "eval_steps_per_second": 1.367,
      "step": 2300
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.960332139356834e-06,
      "logits/chosen": -1.3438931703567505,
      "logits/rejected": -1.1559361219406128,
      "logps/chosen": -311.91070556640625,
      "logps/rejected": -364.8311767578125,
      "loss": 0.482,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.0309386253356934,
      "rewards/margins": 0.7936037182807922,
      "rewards/rejected": -1.8245422840118408,
      "step": 2310
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.95931259572357e-06,
      "logits/chosen": -1.1916639804840088,
      "logits/rejected": -1.0088059902191162,
      "logps/chosen": -383.302490234375,
      "logps/rejected": -496.6766052246094,
      "loss": 0.526,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.490625023841858,
      "rewards/margins": 0.8770058751106262,
      "rewards/rejected": -2.3676304817199707,
      "step": 2320
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9582802231111e-06,
      "logits/chosen": -1.0349695682525635,
      "logits/rejected": -1.0374188423156738,
      "logps/chosen": -386.32452392578125,
      "logps/rejected": -437.10174560546875,
      "loss": 0.507,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7445039749145508,
      "rewards/margins": 0.7765570282936096,
      "rewards/rejected": -2.521061420440674,
      "step": 2330
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.957235026904782e-06,
      "logits/chosen": -1.1867823600769043,
      "logits/rejected": -0.9589727520942688,
      "logps/chosen": -424.76202392578125,
      "logps/rejected": -452.0746154785156,
      "loss": 0.5258,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.6260957717895508,
      "rewards/margins": 0.7842779159545898,
      "rewards/rejected": -2.4103739261627197,
      "step": 2340
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.956177012556875e-06,
      "logits/chosen": -1.2704133987426758,
      "logits/rejected": -1.0597444772720337,
      "logps/chosen": -461.6448669433594,
      "logps/rejected": -458.13299560546875,
      "loss": 0.58,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.17452073097229,
      "rewards/margins": 0.6805617213249207,
      "rewards/rejected": -2.8550829887390137,
      "step": 2350
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9551061855864976e-06,
      "logits/chosen": -0.8694220781326294,
      "logits/rejected": -0.9116750955581665,
      "logps/chosen": -376.36865234375,
      "logps/rejected": -435.46441650390625,
      "loss": 0.562,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.8354053497314453,
      "rewards/margins": 0.548851490020752,
      "rewards/rejected": -2.384256601333618,
      "step": 2360
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.95402255157961e-06,
      "logits/chosen": -1.0590431690216064,
      "logits/rejected": -1.0453912019729614,
      "logps/chosen": -326.3560485839844,
      "logps/rejected": -460.03277587890625,
      "loss": 0.5646,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4129979610443115,
      "rewards/margins": 0.6913048624992371,
      "rewards/rejected": -2.1043028831481934,
      "step": 2370
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.952926116188977e-06,
      "logits/chosen": -1.3469972610473633,
      "logits/rejected": -1.3643994331359863,
      "logps/chosen": -325.8696594238281,
      "logps/rejected": -389.6376037597656,
      "loss": 0.6367,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.4249229431152344,
      "rewards/margins": 0.24563416838645935,
      "rewards/rejected": -1.6705570220947266,
      "step": 2380
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.951816885134143e-06,
      "logits/chosen": -1.2549324035644531,
      "logits/rejected": -1.3286545276641846,
      "logps/chosen": -336.4953308105469,
      "logps/rejected": -380.0245361328125,
      "loss": 0.5888,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3494236469268799,
      "rewards/margins": 0.443899542093277,
      "rewards/rejected": -1.7933231592178345,
      "step": 2390
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.950694864201399e-06,
      "logits/chosen": -1.2524391412734985,
      "logits/rejected": -1.1568235158920288,
      "logps/chosen": -386.32135009765625,
      "logps/rejected": -451.83154296875,
      "loss": 0.4854,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.476393699645996,
      "rewards/margins": 0.6338094472885132,
      "rewards/rejected": -2.110203266143799,
      "step": 2400
    },
    {
      "epoch": 0.16,
      "eval_logits/chosen": -1.2359693050384521,
      "eval_logits/rejected": -1.1046502590179443,
      "eval_logps/chosen": -374.9135437011719,
      "eval_logps/rejected": -417.5105285644531,
      "eval_loss": 0.5439416170120239,
      "eval_rewards/accuracies": 0.640500009059906,
      "eval_rewards/chosen": -1.429085612297058,
      "eval_rewards/margins": 0.6299006342887878,
      "eval_rewards/rejected": -2.0589864253997803,
      "eval_runtime": 733.4368,
      "eval_samples_per_second": 2.727,
      "eval_steps_per_second": 1.363,
      "step": 2400
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9495600592437575e-06,
      "logits/chosen": -1.240966558456421,
      "logits/rejected": -1.223757028579712,
      "logps/chosen": -396.80218505859375,
      "logps/rejected": -469.73724365234375,
      "loss": 0.4975,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.6990772485733032,
      "rewards/margins": 0.6975347399711609,
      "rewards/rejected": -2.3966119289398193,
      "step": 2410
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.948412476180917e-06,
      "logits/chosen": -0.9485028386116028,
      "logits/rejected": -0.7619783878326416,
      "logps/chosen": -390.2615661621094,
      "logps/rejected": -470.1646423339844,
      "loss": 0.4895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.061856985092163,
      "rewards/margins": 1.0263975858688354,
      "rewards/rejected": -3.088254451751709,
      "step": 2420
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.947252120999232e-06,
      "logits/chosen": -0.9852539896965027,
      "logits/rejected": -0.7578643560409546,
      "logps/chosen": -502.28619384765625,
      "logps/rejected": -482.0196228027344,
      "loss": 0.7179,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.351813316345215,
      "rewards/margins": 0.3850732445716858,
      "rewards/rejected": -2.736886501312256,
      "step": 2430
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.946078999751683e-06,
      "logits/chosen": -0.9459400177001953,
      "logits/rejected": -0.7881075739860535,
      "logps/chosen": -340.50762939453125,
      "logps/rejected": -397.1718444824219,
      "loss": 0.476,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6088050603866577,
      "rewards/margins": 0.7822041511535645,
      "rewards/rejected": -2.3910090923309326,
      "step": 2440
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.944893118557847e-06,
      "logits/chosen": -0.9686307907104492,
      "logits/rejected": -0.880931556224823,
      "logps/chosen": -377.506103515625,
      "logps/rejected": -386.24383544921875,
      "loss": 0.5469,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7102149724960327,
      "rewards/margins": 0.6242817640304565,
      "rewards/rejected": -2.3344969749450684,
      "step": 2450
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.943694483603861e-06,
      "logits/chosen": -1.3755087852478027,
      "logits/rejected": -1.023921251296997,
      "logps/chosen": -330.78143310546875,
      "logps/rejected": -368.04034423828125,
      "loss": 0.4777,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.0477737188339233,
      "rewards/margins": 0.8823789358139038,
      "rewards/rejected": -1.9301526546478271,
      "step": 2460
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9424831011423914e-06,
      "logits/chosen": -1.3603510856628418,
      "logits/rejected": -1.3010962009429932,
      "logps/chosen": -402.3771057128906,
      "logps/rejected": -410.086181640625,
      "loss": 0.6041,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1395056247711182,
      "rewards/margins": 0.44723695516586304,
      "rewards/rejected": -1.5867425203323364,
      "step": 2470
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9412589774926015e-06,
      "logits/chosen": -1.409360647201538,
      "logits/rejected": -1.1556938886642456,
      "logps/chosen": -383.52313232421875,
      "logps/rejected": -408.6988525390625,
      "loss": 0.532,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.0933339595794678,
      "rewards/margins": 0.757992148399353,
      "rewards/rejected": -1.8513259887695312,
      "step": 2480
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.940022119040121e-06,
      "logits/chosen": -1.3208924531936646,
      "logits/rejected": -1.1365283727645874,
      "logps/chosen": -410.063720703125,
      "logps/rejected": -431.35003662109375,
      "loss": 0.593,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.181477665901184,
      "rewards/margins": 0.49104517698287964,
      "rewards/rejected": -1.6725225448608398,
      "step": 2490
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.93877253223701e-06,
      "logits/chosen": -1.073589563369751,
      "logits/rejected": -1.0274441242218018,
      "logps/chosen": -459.6949768066406,
      "logps/rejected": -509.46380615234375,
      "loss": 0.4768,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6742274761199951,
      "rewards/margins": 0.8134153485298157,
      "rewards/rejected": -2.487642765045166,
      "step": 2500
    },
    {
      "epoch": 0.16,
      "eval_logits/chosen": -0.9927495121955872,
      "eval_logits/rejected": -0.8693163990974426,
      "eval_logps/chosen": -433.1884460449219,
      "eval_logps/rejected": -486.5682373046875,
      "eval_loss": 0.5401906967163086,
      "eval_rewards/accuracies": 0.6359999775886536,
      "eval_rewards/chosen": -2.0118348598480225,
      "eval_rewards/margins": 0.7377288937568665,
      "eval_rewards/rejected": -2.749563455581665,
      "eval_runtime": 732.9007,
      "eval_samples_per_second": 2.729,
      "eval_steps_per_second": 1.364,
      "step": 2500
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.937510223601725e-06,
      "logits/chosen": -1.2273633480072021,
      "logits/rejected": -1.1791640520095825,
      "logps/chosen": -452.0333557128906,
      "logps/rejected": -472.3573303222656,
      "loss": 0.5202,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.895073652267456,
      "rewards/margins": 0.6253536939620972,
      "rewards/rejected": -2.5204272270202637,
      "step": 2510
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.936235199719085e-06,
      "logits/chosen": -0.900462806224823,
      "logits/rejected": -0.8040259480476379,
      "logps/chosen": -408.2585144042969,
      "logps/rejected": -480.28875732421875,
      "loss": 0.4679,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.359565258026123,
      "rewards/margins": 1.022094488143921,
      "rewards/rejected": -3.381659746170044,
      "step": 2520
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.93494746724024e-06,
      "logits/chosen": -0.9940454363822937,
      "logits/rejected": -0.8842470049858093,
      "logps/chosen": -448.1106872558594,
      "logps/rejected": -553.7257080078125,
      "loss": 0.462,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.2430620193481445,
      "rewards/margins": 0.8815500140190125,
      "rewards/rejected": -3.1246120929718018,
      "step": 2530
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.933647032882635e-06,
      "logits/chosen": -1.041330099105835,
      "logits/rejected": -0.8723998069763184,
      "logps/chosen": -496.6675720214844,
      "logps/rejected": -534.0340576171875,
      "loss": 0.5552,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.518056869506836,
      "rewards/margins": 0.8178606033325195,
      "rewards/rejected": -3.3359177112579346,
      "step": 2540
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.932333903429969e-06,
      "logits/chosen": -0.698738694190979,
      "logits/rejected": -0.5703662633895874,
      "logps/chosen": -446.97027587890625,
      "logps/rejected": -452.32757568359375,
      "loss": 0.7093,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -2.5388777256011963,
      "rewards/margins": 0.3116335868835449,
      "rewards/rejected": -2.850511312484741,
      "step": 2550
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.931008085732172e-06,
      "logits/chosen": -0.9040409326553345,
      "logits/rejected": -0.5525091886520386,
      "logps/chosen": -459.99798583984375,
      "logps/rejected": -480.68487548828125,
      "loss": 0.4977,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.5717508792877197,
      "rewards/margins": 0.7068825364112854,
      "rewards/rejected": -3.2786335945129395,
      "step": 2560
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9296695867053565e-06,
      "logits/chosen": -1.0348608493804932,
      "logits/rejected": -0.7598494291305542,
      "logps/chosen": -526.0467529296875,
      "logps/rejected": -582.1028442382812,
      "loss": 0.4165,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.2544517517089844,
      "rewards/margins": 1.220160722732544,
      "rewards/rejected": -3.4746127128601074,
      "step": 2570
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.928318413331791e-06,
      "logits/chosen": -1.0867879390716553,
      "logits/rejected": -1.023646593093872,
      "logps/chosen": -433.85919189453125,
      "logps/rejected": -448.56402587890625,
      "loss": 0.6846,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.226320743560791,
      "rewards/margins": 0.30690130591392517,
      "rewards/rejected": -2.533221960067749,
      "step": 2580
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.926954572659855e-06,
      "logits/chosen": -0.9278467297554016,
      "logits/rejected": -0.8537686467170715,
      "logps/chosen": -448.48760986328125,
      "logps/rejected": -546.9074096679688,
      "loss": 0.4983,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0629193782806396,
      "rewards/margins": 0.8123389482498169,
      "rewards/rejected": -2.875257968902588,
      "step": 2590
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.925578071804013e-06,
      "logits/chosen": -0.8431898355484009,
      "logits/rejected": -0.8421382904052734,
      "logps/chosen": -424.267578125,
      "logps/rejected": -553.5916748046875,
      "loss": 0.562,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.94561767578125,
      "rewards/margins": 0.7191228866577148,
      "rewards/rejected": -2.6647403240203857,
      "step": 2600
    },
    {
      "epoch": 0.17,
      "eval_logits/chosen": -1.0128614902496338,
      "eval_logits/rejected": -0.891107439994812,
      "eval_logps/chosen": -433.5695495605469,
      "eval_logps/rejected": -486.4391174316406,
      "eval_loss": 0.527816891670227,
      "eval_rewards/accuracies": 0.6604999899864197,
      "eval_rewards/chosen": -2.01564621925354,
      "eval_rewards/margins": 0.7326260209083557,
      "eval_rewards/rejected": -2.748272180557251,
      "eval_runtime": 732.9757,
      "eval_samples_per_second": 2.729,
      "eval_steps_per_second": 1.364,
      "step": 2600
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.924188917944763e-06,
      "logits/chosen": -1.1159459352493286,
      "logits/rejected": -0.8869976997375488,
      "logps/chosen": -421.93389892578125,
      "logps/rejected": -481.2115783691406,
      "loss": 0.491,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0192878246307373,
      "rewards/margins": 0.9279811978340149,
      "rewards/rejected": -2.9472689628601074,
      "step": 2610
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.922787118328617e-06,
      "logits/chosen": -1.141663908958435,
      "logits/rejected": -0.7913933396339417,
      "logps/chosen": -453.99383544921875,
      "logps/rejected": -407.4873046875,
      "loss": 0.6754,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -2.2397615909576416,
      "rewards/margins": 0.2980630099773407,
      "rewards/rejected": -2.537824869155884,
      "step": 2620
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.921372680268045e-06,
      "logits/chosen": -0.980755627155304,
      "logits/rejected": -0.9280126690864563,
      "logps/chosen": -459.7035217285156,
      "logps/rejected": -448.31048583984375,
      "loss": 0.7411,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.296988010406494,
      "rewards/margins": 0.18975761532783508,
      "rewards/rejected": -2.486745834350586,
      "step": 2630
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.919945611141451e-06,
      "logits/chosen": -1.341280460357666,
      "logits/rejected": -1.0558829307556152,
      "logps/chosen": -359.87152099609375,
      "logps/rejected": -346.2890930175781,
      "loss": 0.5557,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3463690280914307,
      "rewards/margins": 0.44936901330947876,
      "rewards/rejected": -1.795737862586975,
      "step": 2640
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.918505918393125e-06,
      "logits/chosen": -1.2368559837341309,
      "logits/rejected": -1.1329363584518433,
      "logps/chosen": -267.66839599609375,
      "logps/rejected": -348.6911315917969,
      "loss": 0.5407,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9863770604133606,
      "rewards/margins": 0.5764933824539185,
      "rewards/rejected": -1.5628705024719238,
      "step": 2650
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.91705360953321e-06,
      "logits/chosen": -1.203028917312622,
      "logits/rejected": -1.117807388305664,
      "logps/chosen": -373.701416015625,
      "logps/rejected": -405.60211181640625,
      "loss": 0.4926,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.2821382284164429,
      "rewards/margins": 0.6455599665641785,
      "rewards/rejected": -1.9276981353759766,
      "step": 2660
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9155886921376615e-06,
      "logits/chosen": -1.1834170818328857,
      "logits/rejected": -1.1657710075378418,
      "logps/chosen": -341.26007080078125,
      "logps/rejected": -403.8941955566406,
      "loss": 0.5907,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3529164791107178,
      "rewards/margins": 0.4096578061580658,
      "rewards/rejected": -1.7625744342803955,
      "step": 2670
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.914111173848205e-06,
      "logits/chosen": -1.3189375400543213,
      "logits/rejected": -1.2658817768096924,
      "logps/chosen": -385.70587158203125,
      "logps/rejected": -400.50225830078125,
      "loss": 0.6556,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.526742935180664,
      "rewards/margins": 0.302143394947052,
      "rewards/rejected": -1.8288863897323608,
      "step": 2680
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9126210623723e-06,
      "logits/chosen": -1.1100072860717773,
      "logits/rejected": -1.185240626335144,
      "logps/chosen": -326.9442138671875,
      "logps/rejected": -431.2759704589844,
      "loss": 0.5286,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2657082080841064,
      "rewards/margins": 0.7055976390838623,
      "rewards/rejected": -1.9713056087493896,
      "step": 2690
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.911118365483098e-06,
      "logits/chosen": -1.2706750631332397,
      "logits/rejected": -1.2281222343444824,
      "logps/chosen": -346.92291259765625,
      "logps/rejected": -448.357177734375,
      "loss": 0.4748,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.451686978340149,
      "rewards/margins": 0.9200803637504578,
      "rewards/rejected": -2.371767520904541,
      "step": 2700
    },
    {
      "epoch": 0.18,
      "eval_logits/chosen": -1.2759194374084473,
      "eval_logits/rejected": -1.1406419277191162,
      "eval_logps/chosen": -376.82635498046875,
      "eval_logps/rejected": -422.0545349121094,
      "eval_loss": 0.5314846038818359,
      "eval_rewards/accuracies": 0.6514999866485596,
      "eval_rewards/chosen": -1.4482139348983765,
      "eval_rewards/margins": 0.6562120914459229,
      "eval_rewards/rejected": -2.1044256687164307,
      "eval_runtime": 733.3385,
      "eval_samples_per_second": 2.727,
      "eval_steps_per_second": 1.364,
      "step": 2700
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.909603091019403e-06,
      "logits/chosen": -1.401475191116333,
      "logits/rejected": -1.179467797279358,
      "logps/chosen": -364.406494140625,
      "logps/rejected": -411.4903259277344,
      "loss": 0.482,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2802919149398804,
      "rewards/margins": 0.8370407223701477,
      "rewards/rejected": -2.1173324584960938,
      "step": 2710
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.908075246885626e-06,
      "logits/chosen": -1.1405667066574097,
      "logits/rejected": -1.1133387088775635,
      "logps/chosen": -269.01739501953125,
      "logps/rejected": -277.61370849609375,
      "loss": 0.6395,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.1828992366790771,
      "rewards/margins": 0.35987579822540283,
      "rewards/rejected": -1.5427749156951904,
      "step": 2720
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.906534841051755e-06,
      "logits/chosen": -1.0903264284133911,
      "logits/rejected": -1.0795084238052368,
      "logps/chosen": -362.6637268066406,
      "logps/rejected": -419.4922790527344,
      "loss": 0.6301,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1926027536392212,
      "rewards/margins": 0.45792943239212036,
      "rewards/rejected": -1.6505321264266968,
      "step": 2730
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.904981881553297e-06,
      "logits/chosen": -1.340993881225586,
      "logits/rejected": -1.1244676113128662,
      "logps/chosen": -337.9717102050781,
      "logps/rejected": -317.94488525390625,
      "loss": 0.5715,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.137753963470459,
      "rewards/margins": 0.4409133791923523,
      "rewards/rejected": -1.578667402267456,
      "step": 2740
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.903416376491252e-06,
      "logits/chosen": -1.4110819101333618,
      "logits/rejected": -1.1954363584518433,
      "logps/chosen": -390.9253845214844,
      "logps/rejected": -444.5638122558594,
      "loss": 0.4639,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1263313293457031,
      "rewards/margins": 0.9423055648803711,
      "rewards/rejected": -2.068636894226074,
      "step": 2750
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.90183833403206e-06,
      "logits/chosen": -1.5229685306549072,
      "logits/rejected": -1.4395592212677002,
      "logps/chosen": -385.73980712890625,
      "logps/rejected": -420.03094482421875,
      "loss": 0.4857,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.2152669429779053,
      "rewards/margins": 0.7054417729377747,
      "rewards/rejected": -1.9207088947296143,
      "step": 2760
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.900247762407564e-06,
      "logits/chosen": -1.0933618545532227,
      "logits/rejected": -1.0539687871932983,
      "logps/chosen": -344.75738525390625,
      "logps/rejected": -438.8707580566406,
      "loss": 0.5357,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.724654197692871,
      "rewards/margins": 0.7207070589065552,
      "rewards/rejected": -2.4453611373901367,
      "step": 2770
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.898644669914965e-06,
      "logits/chosen": -1.0270262956619263,
      "logits/rejected": -0.965491771697998,
      "logps/chosen": -471.7977600097656,
      "logps/rejected": -546.3108520507812,
      "loss": 0.5477,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.4772531986236572,
      "rewards/margins": 0.8833915591239929,
      "rewards/rejected": -3.360644578933716,
      "step": 2780
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.897029064916778e-06,
      "logits/chosen": -0.7744370698928833,
      "logits/rejected": -0.7252274751663208,
      "logps/chosen": -438.18341064453125,
      "logps/rejected": -465.3837890625,
      "loss": 0.7191,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -2.3003344535827637,
      "rewards/margins": 0.3958927094936371,
      "rewards/rejected": -2.6962273120880127,
      "step": 2790
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.895400955840791e-06,
      "logits/chosen": -1.1814029216766357,
      "logits/rejected": -0.6611972451210022,
      "logps/chosen": -378.6611328125,
      "logps/rejected": -414.54052734375,
      "loss": 0.5099,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.58309805393219,
      "rewards/margins": 0.7757716178894043,
      "rewards/rejected": -2.358869791030884,
      "step": 2800
    },
    {
      "epoch": 0.18,
      "eval_logits/chosen": -1.0748944282531738,
      "eval_logits/rejected": -0.9484049081802368,
      "eval_logps/chosen": -392.2981872558594,
      "eval_logps/rejected": -440.3302917480469,
      "eval_loss": 0.5305582284927368,
      "eval_rewards/accuracies": 0.6549999713897705,
      "eval_rewards/chosen": -1.6029322147369385,
      "eval_rewards/margins": 0.684251606464386,
      "eval_rewards/rejected": -2.2871837615966797,
      "eval_runtime": 734.2092,
      "eval_samples_per_second": 2.724,
      "eval_steps_per_second": 1.362,
      "step": 2800
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.893760351180018e-06,
      "logits/chosen": -1.0402562618255615,
      "logits/rejected": -1.0653431415557861,
      "logps/chosen": -370.8222351074219,
      "logps/rejected": -438.2054138183594,
      "loss": 0.6082,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.7228095531463623,
      "rewards/margins": 0.6228219270706177,
      "rewards/rejected": -2.3456313610076904,
      "step": 2810
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.892107259492657e-06,
      "logits/chosen": -1.0466301441192627,
      "logits/rejected": -0.919894814491272,
      "logps/chosen": -394.4199523925781,
      "logps/rejected": -442.1200256347656,
      "loss": 0.6315,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.5456066131591797,
      "rewards/margins": 0.41681140661239624,
      "rewards/rejected": -1.9624179601669312,
      "step": 2820
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.890441689402042e-06,
      "logits/chosen": -1.3622853755950928,
      "logits/rejected": -1.162502408027649,
      "logps/chosen": -481.157958984375,
      "logps/rejected": -511.90240478515625,
      "loss": 0.4618,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4565303325653076,
      "rewards/margins": 0.7874441146850586,
      "rewards/rejected": -2.243974208831787,
      "step": 2830
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.888763649596606e-06,
      "logits/chosen": -1.2705270051956177,
      "logits/rejected": -1.1603574752807617,
      "logps/chosen": -331.831298828125,
      "logps/rejected": -385.3037109375,
      "loss": 0.5833,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.2166961431503296,
      "rewards/margins": 0.6227377653121948,
      "rewards/rejected": -1.839434027671814,
      "step": 2840
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.887073148829824e-06,
      "logits/chosen": -1.1340843439102173,
      "logits/rejected": -1.0734115839004517,
      "logps/chosen": -406.3775939941406,
      "logps/rejected": -445.03936767578125,
      "loss": 0.5258,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.394740104675293,
      "rewards/margins": 0.6159732937812805,
      "rewards/rejected": -2.0107131004333496,
      "step": 2850
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.885370195920177e-06,
      "logits/chosen": -0.9719009399414062,
      "logits/rejected": -0.9220370054244995,
      "logps/chosen": -333.6672058105469,
      "logps/rejected": -379.9608459472656,
      "loss": 0.5954,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.483434796333313,
      "rewards/margins": 0.5486684441566467,
      "rewards/rejected": -2.0321033000946045,
      "step": 2860
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.883654799751101e-06,
      "logits/chosen": -1.0081456899642944,
      "logits/rejected": -0.9878345727920532,
      "logps/chosen": -335.7508544921875,
      "logps/rejected": -447.65374755859375,
      "loss": 0.5001,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.200417160987854,
      "rewards/margins": 0.7716276049613953,
      "rewards/rejected": -1.972044587135315,
      "step": 2870
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8819269692709435e-06,
      "logits/chosen": -1.1583213806152344,
      "logits/rejected": -0.9544271230697632,
      "logps/chosen": -384.2664489746094,
      "logps/rejected": -385.5480651855469,
      "loss": 0.5024,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.1722023487091064,
      "rewards/margins": 0.736437201499939,
      "rewards/rejected": -1.9086395502090454,
      "step": 2880
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.880186713492915e-06,
      "logits/chosen": -1.0247581005096436,
      "logits/rejected": -0.8900755047798157,
      "logps/chosen": -375.8417663574219,
      "logps/rejected": -384.86859130859375,
      "loss": 0.535,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4812443256378174,
      "rewards/margins": 0.626487672328949,
      "rewards/rejected": -2.1077322959899902,
      "step": 2890
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.878434041495041e-06,
      "logits/chosen": -1.0221517086029053,
      "logits/rejected": -1.1312007904052734,
      "logps/chosen": -399.26617431640625,
      "logps/rejected": -516.553955078125,
      "loss": 0.4184,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.6210047006607056,
      "rewards/margins": 1.154254674911499,
      "rewards/rejected": -2.775259494781494,
      "step": 2900
    },
    {
      "epoch": 0.19,
      "eval_logits/chosen": -0.907734751701355,
      "eval_logits/rejected": -0.7929914593696594,
      "eval_logps/chosen": -393.54962158203125,
      "eval_logps/rejected": -452.6503601074219,
      "eval_loss": 0.5266603827476501,
      "eval_rewards/accuracies": 0.6514999866485596,
      "eval_rewards/chosen": -1.6154468059539795,
      "eval_rewards/margins": 0.7949380874633789,
      "eval_rewards/rejected": -2.4103844165802,
      "eval_runtime": 730.902,
      "eval_samples_per_second": 2.736,
      "eval_steps_per_second": 1.368,
      "step": 2900
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.876668962420117e-06,
      "logits/chosen": -0.8577669858932495,
      "logits/rejected": -0.7146766781806946,
      "logps/chosen": -455.3329162597656,
      "logps/rejected": -499.1671447753906,
      "loss": 0.4407,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.656818151473999,
      "rewards/margins": 1.034503698348999,
      "rewards/rejected": -2.691321849822998,
      "step": 2910
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.87489148547566e-06,
      "logits/chosen": -0.8334639668464661,
      "logits/rejected": -0.7265030145645142,
      "logps/chosen": -500.32470703125,
      "logps/rejected": -542.7216796875,
      "loss": 0.6218,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.4046618938446045,
      "rewards/margins": 0.7556990385055542,
      "rewards/rejected": -3.1603610515594482,
      "step": 2920
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.873101619933862e-06,
      "logits/chosen": -0.9287866353988647,
      "logits/rejected": -0.6335155367851257,
      "logps/chosen": -481.3199768066406,
      "logps/rejected": -514.2613525390625,
      "loss": 0.5545,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.1679275035858154,
      "rewards/margins": 0.8225703239440918,
      "rewards/rejected": -2.9904978275299072,
      "step": 2930
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8712993751315385e-06,
      "logits/chosen": -0.9662561416625977,
      "logits/rejected": -0.8811322450637817,
      "logps/chosen": -273.9098205566406,
      "logps/rejected": -336.27252197265625,
      "loss": 0.5461,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.5325188636779785,
      "rewards/margins": 0.5846083164215088,
      "rewards/rejected": -2.1171271800994873,
      "step": 2940
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.869484760470079e-06,
      "logits/chosen": -0.9696873426437378,
      "logits/rejected": -0.8237133026123047,
      "logps/chosen": -332.93359375,
      "logps/rejected": -366.3995361328125,
      "loss": 0.5814,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4427276849746704,
      "rewards/margins": 0.6976507902145386,
      "rewards/rejected": -2.140378475189209,
      "step": 2950
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.867657785415404e-06,
      "logits/chosen": -0.9058094024658203,
      "logits/rejected": -0.7002671360969543,
      "logps/chosen": -431.8790588378906,
      "logps/rejected": -483.63006591796875,
      "loss": 0.4947,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.8278001546859741,
      "rewards/margins": 0.934956431388855,
      "rewards/rejected": -2.762756824493408,
      "step": 2960
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.865818459497911e-06,
      "logits/chosen": -1.0235707759857178,
      "logits/rejected": -0.9009502530097961,
      "logps/chosen": -490.59881591796875,
      "logps/rejected": -472.571044921875,
      "loss": 0.5176,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.0642013549804688,
      "rewards/margins": 0.6676945686340332,
      "rewards/rejected": -2.731895923614502,
      "step": 2970
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.863966792312423e-06,
      "logits/chosen": -1.034481167793274,
      "logits/rejected": -0.6752454042434692,
      "logps/chosen": -434.35595703125,
      "logps/rejected": -499.046875,
      "loss": 0.4784,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9289706945419312,
      "rewards/margins": 1.0574229955673218,
      "rewards/rejected": -2.986393690109253,
      "step": 2980
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.862102793518145e-06,
      "logits/chosen": -0.956391453742981,
      "logits/rejected": -1.0064009428024292,
      "logps/chosen": -385.90771484375,
      "logps/rejected": -467.2142028808594,
      "loss": 0.5787,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9162585735321045,
      "rewards/margins": 0.7438151240348816,
      "rewards/rejected": -2.660073757171631,
      "step": 2990
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8602264728386075e-06,
      "logits/chosen": -1.1530312299728394,
      "logits/rejected": -1.024060845375061,
      "logps/chosen": -411.135009765625,
      "logps/rejected": -497.8170471191406,
      "loss": 0.468,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.586355209350586,
      "rewards/margins": 0.9010065197944641,
      "rewards/rejected": -2.487361431121826,
      "step": 3000
    },
    {
      "epoch": 0.2,
      "eval_logits/chosen": -1.0168744325637817,
      "eval_logits/rejected": -0.8916241526603699,
      "eval_logps/chosen": -405.43792724609375,
      "eval_logps/rejected": -467.9596252441406,
      "eval_loss": 0.5222868919372559,
      "eval_rewards/accuracies": 0.6554999947547913,
      "eval_rewards/chosen": -1.7343300580978394,
      "eval_rewards/margins": 0.8291473388671875,
      "eval_rewards/rejected": -2.563477039337158,
      "eval_runtime": 733.6976,
      "eval_samples_per_second": 2.726,
      "eval_steps_per_second": 1.363,
      "step": 3000
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.858337840061616e-06,
      "logits/chosen": -0.8814601898193359,
      "logits/rejected": -0.9170808792114258,
      "logps/chosen": -346.5814514160156,
      "logps/rejected": -471.98895263671875,
      "loss": 0.5828,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.6350104808807373,
      "rewards/margins": 0.717583954334259,
      "rewards/rejected": -2.3525943756103516,
      "step": 3010
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.856436905039208e-06,
      "logits/chosen": -0.8554957509040833,
      "logits/rejected": -0.6725913286209106,
      "logps/chosen": -409.9076232910156,
      "logps/rejected": -458.8953552246094,
      "loss": 0.4747,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.987026572227478,
      "rewards/margins": 0.8337499499320984,
      "rewards/rejected": -2.8207764625549316,
      "step": 3020
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.854523677687588e-06,
      "logits/chosen": -0.6897114515304565,
      "logits/rejected": -0.8571540713310242,
      "logps/chosen": -395.29534912109375,
      "logps/rejected": -481.5233459472656,
      "loss": 0.5643,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.1404976844787598,
      "rewards/margins": 0.6783149242401123,
      "rewards/rejected": -2.818812608718872,
      "step": 3030
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.85259816798709e-06,
      "logits/chosen": -1.00284743309021,
      "logits/rejected": -0.6904066801071167,
      "logps/chosen": -503.5082092285156,
      "logps/rejected": -529.4007568359375,
      "loss": 0.3823,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.157461404800415,
      "rewards/margins": 1.0644841194152832,
      "rewards/rejected": -3.221945285797119,
      "step": 3040
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.850660385982114e-06,
      "logits/chosen": -0.9757447242736816,
      "logits/rejected": -0.8276710510253906,
      "logps/chosen": -455.947509765625,
      "logps/rejected": -474.081298828125,
      "loss": 0.5308,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.0822834968566895,
      "rewards/margins": 0.7341514825820923,
      "rewards/rejected": -2.816434621810913,
      "step": 3050
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.848710341781081e-06,
      "logits/chosen": -0.640845000743866,
      "logits/rejected": -0.7251681089401245,
      "logps/chosen": -449.40155029296875,
      "logps/rejected": -478.784912109375,
      "loss": 0.6729,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.754999876022339,
      "rewards/margins": 0.35110121965408325,
      "rewards/rejected": -3.1061012744903564,
      "step": 3060
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.846748045556377e-06,
      "logits/chosen": -0.6704039573669434,
      "logits/rejected": -0.5354386568069458,
      "logps/chosen": -501.2828063964844,
      "logps/rejected": -522.3720703125,
      "loss": 0.5013,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.6179566383361816,
      "rewards/margins": 0.8160024881362915,
      "rewards/rejected": -3.4339592456817627,
      "step": 3070
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8447735075442995e-06,
      "logits/chosen": -0.637302577495575,
      "logits/rejected": -0.49296456575393677,
      "logps/chosen": -482.5350646972656,
      "logps/rejected": -590.9215698242188,
      "loss": 0.4817,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.8497281074523926,
      "rewards/margins": 1.117870569229126,
      "rewards/rejected": -3.9675984382629395,
      "step": 3080
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8427867380450075e-06,
      "logits/chosen": -0.9691807627677917,
      "logits/rejected": -0.6462706923484802,
      "logps/chosen": -459.28887939453125,
      "logps/rejected": -493.9776306152344,
      "loss": 0.5673,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.2894959449768066,
      "rewards/margins": 0.8093854784965515,
      "rewards/rejected": -3.098881483078003,
      "step": 3090
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.840787747422462e-06,
      "logits/chosen": -1.2492235898971558,
      "logits/rejected": -1.0929436683654785,
      "logps/chosen": -324.46063232421875,
      "logps/rejected": -351.09698486328125,
      "loss": 0.5857,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.2880451679229736,
      "rewards/margins": 0.5689392685890198,
      "rewards/rejected": -1.8569844961166382,
      "step": 3100
    },
    {
      "epoch": 0.2,
      "eval_logits/chosen": -1.2385766506195068,
      "eval_logits/rejected": -1.1036633253097534,
      "eval_logps/chosen": -358.3794860839844,
      "eval_logps/rejected": -410.8307800292969,
      "eval_loss": 0.5289512872695923,
      "eval_rewards/accuracies": 0.6520000100135803,
      "eval_rewards/chosen": -1.2637453079223633,
      "eval_rewards/margins": 0.7284430861473083,
      "eval_rewards/rejected": -1.9921886920928955,
      "eval_runtime": 732.5166,
      "eval_samples_per_second": 2.73,
      "eval_steps_per_second": 1.365,
      "step": 3100
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.838776546104378e-06,
      "logits/chosen": -1.2828580141067505,
      "logits/rejected": -1.148850679397583,
      "logps/chosen": -423.83135986328125,
      "logps/rejected": -468.1695861816406,
      "loss": 0.4887,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4287811517715454,
      "rewards/margins": 0.8474019169807434,
      "rewards/rejected": -2.2761833667755127,
      "step": 3110
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.836753144582168e-06,
      "logits/chosen": -1.1283457279205322,
      "logits/rejected": -0.7700001001358032,
      "logps/chosen": -421.4664611816406,
      "logps/rejected": -505.2184143066406,
      "loss": 0.443,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7923486232757568,
      "rewards/margins": 1.0786457061767578,
      "rewards/rejected": -2.8709943294525146,
      "step": 3120
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.834717553410884e-06,
      "logits/chosen": -1.1379692554473877,
      "logits/rejected": -1.06531822681427,
      "logps/chosen": -363.9742126464844,
      "logps/rejected": -480.22222900390625,
      "loss": 0.5057,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7469819784164429,
      "rewards/margins": 1.025788426399231,
      "rewards/rejected": -2.7727701663970947,
      "step": 3130
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.832669783209167e-06,
      "logits/chosen": -0.9420145750045776,
      "logits/rejected": -1.055405616760254,
      "logps/chosen": -443.606201171875,
      "logps/rejected": -490.388671875,
      "loss": 0.6093,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.9918018579483032,
      "rewards/margins": 0.46233540773391724,
      "rewards/rejected": -2.454137086868286,
      "step": 3140
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.8306098446591895e-06,
      "logits/chosen": -0.7147270441055298,
      "logits/rejected": -0.7811274528503418,
      "logps/chosen": -347.41033935546875,
      "logps/rejected": -424.43267822265625,
      "loss": 0.5833,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.711414098739624,
      "rewards/margins": 0.5209105610847473,
      "rewards/rejected": -2.2323248386383057,
      "step": 3150
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828537748506601e-06,
      "logits/chosen": -1.4567492008209229,
      "logits/rejected": -1.2312958240509033,
      "logps/chosen": -414.09161376953125,
      "logps/rejected": -411.62432861328125,
      "loss": 0.5788,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4069335460662842,
      "rewards/margins": 0.5377890467643738,
      "rewards/rejected": -1.9447225332260132,
      "step": 3160
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.826453505560469e-06,
      "logits/chosen": -1.0341746807098389,
      "logits/rejected": -1.0024795532226562,
      "logps/chosen": -364.7231140136719,
      "logps/rejected": -411.63037109375,
      "loss": 0.5837,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.7374398708343506,
      "rewards/margins": 0.6060377359390259,
      "rewards/rejected": -2.343477725982666,
      "step": 3170
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.824357126693226e-06,
      "logits/chosen": -1.0167440176010132,
      "logits/rejected": -0.9091693162918091,
      "logps/chosen": -429.69970703125,
      "logps/rejected": -449.07855224609375,
      "loss": 0.5235,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7095658779144287,
      "rewards/margins": 0.6810619831085205,
      "rewards/rejected": -2.39062762260437,
      "step": 3180
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.8222486228406105e-06,
      "logits/chosen": -1.2612932920455933,
      "logits/rejected": -1.0270341634750366,
      "logps/chosen": -406.03485107421875,
      "logps/rejected": -476.10479736328125,
      "loss": 0.4328,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9479446411132812,
      "rewards/margins": 1.0475102663040161,
      "rewards/rejected": -2.995454788208008,
      "step": 3190
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.820128005001612e-06,
      "logits/chosen": -0.9638387560844421,
      "logits/rejected": -0.8402373194694519,
      "logps/chosen": -445.51678466796875,
      "logps/rejected": -538.071044921875,
      "loss": 0.4504,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2652316093444824,
      "rewards/margins": 1.1409714221954346,
      "rewards/rejected": -3.406203031539917,
      "step": 3200
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": -1.1166863441467285,
      "eval_logits/rejected": -0.9832271337509155,
      "eval_logps/chosen": -494.8057861328125,
      "eval_logps/rejected": -568.17138671875,
      "eval_loss": 0.5196487307548523,
      "eval_rewards/accuracies": 0.6514999866485596,
      "eval_rewards/chosen": -2.6280083656311035,
      "eval_rewards/margins": 0.9375866055488586,
      "eval_rewards/rejected": -3.5655946731567383,
      "eval_runtime": 732.8446,
      "eval_samples_per_second": 2.729,
      "eval_steps_per_second": 1.365,
      "step": 3200
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.817995284238412e-06,
      "logits/chosen": -0.996303915977478,
      "logits/rejected": -1.036203145980835,
      "logps/chosen": -439.01483154296875,
      "logps/rejected": -553.1556396484375,
      "loss": 0.5445,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.4249050617218018,
      "rewards/margins": 0.866560161113739,
      "rewards/rejected": -3.2914650440216064,
      "step": 3210
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.815850471676327e-06,
      "logits/chosen": -1.1847374439239502,
      "logits/rejected": -0.9033486247062683,
      "logps/chosen": -514.692626953125,
      "logps/rejected": -615.4830322265625,
      "loss": 0.4885,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.7627861499786377,
      "rewards/margins": 1.050376534461975,
      "rewards/rejected": -3.8131625652313232,
      "step": 3220
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.813693578503751e-06,
      "logits/chosen": -1.0441510677337646,
      "logits/rejected": -0.860817551612854,
      "logps/chosen": -532.6041870117188,
      "logps/rejected": -567.3224487304688,
      "loss": 0.4816,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.360973834991455,
      "rewards/margins": 0.9017590284347534,
      "rewards/rejected": -3.262732982635498,
      "step": 3230
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.811524615972093e-06,
      "logits/chosen": -0.7930616140365601,
      "logits/rejected": -0.8713744282722473,
      "logps/chosen": -501.078125,
      "logps/rejected": -601.9039916992188,
      "loss": 0.5722,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.7114250659942627,
      "rewards/margins": 0.9262357950210571,
      "rewards/rejected": -3.6376609802246094,
      "step": 3240
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.809343595395724e-06,
      "logits/chosen": -1.4246814250946045,
      "logits/rejected": -1.1265549659729004,
      "logps/chosen": -415.13128662109375,
      "logps/rejected": -465.318359375,
      "loss": 0.5769,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.260756254196167,
      "rewards/margins": 0.8077578544616699,
      "rewards/rejected": -3.068514108657837,
      "step": 3250
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.807150528151918e-06,
      "logits/chosen": -1.0579570531845093,
      "logits/rejected": -1.0017118453979492,
      "logps/chosen": -337.6005554199219,
      "logps/rejected": -462.49261474609375,
      "loss": 0.4633,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7090799808502197,
      "rewards/margins": 1.0702515840530396,
      "rewards/rejected": -2.779331684112549,
      "step": 3260
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.804945425680787e-06,
      "logits/chosen": -1.1110305786132812,
      "logits/rejected": -1.067275881767273,
      "logps/chosen": -423.29437255859375,
      "logps/rejected": -457.4410095214844,
      "loss": 0.5568,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.3485116958618164,
      "rewards/margins": 0.5422489047050476,
      "rewards/rejected": -2.8907601833343506,
      "step": 3270
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.802728299485225e-06,
      "logits/chosen": -0.9096890687942505,
      "logits/rejected": -0.8615697026252747,
      "logps/chosen": -394.5549011230469,
      "logps/rejected": -478.35870361328125,
      "loss": 0.6697,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.439504384994507,
      "rewards/margins": 0.6227213144302368,
      "rewards/rejected": -3.0622260570526123,
      "step": 3280
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.8004991611308495e-06,
      "logits/chosen": -1.2388406991958618,
      "logits/rejected": -1.0739996433258057,
      "logps/chosen": -403.4649658203125,
      "logps/rejected": -469.5654296875,
      "loss": 0.498,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.6191120147705078,
      "rewards/margins": 0.8215069770812988,
      "rewards/rejected": -2.4406189918518066,
      "step": 3290
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.798258022245937e-06,
      "logits/chosen": -1.2814171314239502,
      "logits/rejected": -1.0564707517623901,
      "logps/chosen": -368.33807373046875,
      "logps/rejected": -409.11407470703125,
      "loss": 0.5336,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.5073158740997314,
      "rewards/margins": 0.7924111485481262,
      "rewards/rejected": -2.299726963043213,
      "step": 3300
    },
    {
      "epoch": 0.22,
      "eval_logits/chosen": -1.2265251874923706,
      "eval_logits/rejected": -1.0916633605957031,
      "eval_logps/chosen": -364.0115051269531,
      "eval_logps/rejected": -422.55963134765625,
      "eval_loss": 0.521187961101532,
      "eval_rewards/accuracies": 0.6514999866485596,
      "eval_rewards/chosen": -1.3200657367706299,
      "eval_rewards/margins": 0.7894111275672913,
      "eval_rewards/rejected": -2.1094770431518555,
      "eval_runtime": 730.6664,
      "eval_samples_per_second": 2.737,
      "eval_steps_per_second": 1.369,
      "step": 3300
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.796004894521365e-06,
      "logits/chosen": -1.304469347000122,
      "logits/rejected": -1.0183453559875488,
      "logps/chosen": -370.98699951171875,
      "logps/rejected": -483.6045837402344,
      "loss": 0.5777,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.424172282218933,
      "rewards/margins": 0.9136108160018921,
      "rewards/rejected": -2.337782859802246,
      "step": 3310
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7937397897105545e-06,
      "logits/chosen": -1.218902587890625,
      "logits/rejected": -1.1205322742462158,
      "logps/chosen": -343.5941467285156,
      "logps/rejected": -372.71905517578125,
      "loss": 0.5801,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.383612871170044,
      "rewards/margins": 0.5433837175369263,
      "rewards/rejected": -1.9269965887069702,
      "step": 3320
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.791462719629399e-06,
      "logits/chosen": -1.0824165344238281,
      "logits/rejected": -1.012694239616394,
      "logps/chosen": -321.79803466796875,
      "logps/rejected": -382.03997802734375,
      "loss": 0.5033,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3607285022735596,
      "rewards/margins": 0.8265340924263,
      "rewards/rejected": -2.187262773513794,
      "step": 3330
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.789173696156212e-06,
      "logits/chosen": -1.1417255401611328,
      "logits/rejected": -0.8498822450637817,
      "logps/chosen": -445.33349609375,
      "logps/rejected": -552.0372314453125,
      "loss": 0.3828,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.6889288425445557,
      "rewards/margins": 1.2535320520401,
      "rewards/rejected": -2.942460536956787,
      "step": 3340
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.786872731231662e-06,
      "logits/chosen": -1.1248489618301392,
      "logits/rejected": -1.0287617444992065,
      "logps/chosen": -451.65765380859375,
      "logps/rejected": -502.49786376953125,
      "loss": 0.6218,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.3476672172546387,
      "rewards/margins": 0.6220924854278564,
      "rewards/rejected": -2.969759464263916,
      "step": 3350
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.784559836858709e-06,
      "logits/chosen": -0.9597482681274414,
      "logits/rejected": -0.7844263911247253,
      "logps/chosen": -422.6121520996094,
      "logps/rejected": -501.7942810058594,
      "loss": 0.4192,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8869502544403076,
      "rewards/margins": 1.0960309505462646,
      "rewards/rejected": -2.9829812049865723,
      "step": 3360
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.782235025102542e-06,
      "logits/chosen": -0.9880424737930298,
      "logits/rejected": -0.9343817830085754,
      "logps/chosen": -464.04107666015625,
      "logps/rejected": -534.37109375,
      "loss": 0.5225,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.3772170543670654,
      "rewards/margins": 0.8781053423881531,
      "rewards/rejected": -3.2553226947784424,
      "step": 3370
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.779898308090519e-06,
      "logits/chosen": -1.0745431184768677,
      "logits/rejected": -0.8649323582649231,
      "logps/chosen": -502.0604553222656,
      "logps/rejected": -583.8115234375,
      "loss": 0.4702,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.273831844329834,
      "rewards/margins": 1.1858172416687012,
      "rewards/rejected": -3.4596495628356934,
      "step": 3380
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.777549698012101e-06,
      "logits/chosen": -0.9469674229621887,
      "logits/rejected": -0.8883152008056641,
      "logps/chosen": -492.7947692871094,
      "logps/rejected": -608.96044921875,
      "loss": 0.4002,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.495781421661377,
      "rewards/margins": 1.3532531261444092,
      "rewards/rejected": -3.849034547805786,
      "step": 3390
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.775189207118787e-06,
      "logits/chosen": -0.8769720792770386,
      "logits/rejected": -0.8017680048942566,
      "logps/chosen": -508.2986755371094,
      "logps/rejected": -565.0167236328125,
      "loss": 0.5781,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.36480975151062,
      "rewards/margins": 0.7905575633049011,
      "rewards/rejected": -3.155367374420166,
      "step": 3400
    },
    {
      "epoch": 0.22,
      "eval_logits/chosen": -1.0673307180404663,
      "eval_logits/rejected": -0.9397266507148743,
      "eval_logps/chosen": -407.0195617675781,
      "eval_logps/rejected": -473.85296630859375,
      "eval_loss": 0.517623782157898,
      "eval_rewards/accuracies": 0.6575000286102295,
      "eval_rewards/chosen": -1.7501461505889893,
      "eval_rewards/margins": 0.8722642064094543,
      "eval_rewards/rejected": -2.6224100589752197,
      "eval_runtime": 731.9001,
      "eval_samples_per_second": 2.733,
      "eval_steps_per_second": 1.366,
      "step": 3400
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.772816847724054e-06,
      "logits/chosen": -1.276422381401062,
      "logits/rejected": -1.2553038597106934,
      "logps/chosen": -358.34124755859375,
      "logps/rejected": -426.73919677734375,
      "loss": 0.6499,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.3578667640686035,
      "rewards/margins": 0.6890023946762085,
      "rewards/rejected": -2.0468690395355225,
      "step": 3410
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.770432632203294e-06,
      "logits/chosen": -1.2462594509124756,
      "logits/rejected": -1.1783959865570068,
      "logps/chosen": -340.1909484863281,
      "logps/rejected": -344.7999572753906,
      "loss": 0.5608,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9654380679130554,
      "rewards/margins": 0.5492728352546692,
      "rewards/rejected": -1.5147110223770142,
      "step": 3420
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.768036572993738e-06,
      "logits/chosen": -1.4957689046859741,
      "logits/rejected": -1.4557548761367798,
      "logps/chosen": -371.65008544921875,
      "logps/rejected": -415.08367919921875,
      "loss": 0.5477,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9163110852241516,
      "rewards/margins": 0.5903352499008179,
      "rewards/rejected": -1.5066463947296143,
      "step": 3430
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.765628682594409e-06,
      "logits/chosen": -1.5271755456924438,
      "logits/rejected": -1.4071924686431885,
      "logps/chosen": -328.58270263671875,
      "logps/rejected": -370.56390380859375,
      "loss": 0.5162,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8482564091682434,
      "rewards/margins": 0.6588364839553833,
      "rewards/rejected": -1.5070927143096924,
      "step": 3440
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.763208973566041e-06,
      "logits/chosen": -1.331571340560913,
      "logits/rejected": -1.2520661354064941,
      "logps/chosen": -322.298828125,
      "logps/rejected": -401.99053955078125,
      "loss": 0.5404,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.3816914558410645,
      "rewards/margins": 0.6097648739814758,
      "rewards/rejected": -1.9914562702178955,
      "step": 3450
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.76077745853102e-06,
      "logits/chosen": -1.48663330078125,
      "logits/rejected": -1.4448163509368896,
      "logps/chosen": -415.95184326171875,
      "logps/rejected": -490.90899658203125,
      "loss": 0.5291,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6589769124984741,
      "rewards/margins": 0.7383204698562622,
      "rewards/rejected": -2.3972973823547363,
      "step": 3460
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.758334150173322e-06,
      "logits/chosen": -1.3083218336105347,
      "logits/rejected": -1.2009087800979614,
      "logps/chosen": -372.3233337402344,
      "logps/rejected": -440.8482360839844,
      "loss": 0.4207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0923935174942017,
      "rewards/margins": 0.9357854723930359,
      "rewards/rejected": -2.0281789302825928,
      "step": 3470
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.755879061238439e-06,
      "logits/chosen": -1.3399627208709717,
      "logits/rejected": -1.1547447443008423,
      "logps/chosen": -425.87567138671875,
      "logps/rejected": -493.44903564453125,
      "loss": 0.5857,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6970230340957642,
      "rewards/margins": 0.8021049499511719,
      "rewards/rejected": -2.4991281032562256,
      "step": 3480
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.753412204533317e-06,
      "logits/chosen": -1.462085485458374,
      "logits/rejected": -0.9828954935073853,
      "logps/chosen": -426.71044921875,
      "logps/rejected": -479.6546936035156,
      "loss": 0.514,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6385246515274048,
      "rewards/margins": 0.9833550453186035,
      "rewards/rejected": -2.6218795776367188,
      "step": 3490
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.750933592926292e-06,
      "logits/chosen": -1.3087482452392578,
      "logits/rejected": -0.9995695352554321,
      "logps/chosen": -407.4541320800781,
      "logps/rejected": -479.6214294433594,
      "loss": 0.4228,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.887264609336853,
      "rewards/margins": 0.9938368797302246,
      "rewards/rejected": -2.881101131439209,
      "step": 3500
    },
    {
      "epoch": 0.23,
      "eval_logits/chosen": -1.1501306295394897,
      "eval_logits/rejected": -1.0210661888122559,
      "eval_logps/chosen": -404.4117736816406,
      "eval_logps/rejected": -466.7912902832031,
      "eval_loss": 0.5153305530548096,
      "eval_rewards/accuracies": 0.6589999794960022,
      "eval_rewards/chosen": -1.7240681648254395,
      "eval_rewards/margins": 0.8277252912521362,
      "eval_rewards/rejected": -2.5517935752868652,
      "eval_runtime": 734.8426,
      "eval_samples_per_second": 2.722,
      "eval_steps_per_second": 1.361,
      "step": 3500
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7484432393470124e-06,
      "logits/chosen": -1.3669573068618774,
      "logits/rejected": -0.9049302339553833,
      "logps/chosen": -343.1621398925781,
      "logps/rejected": -392.91400146484375,
      "loss": 0.4902,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.410801649093628,
      "rewards/margins": 1.0514872074127197,
      "rewards/rejected": -2.4622886180877686,
      "step": 3510
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.745941156786385e-06,
      "logits/chosen": -0.8993209004402161,
      "logits/rejected": -0.9887765049934387,
      "logps/chosen": -308.3384094238281,
      "logps/rejected": -441.26788330078125,
      "loss": 0.466,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.552992820739746,
      "rewards/margins": 1.0208826065063477,
      "rewards/rejected": -2.573875665664673,
      "step": 3520
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.743427358296497e-06,
      "logits/chosen": -1.115281343460083,
      "logits/rejected": -0.955095112323761,
      "logps/chosen": -328.03436279296875,
      "logps/rejected": -452.24737548828125,
      "loss": 0.4754,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.38326096534729,
      "rewards/margins": 1.1290562152862549,
      "rewards/rejected": -2.512317419052124,
      "step": 3530
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.740901856990553e-06,
      "logits/chosen": -0.9862316846847534,
      "logits/rejected": -0.836779773235321,
      "logps/chosen": -399.65155029296875,
      "logps/rejected": -411.66839599609375,
      "loss": 0.6233,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.429596185684204,
      "rewards/margins": 0.5608941316604614,
      "rewards/rejected": -1.990490198135376,
      "step": 3540
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.738364666042804e-06,
      "logits/chosen": -1.303762435913086,
      "logits/rejected": -0.9813443422317505,
      "logps/chosen": -376.2354736328125,
      "logps/rejected": -400.13238525390625,
      "loss": 0.4688,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.844904899597168,
      "rewards/margins": 0.875367283821106,
      "rewards/rejected": -1.7202720642089844,
      "step": 3550
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.735815798688483e-06,
      "logits/chosen": -1.18997061252594,
      "logits/rejected": -0.980979323387146,
      "logps/chosen": -313.696533203125,
      "logps/rejected": -416.5016174316406,
      "loss": 0.5036,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.153645396232605,
      "rewards/margins": 0.8266595005989075,
      "rewards/rejected": -1.9803049564361572,
      "step": 3560
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7332552682237285e-06,
      "logits/chosen": -1.060894250869751,
      "logits/rejected": -0.7684231996536255,
      "logps/chosen": -291.71331787109375,
      "logps/rejected": -369.025390625,
      "loss": 0.4658,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1795337200164795,
      "rewards/margins": 0.9308455586433411,
      "rewards/rejected": -2.1103789806365967,
      "step": 3570
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7306830880055234e-06,
      "logits/chosen": -1.1081796884536743,
      "logits/rejected": -1.0646052360534668,
      "logps/chosen": -383.7330627441406,
      "logps/rejected": -464.07659912109375,
      "loss": 0.5619,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.9299920797348022,
      "rewards/margins": 0.7200648784637451,
      "rewards/rejected": -2.650057077407837,
      "step": 3580
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.728099271451619e-06,
      "logits/chosen": -1.0864530801773071,
      "logits/rejected": -1.1386919021606445,
      "logps/chosen": -353.6946716308594,
      "logps/rejected": -416.392822265625,
      "loss": 0.5043,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.626866102218628,
      "rewards/margins": 0.6965071558952332,
      "rewards/rejected": -2.323373317718506,
      "step": 3590
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.725503832040466e-06,
      "logits/chosen": -0.8184248208999634,
      "logits/rejected": -0.6163265705108643,
      "logps/chosen": -312.90264892578125,
      "logps/rejected": -412.98785400390625,
      "loss": 0.5345,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6374473571777344,
      "rewards/margins": 0.7399784326553345,
      "rewards/rejected": -2.3774256706237793,
      "step": 3600
    },
    {
      "epoch": 0.24,
      "eval_logits/chosen": -0.8562494516372681,
      "eval_logits/rejected": -0.743854820728302,
      "eval_logps/chosen": -430.83062744140625,
      "eval_logps/rejected": -490.9767150878906,
      "eval_loss": 0.5146420001983643,
      "eval_rewards/accuracies": 0.6579999923706055,
      "eval_rewards/chosen": -1.9882564544677734,
      "eval_rewards/margins": 0.8053914904594421,
      "eval_rewards/rejected": -2.7936480045318604,
      "eval_runtime": 734.536,
      "eval_samples_per_second": 2.723,
      "eval_steps_per_second": 1.361,
      "step": 3600
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.722896783311152e-06,
      "logits/chosen": -0.9259394407272339,
      "logits/rejected": -0.7786766290664673,
      "logps/chosen": -487.031982421875,
      "logps/rejected": -614.58251953125,
      "loss": 0.6077,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.279829740524292,
      "rewards/margins": 0.764165997505188,
      "rewards/rejected": -3.0439953804016113,
      "step": 3610
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.720278138863318e-06,
      "logits/chosen": -0.9295158386230469,
      "logits/rejected": -0.9707782864570618,
      "logps/chosen": -387.5835266113281,
      "logps/rejected": -414.12506103515625,
      "loss": 0.6133,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.9746183156967163,
      "rewards/margins": 0.5834863781929016,
      "rewards/rejected": -2.5581045150756836,
      "step": 3620
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.717647912357095e-06,
      "logits/chosen": -1.2270703315734863,
      "logits/rejected": -1.2645610570907593,
      "logps/chosen": -470.46527099609375,
      "logps/rejected": -520.5504760742188,
      "loss": 0.6555,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.979880928993225,
      "rewards/margins": 0.3715277314186096,
      "rewards/rejected": -2.3514084815979004,
      "step": 3630
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.715006117513035e-06,
      "logits/chosen": -1.2985150814056396,
      "logits/rejected": -1.1814923286437988,
      "logps/chosen": -514.2528076171875,
      "logps/rejected": -529.6771240234375,
      "loss": 0.4864,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9012712240219116,
      "rewards/margins": 0.7060447931289673,
      "rewards/rejected": -2.6073157787323,
      "step": 3640
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.7123527681120326e-06,
      "logits/chosen": -1.1452562808990479,
      "logits/rejected": -0.959757924079895,
      "logps/chosen": -443.2506408691406,
      "logps/rejected": -499.8810119628906,
      "loss": 0.4578,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9607951641082764,
      "rewards/margins": 0.8478196859359741,
      "rewards/rejected": -2.80861496925354,
      "step": 3650
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.7096878779952594e-06,
      "logits/chosen": -1.3756132125854492,
      "logits/rejected": -1.319220781326294,
      "logps/chosen": -489.84930419921875,
      "logps/rejected": -562.1311645507812,
      "loss": 0.4599,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.144359588623047,
      "rewards/margins": 0.7486457824707031,
      "rewards/rejected": -2.89300537109375,
      "step": 3660
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.707011461064086e-06,
      "logits/chosen": -1.172327995300293,
      "logits/rejected": -0.9126516580581665,
      "logps/chosen": -487.898193359375,
      "logps/rejected": -538.8970947265625,
      "loss": 0.4521,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7903915643692017,
      "rewards/margins": 0.951895534992218,
      "rewards/rejected": -2.7422873973846436,
      "step": 3670
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.704323531280016e-06,
      "logits/chosen": -1.0632213354110718,
      "logits/rejected": -0.9930667877197266,
      "logps/chosen": -504.03582763671875,
      "logps/rejected": -516.4459228515625,
      "loss": 0.484,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7682430744171143,
      "rewards/margins": 0.9560562372207642,
      "rewards/rejected": -2.724299192428589,
      "step": 3680
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.701624102664606e-06,
      "logits/chosen": -1.2993719577789307,
      "logits/rejected": -1.1282496452331543,
      "logps/chosen": -481.0072326660156,
      "logps/rejected": -521.73876953125,
      "loss": 0.5127,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.198913097381592,
      "rewards/margins": 0.9483180046081543,
      "rewards/rejected": -3.147231101989746,
      "step": 3690
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.698913189299399e-06,
      "logits/chosen": -1.1892201900482178,
      "logits/rejected": -1.1501682996749878,
      "logps/chosen": -415.744873046875,
      "logps/rejected": -520.8284301757812,
      "loss": 0.6089,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.2959303855895996,
      "rewards/margins": 0.7257683277130127,
      "rewards/rejected": -3.0216987133026123,
      "step": 3700
    },
    {
      "epoch": 0.24,
      "eval_logits/chosen": -1.1420601606369019,
      "eval_logits/rejected": -1.0099586248397827,
      "eval_logps/chosen": -474.0902099609375,
      "eval_logps/rejected": -541.633056640625,
      "eval_loss": 0.5181785821914673,
      "eval_rewards/accuracies": 0.6504999995231628,
      "eval_rewards/chosen": -2.4208528995513916,
      "eval_rewards/margins": 0.8793590664863586,
      "eval_rewards/rejected": -3.3002119064331055,
      "eval_runtime": 729.6855,
      "eval_samples_per_second": 2.741,
      "eval_steps_per_second": 1.37,
      "step": 3700
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.696190805325847e-06,
      "logits/chosen": -1.2062838077545166,
      "logits/rejected": -1.0717065334320068,
      "logps/chosen": -442.6526794433594,
      "logps/rejected": -527.166015625,
      "loss": 0.4301,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.362738847732544,
      "rewards/margins": 1.1163194179534912,
      "rewards/rejected": -3.479058027267456,
      "step": 3710
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.693456964945239e-06,
      "logits/chosen": -1.227285385131836,
      "logits/rejected": -0.876573920249939,
      "logps/chosen": -574.7136840820312,
      "logps/rejected": -603.5750122070312,
      "loss": 0.3663,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.7660810947418213,
      "rewards/margins": 1.2995359897613525,
      "rewards/rejected": -4.065616607666016,
      "step": 3720
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.6907116824186245e-06,
      "logits/chosen": -1.271610975265503,
      "logits/rejected": -1.2354601621627808,
      "logps/chosen": -478.7208557128906,
      "logps/rejected": -537.8218994140625,
      "loss": 0.6851,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -2.513932704925537,
      "rewards/margins": 0.5929728746414185,
      "rewards/rejected": -3.106905937194824,
      "step": 3730
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.687954972066742e-06,
      "logits/chosen": -1.354651689529419,
      "logits/rejected": -1.2123476266860962,
      "logps/chosen": -435.0054626464844,
      "logps/rejected": -538.2466430664062,
      "loss": 0.4836,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.051891803741455,
      "rewards/margins": 1.241121768951416,
      "rewards/rejected": -3.293013334274292,
      "step": 3740
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.685186848269944e-06,
      "logits/chosen": -1.3648970127105713,
      "logits/rejected": -1.206362009048462,
      "logps/chosen": -378.1536560058594,
      "logps/rejected": -416.2786560058594,
      "loss": 0.588,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.6628129482269287,
      "rewards/margins": 0.7550392150878906,
      "rewards/rejected": -2.4178521633148193,
      "step": 3750
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.682407325468119e-06,
      "logits/chosen": -1.3208792209625244,
      "logits/rejected": -1.028571367263794,
      "logps/chosen": -410.6923828125,
      "logps/rejected": -480.15234375,
      "loss": 0.4415,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9327335357666016,
      "rewards/margins": 1.0308973789215088,
      "rewards/rejected": -2.9636306762695312,
      "step": 3760
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.67961641816062e-06,
      "logits/chosen": -1.1115623712539673,
      "logits/rejected": -0.9650887250900269,
      "logps/chosen": -551.06494140625,
      "logps/rejected": -606.406982421875,
      "loss": 0.5092,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.7526817321777344,
      "rewards/margins": 0.9667217135429382,
      "rewards/rejected": -3.7194037437438965,
      "step": 3770
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.676814140906188e-06,
      "logits/chosen": -0.6974936723709106,
      "logits/rejected": -0.6922891736030579,
      "logps/chosen": -617.8670654296875,
      "logps/rejected": -644.30029296875,
      "loss": 0.6464,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -3.783308506011963,
      "rewards/margins": 0.5703891515731812,
      "rewards/rejected": -4.353697776794434,
      "step": 3780
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.674000508322872e-06,
      "logits/chosen": -0.44656461477279663,
      "logits/rejected": -0.5972496867179871,
      "logps/chosen": -527.9207153320312,
      "logps/rejected": -598.678955078125,
      "loss": 0.5704,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -3.10385799407959,
      "rewards/margins": 0.6157468557357788,
      "rewards/rejected": -3.719604969024658,
      "step": 3790
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.671175535087959e-06,
      "logits/chosen": -0.6944919228553772,
      "logits/rejected": -0.7936210632324219,
      "logps/chosen": -692.2608642578125,
      "logps/rejected": -807.3065185546875,
      "loss": 0.4123,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -4.009331703186035,
      "rewards/margins": 1.1827731132507324,
      "rewards/rejected": -5.192105293273926,
      "step": 3800
    },
    {
      "epoch": 0.25,
      "eval_logits/chosen": -0.6038689613342285,
      "eval_logits/rejected": -0.5056295990943909,
      "eval_logps/chosen": -590.8090209960938,
      "eval_logps/rejected": -636.2662353515625,
      "eval_loss": 0.5433777570724487,
      "eval_rewards/accuracies": 0.6359999775886536,
      "eval_rewards/chosen": -3.588040828704834,
      "eval_rewards/margins": 0.6585022211074829,
      "eval_rewards/rejected": -4.246542930603027,
      "eval_runtime": 732.5781,
      "eval_samples_per_second": 2.73,
      "eval_steps_per_second": 1.365,
      "step": 3800
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6683392359378924e-06,
      "logits/chosen": -0.5370118021965027,
      "logits/rejected": -0.40376004576683044,
      "logps/chosen": -573.9254150390625,
      "logps/rejected": -630.9413452148438,
      "loss": 0.5425,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -3.3877639770507812,
      "rewards/margins": 0.825324535369873,
      "rewards/rejected": -4.213088512420654,
      "step": 3810
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.665491625668198e-06,
      "logits/chosen": -0.3310016691684723,
      "logits/rejected": -0.38532063364982605,
      "logps/chosen": -552.151611328125,
      "logps/rejected": -631.1104736328125,
      "loss": 0.6136,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.95904278755188,
      "rewards/margins": 0.5540381669998169,
      "rewards/rejected": -4.5130815505981445,
      "step": 3820
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.662632719133407e-06,
      "logits/chosen": -0.5547534823417664,
      "logits/rejected": -0.3702434301376343,
      "logps/chosen": -486.48529052734375,
      "logps/rejected": -477.41943359375,
      "loss": 0.5439,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.556877374649048,
      "rewards/margins": 0.5794159770011902,
      "rewards/rejected": -3.136293411254883,
      "step": 3830
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.659762531246974e-06,
      "logits/chosen": -0.3453252911567688,
      "logits/rejected": -0.4048599302768707,
      "logps/chosen": -570.9554443359375,
      "logps/rejected": -577.3557739257812,
      "loss": 0.6338,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -3.563767194747925,
      "rewards/margins": 0.4035702645778656,
      "rewards/rejected": -3.9673373699188232,
      "step": 3840
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.656881076981207e-06,
      "logits/chosen": -0.7550775408744812,
      "logits/rejected": -0.686637282371521,
      "logps/chosen": -476.0919494628906,
      "logps/rejected": -483.2123107910156,
      "loss": 0.6476,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.6290855407714844,
      "rewards/margins": 0.250190794467926,
      "rewards/rejected": -2.8792760372161865,
      "step": 3850
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.653988371367183e-06,
      "logits/chosen": -0.7826381921768188,
      "logits/rejected": -0.47376060485839844,
      "logps/chosen": -506.4967346191406,
      "logps/rejected": -497.3008728027344,
      "loss": 0.5432,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.6569275856018066,
      "rewards/margins": 0.5201212167739868,
      "rewards/rejected": -3.177048444747925,
      "step": 3860
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.651084429494671e-06,
      "logits/chosen": -0.9120081663131714,
      "logits/rejected": -0.6307070255279541,
      "logps/chosen": -532.2633056640625,
      "logps/rejected": -535.4483642578125,
      "loss": 0.4548,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.569122791290283,
      "rewards/margins": 0.8270888328552246,
      "rewards/rejected": -3.3962111473083496,
      "step": 3870
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.648169266512053e-06,
      "logits/chosen": -1.1297684907913208,
      "logits/rejected": -0.8716074824333191,
      "logps/chosen": -457.02001953125,
      "logps/rejected": -477.3675231933594,
      "loss": 0.5569,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.3169193267822266,
      "rewards/margins": 0.653015673160553,
      "rewards/rejected": -2.969935178756714,
      "step": 3880
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6452428976262505e-06,
      "logits/chosen": -0.7666735053062439,
      "logits/rejected": -0.477083683013916,
      "logps/chosen": -437.20770263671875,
      "logps/rejected": -507.29058837890625,
      "loss": 0.3827,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.3341622352600098,
      "rewards/margins": 1.1440565586090088,
      "rewards/rejected": -3.4782187938690186,
      "step": 3890
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.642305338102633e-06,
      "logits/chosen": -0.6205832362174988,
      "logits/rejected": -0.8550397157669067,
      "logps/chosen": -445.5555725097656,
      "logps/rejected": -522.7137451171875,
      "loss": 0.6359,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.838059902191162,
      "rewards/margins": 0.5901249647140503,
      "rewards/rejected": -3.428185224533081,
      "step": 3900
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -0.794431209564209,
      "eval_logits/rejected": -0.6801789402961731,
      "eval_logps/chosen": -498.5151672363281,
      "eval_logps/rejected": -564.9202880859375,
      "eval_loss": 0.52691650390625,
      "eval_rewards/accuracies": 0.640999972820282,
      "eval_rewards/chosen": -2.665102243423462,
      "eval_rewards/margins": 0.8679810762405396,
      "eval_rewards/rejected": -3.533083438873291,
      "eval_runtime": 734.9734,
      "eval_samples_per_second": 2.721,
      "eval_steps_per_second": 1.361,
      "step": 3900
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.639356603264953e-06,
      "logits/chosen": -0.9367401003837585,
      "logits/rejected": -0.9064401388168335,
      "logps/chosen": -460.2066345214844,
      "logps/rejected": -513.5260009765625,
      "loss": 0.5192,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.183748483657837,
      "rewards/margins": 0.7905017137527466,
      "rewards/rejected": -2.974250078201294,
      "step": 3910
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.636396708495255e-06,
      "logits/chosen": -0.751909613609314,
      "logits/rejected": -0.7050498127937317,
      "logps/chosen": -429.04901123046875,
      "logps/rejected": -495.9883728027344,
      "loss": 0.4816,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9985395669937134,
      "rewards/margins": 0.9178323745727539,
      "rewards/rejected": -2.916372060775757,
      "step": 3920
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.633425669233799e-06,
      "logits/chosen": -1.1628475189208984,
      "logits/rejected": -1.059190034866333,
      "logps/chosen": -467.9149475097656,
      "logps/rejected": -539.2251586914062,
      "loss": 0.5287,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.330883264541626,
      "rewards/margins": 0.7575754523277283,
      "rewards/rejected": -3.08845853805542,
      "step": 3930
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6304435009789825e-06,
      "logits/chosen": -1.1164617538452148,
      "logits/rejected": -0.8136693835258484,
      "logps/chosen": -449.53515625,
      "logps/rejected": -475.35546875,
      "loss": 0.4778,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.12027907371521,
      "rewards/margins": 0.9673018455505371,
      "rewards/rejected": -3.087581157684326,
      "step": 3940
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.627450219287256e-06,
      "logits/chosen": -1.1186714172363281,
      "logits/rejected": -1.0646129846572876,
      "logps/chosen": -405.68780517578125,
      "logps/rejected": -469.1590270996094,
      "loss": 0.5183,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.2412867546081543,
      "rewards/margins": 0.8598229289054871,
      "rewards/rejected": -3.101109743118286,
      "step": 3950
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.624445839773042e-06,
      "logits/chosen": -0.9542514085769653,
      "logits/rejected": -0.9420453906059265,
      "logps/chosen": -403.17547607421875,
      "logps/rejected": -441.06231689453125,
      "loss": 0.6339,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.331641674041748,
      "rewards/margins": 0.38950642943382263,
      "rewards/rejected": -2.7211480140686035,
      "step": 3960
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.621430378108656e-06,
      "logits/chosen": -1.0336421728134155,
      "logits/rejected": -0.821884036064148,
      "logps/chosen": -553.2286987304688,
      "logps/rejected": -666.0933837890625,
      "loss": 0.4842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.9547042846679688,
      "rewards/margins": 1.1962896585464478,
      "rewards/rejected": -4.150994300842285,
      "step": 3970
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.618403850024223e-06,
      "logits/chosen": -0.7831366658210754,
      "logits/rejected": -0.7036348581314087,
      "logps/chosen": -515.8258056640625,
      "logps/rejected": -521.7974853515625,
      "loss": 0.6226,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.603356122970581,
      "rewards/margins": 0.5263395309448242,
      "rewards/rejected": -3.129695415496826,
      "step": 3980
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.615366271307598e-06,
      "logits/chosen": -1.0155903100967407,
      "logits/rejected": -0.8377137184143066,
      "logps/chosen": -435.54388427734375,
      "logps/rejected": -492.73297119140625,
      "loss": 0.5152,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.436009168624878,
      "rewards/margins": 0.6908141374588013,
      "rewards/rejected": -3.126823663711548,
      "step": 3990
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.612317657804277e-06,
      "logits/chosen": -1.012681007385254,
      "logits/rejected": -0.9964278340339661,
      "logps/chosen": -388.09417724609375,
      "logps/rejected": -510.4710388183594,
      "loss": 0.5634,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.408611536026001,
      "rewards/margins": 0.7102895975112915,
      "rewards/rejected": -3.118901014328003,
      "step": 4000
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -1.0345293283462524,
      "eval_logits/rejected": -0.9063413143157959,
      "eval_logps/chosen": -468.72064208984375,
      "eval_logps/rejected": -528.831298828125,
      "eval_loss": 0.5224303007125854,
      "eval_rewards/accuracies": 0.6514999866485596,
      "eval_rewards/chosen": -2.367156744003296,
      "eval_rewards/margins": 0.8050370812416077,
      "eval_rewards/rejected": -3.172193765640259,
      "eval_runtime": 735.7061,
      "eval_samples_per_second": 2.718,
      "eval_steps_per_second": 1.359,
      "step": 4000
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6092580254173236e-06,
      "logits/chosen": -0.9374804496765137,
      "logits/rejected": -0.8715485334396362,
      "logps/chosen": -493.97869873046875,
      "logps/rejected": -579.1528930664062,
      "loss": 0.4506,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.3665592670440674,
      "rewards/margins": 1.0384232997894287,
      "rewards/rejected": -3.404982805252075,
      "step": 4010
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.606187390107277e-06,
      "logits/chosen": -0.930155873298645,
      "logits/rejected": -0.8629436492919922,
      "logps/chosen": -480.35479736328125,
      "logps/rejected": -514.3657836914062,
      "loss": 0.5612,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.554793119430542,
      "rewards/margins": 0.7300376296043396,
      "rewards/rejected": -3.2848312854766846,
      "step": 4020
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.603105767892077e-06,
      "logits/chosen": -1.12532639503479,
      "logits/rejected": -1.0434709787368774,
      "logps/chosen": -453.9189453125,
      "logps/rejected": -564.5955200195312,
      "loss": 0.4922,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.5922884941101074,
      "rewards/margins": 0.9272336959838867,
      "rewards/rejected": -3.5195224285125732,
      "step": 4030
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6000131748469725e-06,
      "logits/chosen": -1.0651403665542603,
      "logits/rejected": -0.9068595170974731,
      "logps/chosen": -488.0533752441406,
      "logps/rejected": -489.08306884765625,
      "loss": 0.557,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.3791310787200928,
      "rewards/margins": 0.7534803748130798,
      "rewards/rejected": -3.1326117515563965,
      "step": 4040
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.596909627104445e-06,
      "logits/chosen": -1.3376210927963257,
      "logits/rejected": -1.189888596534729,
      "logps/chosen": -528.3531494140625,
      "logps/rejected": -602.3948974609375,
      "loss": 0.5493,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.7961485385894775,
      "rewards/margins": 1.0667587518692017,
      "rewards/rejected": -3.8629074096679688,
      "step": 4050
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5937951408541215e-06,
      "logits/chosen": -1.3444876670837402,
      "logits/rejected": -0.9238430857658386,
      "logps/chosen": -442.65191650390625,
      "logps/rejected": -513.8806762695312,
      "loss": 0.4464,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9001344442367554,
      "rewards/margins": 1.1660583019256592,
      "rewards/rejected": -3.066192865371704,
      "step": 4060
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.590669732342685e-06,
      "logits/chosen": -1.1866027116775513,
      "logits/rejected": -1.0306710004806519,
      "logps/chosen": -351.868896484375,
      "logps/rejected": -462.2666015625,
      "loss": 0.4385,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3972125053405762,
      "rewards/margins": 1.085050344467163,
      "rewards/rejected": -2.4822630882263184,
      "step": 4070
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.587533417873799e-06,
      "logits/chosen": -1.2526047229766846,
      "logits/rejected": -1.2084218263626099,
      "logps/chosen": -376.30963134765625,
      "logps/rejected": -536.0862426757812,
      "loss": 0.508,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.831455945968628,
      "rewards/margins": 1.0007293224334717,
      "rewards/rejected": -2.832185745239258,
      "step": 4080
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.584386213808016e-06,
      "logits/chosen": -1.204489827156067,
      "logits/rejected": -1.0284204483032227,
      "logps/chosen": -381.1136779785156,
      "logps/rejected": -417.54241943359375,
      "loss": 0.4861,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.594346523284912,
      "rewards/margins": 0.8220494985580444,
      "rewards/rejected": -2.416395902633667,
      "step": 4090
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.581228136562693e-06,
      "logits/chosen": -1.2890905141830444,
      "logits/rejected": -1.2788188457489014,
      "logps/chosen": -375.24700927734375,
      "logps/rejected": -383.44696044921875,
      "loss": 0.7537,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.370665192604065,
      "rewards/margins": 0.33884650468826294,
      "rewards/rejected": -1.7095117568969727,
      "step": 4100
    },
    {
      "epoch": 0.27,
      "eval_logits/chosen": -1.455415964126587,
      "eval_logits/rejected": -1.3053381443023682,
      "eval_logps/chosen": -354.7429504394531,
      "eval_logps/rejected": -415.7259826660156,
      "eval_loss": 0.5228714346885681,
      "eval_rewards/accuracies": 0.6524999737739563,
      "eval_rewards/chosen": -1.2273799180984497,
      "eval_rewards/margins": 0.8137608766555786,
      "eval_rewards/rejected": -2.041140556335449,
      "eval_runtime": 733.1087,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 4100
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.578059202611909e-06,
      "logits/chosen": -1.5341708660125732,
      "logits/rejected": -1.3989231586456299,
      "logps/chosen": -353.5614013671875,
      "logps/rejected": -389.3527526855469,
      "loss": 0.5833,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9640250205993652,
      "rewards/margins": 0.5054730772972107,
      "rewards/rejected": -1.4694980382919312,
      "step": 4110
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.574879428486376e-06,
      "logits/chosen": -1.5479503870010376,
      "logits/rejected": -1.4209856986999512,
      "logps/chosen": -326.52545166015625,
      "logps/rejected": -405.92315673828125,
      "loss": 0.4775,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.1360893249511719,
      "rewards/margins": 0.8231987953186035,
      "rewards/rejected": -1.9592880010604858,
      "step": 4120
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.571688830773352e-06,
      "logits/chosen": -1.571954607963562,
      "logits/rejected": -1.4492496252059937,
      "logps/chosen": -316.35675048828125,
      "logps/rejected": -350.2168884277344,
      "loss": 0.5704,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9428855776786804,
      "rewards/margins": 0.5373916625976562,
      "rewards/rejected": -1.4802772998809814,
      "step": 4130
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.568487426116559e-06,
      "logits/chosen": -1.3130545616149902,
      "logits/rejected": -1.2582907676696777,
      "logps/chosen": -284.15936279296875,
      "logps/rejected": -345.6742248535156,
      "loss": 0.495,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1218750476837158,
      "rewards/margins": 0.6904827356338501,
      "rewards/rejected": -1.8123579025268555,
      "step": 4140
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.565275231216092e-06,
      "logits/chosen": -0.9784256815910339,
      "logits/rejected": -0.9782289266586304,
      "logps/chosen": -308.6110534667969,
      "logps/rejected": -411.9737243652344,
      "loss": 0.5705,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.5726760625839233,
      "rewards/margins": 0.5737274289131165,
      "rewards/rejected": -2.1464033126831055,
      "step": 4150
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.562052262828331e-06,
      "logits/chosen": -1.1103951930999756,
      "logits/rejected": -1.0181940793991089,
      "logps/chosen": -407.5032958984375,
      "logps/rejected": -475.49053955078125,
      "loss": 0.5093,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.0869228839874268,
      "rewards/margins": 0.7326322793960571,
      "rewards/rejected": -2.819554567337036,
      "step": 4160
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.558818537765861e-06,
      "logits/chosen": -1.2901301383972168,
      "logits/rejected": -0.8406227231025696,
      "logps/chosen": -489.4602966308594,
      "logps/rejected": -529.7099609375,
      "loss": 0.5104,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.534459114074707,
      "rewards/margins": 0.7511767148971558,
      "rewards/rejected": -3.2856357097625732,
      "step": 4170
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.555574072897374e-06,
      "logits/chosen": -1.0267504453659058,
      "logits/rejected": -1.101967215538025,
      "logps/chosen": -465.4263610839844,
      "logps/rejected": -522.2689819335938,
      "loss": 0.5988,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.634650230407715,
      "rewards/margins": 0.5911431312561035,
      "rewards/rejected": -3.2257933616638184,
      "step": 4180
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.552318885147589e-06,
      "logits/chosen": -1.147258996963501,
      "logits/rejected": -0.8654224276542664,
      "logps/chosen": -512.3572998046875,
      "logps/rejected": -526.5074462890625,
      "loss": 0.5374,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.714397430419922,
      "rewards/margins": 0.7353962659835815,
      "rewards/rejected": -3.449793577194214,
      "step": 4190
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.549052991497159e-06,
      "logits/chosen": -1.0763828754425049,
      "logits/rejected": -1.0327566862106323,
      "logps/chosen": -412.31451416015625,
      "logps/rejected": -481.2054748535156,
      "loss": 0.5164,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.3214733600616455,
      "rewards/margins": 0.6847215890884399,
      "rewards/rejected": -3.006194829940796,
      "step": 4200
    },
    {
      "epoch": 0.27,
      "eval_logits/chosen": -1.0663347244262695,
      "eval_logits/rejected": -0.9360623359680176,
      "eval_logps/chosen": -458.21832275390625,
      "eval_logps/rejected": -521.7139892578125,
      "eval_loss": 0.5160852074623108,
      "eval_rewards/accuracies": 0.6489999890327454,
      "eval_rewards/chosen": -2.2621333599090576,
      "eval_rewards/margins": 0.8388875722885132,
      "eval_rewards/rejected": -3.1010210514068604,
      "eval_runtime": 732.0631,
      "eval_samples_per_second": 2.732,
      "eval_steps_per_second": 1.366,
      "step": 4200
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.545776408982585e-06,
      "logits/chosen": -1.1501169204711914,
      "logits/rejected": -1.0862586498260498,
      "logps/chosen": -438.3699645996094,
      "logps/rejected": -511.65948486328125,
      "loss": 0.5717,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0662589073181152,
      "rewards/margins": 0.8230277895927429,
      "rewards/rejected": -2.889286756515503,
      "step": 4210
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.542489154696128e-06,
      "logits/chosen": -1.4092345237731934,
      "logits/rejected": -1.092861533164978,
      "logps/chosen": -429.67022705078125,
      "logps/rejected": -432.35528564453125,
      "loss": 0.5422,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6194988489151,
      "rewards/margins": 0.6393571496009827,
      "rewards/rejected": -2.2588562965393066,
      "step": 4220
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.5391912457857145e-06,
      "logits/chosen": -1.3653011322021484,
      "logits/rejected": -1.1926562786102295,
      "logps/chosen": -447.32830810546875,
      "logps/rejected": -471.3085021972656,
      "loss": 0.5818,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8235870599746704,
      "rewards/margins": 0.7108983993530273,
      "rewards/rejected": -2.534485101699829,
      "step": 4230
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.535882699454854e-06,
      "logits/chosen": -1.38186514377594,
      "logits/rejected": -1.286101222038269,
      "logps/chosen": -433.41796875,
      "logps/rejected": -538.211669921875,
      "loss": 0.5182,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6236693859100342,
      "rewards/margins": 0.8313106298446655,
      "rewards/rejected": -2.4549801349639893,
      "step": 4240
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.532563532962546e-06,
      "logits/chosen": -1.4803118705749512,
      "logits/rejected": -1.4818512201309204,
      "logps/chosen": -385.42938232421875,
      "logps/rejected": -472.23822021484375,
      "loss": 0.6099,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.9553512334823608,
      "rewards/margins": 0.6521388292312622,
      "rewards/rejected": -2.607490062713623,
      "step": 4250
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.529233763623187e-06,
      "logits/chosen": -1.283656358718872,
      "logits/rejected": -1.011871576309204,
      "logps/chosen": -374.8547058105469,
      "logps/rejected": -392.13287353515625,
      "loss": 0.5419,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7275416851043701,
      "rewards/margins": 0.6523784399032593,
      "rewards/rejected": -2.379920244216919,
      "step": 4260
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.5258934088064854e-06,
      "logits/chosen": -1.2089446783065796,
      "logits/rejected": -0.9043632745742798,
      "logps/chosen": -398.29693603515625,
      "logps/rejected": -429.73468017578125,
      "loss": 0.5448,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7831614017486572,
      "rewards/margins": 0.8613526225090027,
      "rewards/rejected": -2.6445140838623047,
      "step": 4270
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.522542485937369e-06,
      "logits/chosen": -1.2879055738449097,
      "logits/rejected": -1.0973680019378662,
      "logps/chosen": -433.8949279785156,
      "logps/rejected": -444.53936767578125,
      "loss": 0.3823,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4324241876602173,
      "rewards/margins": 1.0728528499603271,
      "rewards/rejected": -2.505277156829834,
      "step": 4280
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.519181012495892e-06,
      "logits/chosen": -1.4716758728027344,
      "logits/rejected": -1.2221190929412842,
      "logps/chosen": -359.55841064453125,
      "logps/rejected": -416.4119567871094,
      "loss": 0.5422,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.173391342163086,
      "rewards/margins": 0.8592764139175415,
      "rewards/rejected": -2.032667875289917,
      "step": 4290
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.515809006017147e-06,
      "logits/chosen": -1.356048583984375,
      "logits/rejected": -1.1803017854690552,
      "logps/chosen": -332.74713134765625,
      "logps/rejected": -356.1148376464844,
      "loss": 0.6486,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9933754801750183,
      "rewards/margins": 0.5887696743011475,
      "rewards/rejected": -1.5821453332901,
      "step": 4300
    },
    {
      "epoch": 0.28,
      "eval_logits/chosen": -1.4796526432037354,
      "eval_logits/rejected": -1.3301215171813965,
      "eval_logps/chosen": -309.6466979980469,
      "eval_logps/rejected": -364.43499755859375,
      "eval_loss": 0.5246568322181702,
      "eval_rewards/accuracies": 0.6549999713897705,
      "eval_rewards/chosen": -0.7764175534248352,
      "eval_rewards/margins": 0.7518131732940674,
      "eval_rewards/rejected": -1.5282306671142578,
      "eval_runtime": 730.8262,
      "eval_samples_per_second": 2.737,
      "eval_steps_per_second": 1.368,
      "step": 4300
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.512426484091171e-06,
      "logits/chosen": -1.6821578741073608,
      "logits/rejected": -1.423906922340393,
      "logps/chosen": -345.78094482421875,
      "logps/rejected": -375.2535400390625,
      "loss": 0.5051,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6819978952407837,
      "rewards/margins": 0.657656192779541,
      "rewards/rejected": -1.3396542072296143,
      "step": 4310
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.509033464362858e-06,
      "logits/chosen": -1.397766351699829,
      "logits/rejected": -1.3964592218399048,
      "logps/chosen": -296.751220703125,
      "logps/rejected": -381.7581481933594,
      "loss": 0.559,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5548534393310547,
      "rewards/margins": 0.7077598571777344,
      "rewards/rejected": -1.262613296508789,
      "step": 4320
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.505629964531857e-06,
      "logits/chosen": -1.527769923210144,
      "logits/rejected": -1.3648316860198975,
      "logps/chosen": -304.9037170410156,
      "logps/rejected": -363.18463134765625,
      "loss": 0.4798,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8169935345649719,
      "rewards/margins": 0.8909414410591125,
      "rewards/rejected": -1.7079349756240845,
      "step": 4330
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.502216002352492e-06,
      "logits/chosen": -1.4147685766220093,
      "logits/rejected": -1.2035995721817017,
      "logps/chosen": -251.2743682861328,
      "logps/rejected": -303.8065490722656,
      "loss": 0.5584,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8823345899581909,
      "rewards/margins": 0.7194640040397644,
      "rewards/rejected": -1.6017987728118896,
      "step": 4340
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.498791595633663e-06,
      "logits/chosen": -1.1233395338058472,
      "logits/rejected": -0.8900758624076843,
      "logps/chosen": -366.68353271484375,
      "logps/rejected": -360.0097351074219,
      "loss": 0.5087,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.042992353439331,
      "rewards/margins": 0.8052667379379272,
      "rewards/rejected": -1.8482589721679688,
      "step": 4350
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.495356762238751e-06,
      "logits/chosen": -1.4454867839813232,
      "logits/rejected": -1.0401540994644165,
      "logps/chosen": -375.6465759277344,
      "logps/rejected": -380.46240234375,
      "loss": 0.4262,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9244867563247681,
      "rewards/margins": 1.0226781368255615,
      "rewards/rejected": -1.9471648931503296,
      "step": 4360
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.491911520085532e-06,
      "logits/chosen": -0.9478855133056641,
      "logits/rejected": -0.9101594090461731,
      "logps/chosen": -300.29791259765625,
      "logps/rejected": -386.93060302734375,
      "loss": 0.4718,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9989013671875,
      "rewards/margins": 0.7586010694503784,
      "rewards/rejected": -1.757502555847168,
      "step": 4370
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.488455887146075e-06,
      "logits/chosen": -1.1051117181777954,
      "logits/rejected": -0.9938226938247681,
      "logps/chosen": -313.7795104980469,
      "logps/rejected": -451.63238525390625,
      "loss": 0.4554,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4366739988327026,
      "rewards/margins": 1.2468922138214111,
      "rewards/rejected": -2.6835663318634033,
      "step": 4380
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.484989881446654e-06,
      "logits/chosen": -1.171975016593933,
      "logits/rejected": -1.1171002388000488,
      "logps/chosen": -373.6925964355469,
      "logps/rejected": -412.12152099609375,
      "loss": 0.5907,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7121661901474,
      "rewards/margins": 0.5637361407279968,
      "rewards/rejected": -2.275902271270752,
      "step": 4390
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.481513521067654e-06,
      "logits/chosen": -1.1522445678710938,
      "logits/rejected": -1.0051096677780151,
      "logps/chosen": -414.10693359375,
      "logps/rejected": -494.4396057128906,
      "loss": 0.4663,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.893323302268982,
      "rewards/margins": 1.1574846506118774,
      "rewards/rejected": -3.0508081912994385,
      "step": 4400
    },
    {
      "epoch": 0.29,
      "eval_logits/chosen": -1.0872026681900024,
      "eval_logits/rejected": -0.9511972069740295,
      "eval_logps/chosen": -398.82080078125,
      "eval_logps/rejected": -475.67913818359375,
      "eval_loss": 0.5215036273002625,
      "eval_rewards/accuracies": 0.6524999737739563,
      "eval_rewards/chosen": -1.6681584119796753,
      "eval_rewards/margins": 0.9725137948989868,
      "eval_rewards/rejected": -2.640672206878662,
      "eval_runtime": 733.2598,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 4400
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.478026824143473e-06,
      "logits/chosen": -1.1751139163970947,
      "logits/rejected": -1.1155116558074951,
      "logps/chosen": -443.25335693359375,
      "logps/rejected": -497.99713134765625,
      "loss": 0.5358,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7719179391860962,
      "rewards/margins": 1.1131114959716797,
      "rewards/rejected": -2.8850293159484863,
      "step": 4410
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.474529808862429e-06,
      "logits/chosen": -0.9981487393379211,
      "logits/rejected": -1.037527084350586,
      "logps/chosen": -364.75,
      "logps/rejected": -485.3653869628906,
      "loss": 0.4831,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7592971324920654,
      "rewards/margins": 1.0299116373062134,
      "rewards/rejected": -2.7892086505889893,
      "step": 4420
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.471022493466669e-06,
      "logits/chosen": -1.0802857875823975,
      "logits/rejected": -0.8232300877571106,
      "logps/chosen": -509.3580627441406,
      "logps/rejected": -502.99908447265625,
      "loss": 0.5391,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.0741891860961914,
      "rewards/margins": 0.7065232396125793,
      "rewards/rejected": -2.780712604522705,
      "step": 4430
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.467504896252066e-06,
      "logits/chosen": -1.2374316453933716,
      "logits/rejected": -1.1976972818374634,
      "logps/chosen": -458.05401611328125,
      "logps/rejected": -505.687744140625,
      "loss": 0.5669,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.083527088165283,
      "rewards/margins": 0.7637595534324646,
      "rewards/rejected": -2.8472867012023926,
      "step": 4440
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.463977035568132e-06,
      "logits/chosen": -0.9392198324203491,
      "logits/rejected": -1.0262138843536377,
      "logps/chosen": -443.80462646484375,
      "logps/rejected": -533.6358642578125,
      "loss": 0.6599,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.314218282699585,
      "rewards/margins": 0.3812568783760071,
      "rewards/rejected": -2.6954751014709473,
      "step": 4450
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.460438929817914e-06,
      "logits/chosen": -0.880610466003418,
      "logits/rejected": -0.651411235332489,
      "logps/chosen": -464.05908203125,
      "logps/rejected": -528.5072021484375,
      "loss": 0.5399,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.576392889022827,
      "rewards/margins": 0.6873677968978882,
      "rewards/rejected": -3.263760805130005,
      "step": 4460
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.456890597457907e-06,
      "logits/chosen": -0.8304548263549805,
      "logits/rejected": -0.7910754084587097,
      "logps/chosen": -479.0946350097656,
      "logps/rejected": -580.2449340820312,
      "loss": 0.4911,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.6695892810821533,
      "rewards/margins": 0.829459011554718,
      "rewards/rejected": -3.4990482330322266,
      "step": 4470
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.453332056997951e-06,
      "logits/chosen": -0.7744184732437134,
      "logits/rejected": -0.7223728895187378,
      "logps/chosen": -383.31866455078125,
      "logps/rejected": -492.0791931152344,
      "loss": 0.3992,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0335917472839355,
      "rewards/margins": 1.1308565139770508,
      "rewards/rejected": -3.1644484996795654,
      "step": 4480
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.449763327001134e-06,
      "logits/chosen": -0.9835674166679382,
      "logits/rejected": -1.0188270807266235,
      "logps/chosen": -452.4715270996094,
      "logps/rejected": -548.7013549804688,
      "loss": 0.6053,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.6370930671691895,
      "rewards/margins": 0.6781438589096069,
      "rewards/rejected": -3.315237045288086,
      "step": 4490
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.446184426083702e-06,
      "logits/chosen": -1.0631225109100342,
      "logits/rejected": -0.8816837072372437,
      "logps/chosen": -470.7005920410156,
      "logps/rejected": -597.3644409179688,
      "loss": 0.5322,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.771361827850342,
      "rewards/margins": 1.1829630136489868,
      "rewards/rejected": -3.9543251991271973,
      "step": 4500
    },
    {
      "epoch": 0.29,
      "eval_logits/chosen": -1.0830296277999878,
      "eval_logits/rejected": -0.9450811743736267,
      "eval_logps/chosen": -466.5963439941406,
      "eval_logps/rejected": -540.9030151367188,
      "eval_loss": 0.5166224837303162,
      "eval_rewards/accuracies": 0.6485000252723694,
      "eval_rewards/chosen": -2.345913887023926,
      "eval_rewards/margins": 0.9469969868659973,
      "eval_rewards/rejected": -3.292910575866699,
      "eval_runtime": 733.2167,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 4500
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.442595372914954e-06,
      "logits/chosen": -1.1284606456756592,
      "logits/rejected": -1.064886450767517,
      "logps/chosen": -433.7189025878906,
      "logps/rejected": -444.3348083496094,
      "loss": 0.4277,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9673858880996704,
      "rewards/margins": 0.9589015245437622,
      "rewards/rejected": -2.9262874126434326,
      "step": 4510
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.43899618621715e-06,
      "logits/chosen": -1.1971828937530518,
      "logits/rejected": -0.9432374238967896,
      "logps/chosen": -486.7794494628906,
      "logps/rejected": -599.4856567382812,
      "loss": 0.5682,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.3572559356689453,
      "rewards/margins": 1.0950950384140015,
      "rewards/rejected": -3.4523510932922363,
      "step": 4520
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.4353868847654105e-06,
      "logits/chosen": -1.4479076862335205,
      "logits/rejected": -1.1744765043258667,
      "logps/chosen": -417.41510009765625,
      "logps/rejected": -470.2010803222656,
      "loss": 0.5878,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7155687808990479,
      "rewards/margins": 0.8285356760025024,
      "rewards/rejected": -2.5441043376922607,
      "step": 4530
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.43176748738762e-06,
      "logits/chosen": -1.175443410873413,
      "logits/rejected": -1.074643850326538,
      "logps/chosen": -436.22784423828125,
      "logps/rejected": -551.2193603515625,
      "loss": 0.5606,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.0427985191345215,
      "rewards/margins": 1.1060316562652588,
      "rewards/rejected": -3.148829936981201,
      "step": 4540
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.4281380129643295e-06,
      "logits/chosen": -1.1706737279891968,
      "logits/rejected": -0.9897640347480774,
      "logps/chosen": -417.036865234375,
      "logps/rejected": -527.3284912109375,
      "loss": 0.5259,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.872867226600647,
      "rewards/margins": 1.2076570987701416,
      "rewards/rejected": -3.08052396774292,
      "step": 4550
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.424498480428654e-06,
      "logits/chosen": -1.2937458753585815,
      "logits/rejected": -1.1463016271591187,
      "logps/chosen": -421.75616455078125,
      "logps/rejected": -444.4380798339844,
      "loss": 0.6206,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.744842529296875,
      "rewards/margins": 0.627072811126709,
      "rewards/rejected": -2.371915340423584,
      "step": 4560
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.420848908766178e-06,
      "logits/chosen": -1.4309735298156738,
      "logits/rejected": -1.3544584512710571,
      "logps/chosen": -376.6891174316406,
      "logps/rejected": -450.12078857421875,
      "loss": 0.6033,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.6946220397949219,
      "rewards/margins": 0.6455153226852417,
      "rewards/rejected": -2.340137243270874,
      "step": 4570
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.417189317014855e-06,
      "logits/chosen": -1.3555351495742798,
      "logits/rejected": -1.5094717741012573,
      "logps/chosen": -344.8653564453125,
      "logps/rejected": -409.7660217285156,
      "loss": 0.7424,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.4406945705413818,
      "rewards/margins": 0.34203729033470154,
      "rewards/rejected": -1.7827320098876953,
      "step": 4580
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.41351972426491e-06,
      "logits/chosen": -1.2469980716705322,
      "logits/rejected": -1.2732794284820557,
      "logps/chosen": -381.6789245605469,
      "logps/rejected": -504.3853454589844,
      "loss": 0.4827,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3493956327438354,
      "rewards/margins": 0.6865336298942566,
      "rewards/rejected": -2.0359294414520264,
      "step": 4590
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.409840149658735e-06,
      "logits/chosen": -1.3501039743423462,
      "logits/rejected": -1.2072316408157349,
      "logps/chosen": -397.30950927734375,
      "logps/rejected": -402.70428466796875,
      "loss": 0.5485,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.127922534942627,
      "rewards/margins": 0.5554654002189636,
      "rewards/rejected": -1.6833879947662354,
      "step": 4600
    },
    {
      "epoch": 0.3,
      "eval_logits/chosen": -1.3868718147277832,
      "eval_logits/rejected": -1.245092749595642,
      "eval_logps/chosen": -361.0744323730469,
      "eval_logps/rejected": -399.0142822265625,
      "eval_loss": 0.5370956063270569,
      "eval_rewards/accuracies": 0.6510000228881836,
      "eval_rewards/chosen": -1.2906944751739502,
      "eval_rewards/margins": 0.5833296775817871,
      "eval_rewards/rejected": -1.8740240335464478,
      "eval_runtime": 733.0152,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 4600
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.4061506123907925e-06,
      "logits/chosen": -1.3194220066070557,
      "logits/rejected": -1.1732145547866821,
      "logps/chosen": -410.449462890625,
      "logps/rejected": -420.6064453125,
      "loss": 0.5794,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4635764360427856,
      "rewards/margins": 0.504631519317627,
      "rewards/rejected": -1.9682079553604126,
      "step": 4610
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.402451131707519e-06,
      "logits/chosen": -1.479317545890808,
      "logits/rejected": -1.1270101070404053,
      "logps/chosen": -348.9317321777344,
      "logps/rejected": -352.8699645996094,
      "loss": 0.4994,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.394663691520691,
      "rewards/margins": 0.794195830821991,
      "rewards/rejected": -2.188859462738037,
      "step": 4620
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.398741726907215e-06,
      "logits/chosen": -1.6055643558502197,
      "logits/rejected": -1.2944947481155396,
      "logps/chosen": -413.595947265625,
      "logps/rejected": -456.23095703125,
      "loss": 0.4513,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.343385934829712,
      "rewards/margins": 0.829616367816925,
      "rewards/rejected": -2.173002243041992,
      "step": 4630
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.395022417339955e-06,
      "logits/chosen": -1.2120579481124878,
      "logits/rejected": -1.1644296646118164,
      "logps/chosen": -390.01263427734375,
      "logps/rejected": -480.06280517578125,
      "loss": 0.5869,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8338760137557983,
      "rewards/margins": 0.8156105279922485,
      "rewards/rejected": -2.649486541748047,
      "step": 4640
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.391293222407479e-06,
      "logits/chosen": -1.1576309204101562,
      "logits/rejected": -1.2002023458480835,
      "logps/chosen": -268.47930908203125,
      "logps/rejected": -357.7801208496094,
      "loss": 0.5439,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.3192216157913208,
      "rewards/margins": 0.71070796251297,
      "rewards/rejected": -2.0299293994903564,
      "step": 4650
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.387554161563094e-06,
      "logits/chosen": -1.2241219282150269,
      "logits/rejected": -1.166908860206604,
      "logps/chosen": -400.4576721191406,
      "logps/rejected": -520.9844360351562,
      "loss": 0.4558,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.015934467315674,
      "rewards/margins": 1.3234636783599854,
      "rewards/rejected": -3.3393986225128174,
      "step": 4660
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.383805254311575e-06,
      "logits/chosen": -1.3214675188064575,
      "logits/rejected": -0.9820463061332703,
      "logps/chosen": -452.3216857910156,
      "logps/rejected": -490.5860290527344,
      "loss": 0.4993,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.961806058883667,
      "rewards/margins": 0.8383065462112427,
      "rewards/rejected": -2.80011248588562,
      "step": 4670
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.380046520209056e-06,
      "logits/chosen": -1.1236032247543335,
      "logits/rejected": -0.865643322467804,
      "logps/chosen": -391.35614013671875,
      "logps/rejected": -480.70025634765625,
      "loss": 0.4831,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.910897970199585,
      "rewards/margins": 1.1318318843841553,
      "rewards/rejected": -3.0427298545837402,
      "step": 4680
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.376277978862936e-06,
      "logits/chosen": -0.8476250767707825,
      "logits/rejected": -0.7707692384719849,
      "logps/chosen": -436.7411193847656,
      "logps/rejected": -474.6356506347656,
      "loss": 0.5271,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.1075243949890137,
      "rewards/margins": 0.7842603921890259,
      "rewards/rejected": -2.891785144805908,
      "step": 4690
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.372499649931774e-06,
      "logits/chosen": -1.027212142944336,
      "logits/rejected": -0.9219115972518921,
      "logps/chosen": -472.2177734375,
      "logps/rejected": -634.4398193359375,
      "loss": 0.4012,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.6312623023986816,
      "rewards/margins": 1.5426554679870605,
      "rewards/rejected": -4.173917770385742,
      "step": 4700
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": -0.963469386100769,
      "eval_logits/rejected": -0.8301882147789001,
      "eval_logps/chosen": -495.0129089355469,
      "eval_logps/rejected": -579.796142578125,
      "eval_loss": 0.518997848033905,
      "eval_rewards/accuracies": 0.6514999866485596,
      "eval_rewards/chosen": -2.630079984664917,
      "eval_rewards/margins": 1.0517627000808716,
      "eval_rewards/rejected": -3.68184232711792,
      "eval_runtime": 734.3909,
      "eval_samples_per_second": 2.723,
      "eval_steps_per_second": 1.362,
      "step": 4700
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.368711553125185e-06,
      "logits/chosen": -1.170462727546692,
      "logits/rejected": -1.1017426252365112,
      "logps/chosen": -520.1934814453125,
      "logps/rejected": -549.4090576171875,
      "loss": 0.5431,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.430034875869751,
      "rewards/margins": 0.876590371131897,
      "rewards/rejected": -3.3066253662109375,
      "step": 4710
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.364913708203734e-06,
      "logits/chosen": -1.2170623540878296,
      "logits/rejected": -1.0090713500976562,
      "logps/chosen": -513.0048217773438,
      "logps/rejected": -546.5401611328125,
      "loss": 0.4402,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.28108549118042,
      "rewards/margins": 1.0895252227783203,
      "rewards/rejected": -3.3706107139587402,
      "step": 4720
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.361106134978844e-06,
      "logits/chosen": -1.0908490419387817,
      "logits/rejected": -0.8982577323913574,
      "logps/chosen": -496.34521484375,
      "logps/rejected": -563.3717041015625,
      "loss": 0.5615,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.259147882461548,
      "rewards/margins": 0.7997722029685974,
      "rewards/rejected": -3.058920383453369,
      "step": 4730
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.357288853312681e-06,
      "logits/chosen": -1.200878381729126,
      "logits/rejected": -1.1836717128753662,
      "logps/chosen": -469.6875915527344,
      "logps/rejected": -538.845703125,
      "loss": 0.5054,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8620973825454712,
      "rewards/margins": 0.7389506101608276,
      "rewards/rejected": -2.601047992706299,
      "step": 4740
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353461883118056e-06,
      "logits/chosen": -1.1654294729232788,
      "logits/rejected": -1.0323411226272583,
      "logps/chosen": -443.4126892089844,
      "logps/rejected": -486.7259826660156,
      "loss": 0.5934,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.141221284866333,
      "rewards/margins": 0.6446442604064941,
      "rewards/rejected": -2.785865306854248,
      "step": 4750
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.34962524435832e-06,
      "logits/chosen": -1.076738953590393,
      "logits/rejected": -0.9777164459228516,
      "logps/chosen": -405.94659423828125,
      "logps/rejected": -497.431396484375,
      "loss": 0.4841,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.8626617193222046,
      "rewards/margins": 1.283182144165039,
      "rewards/rejected": -3.145843744277954,
      "step": 4760
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.34577895704726e-06,
      "logits/chosen": -1.4177812337875366,
      "logits/rejected": -1.2205036878585815,
      "logps/chosen": -461.11138916015625,
      "logps/rejected": -569.2667846679688,
      "loss": 0.4604,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9949737787246704,
      "rewards/margins": 1.3387939929962158,
      "rewards/rejected": -3.3337676525115967,
      "step": 4770
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3419230412489954e-06,
      "logits/chosen": -1.3925700187683105,
      "logits/rejected": -1.1407649517059326,
      "logps/chosen": -505.38604736328125,
      "logps/rejected": -519.0608520507812,
      "loss": 0.5027,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.160182237625122,
      "rewards/margins": 0.897386372089386,
      "rewards/rejected": -3.0575685501098633,
      "step": 4780
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.338057517077872e-06,
      "logits/chosen": -1.2423205375671387,
      "logits/rejected": -1.012940764427185,
      "logps/chosen": -383.737060546875,
      "logps/rejected": -526.2789306640625,
      "loss": 0.3283,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.896958589553833,
      "rewards/margins": 1.8589407205581665,
      "rewards/rejected": -3.755898952484131,
      "step": 4790
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.334182404698356e-06,
      "logits/chosen": -1.266410231590271,
      "logits/rejected": -0.9841598272323608,
      "logps/chosen": -423.17169189453125,
      "logps/rejected": -452.947265625,
      "loss": 0.4963,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9306468963623047,
      "rewards/margins": 1.0817534923553467,
      "rewards/rejected": -3.0124001502990723,
      "step": 4800
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": -1.153753638267517,
      "eval_logits/rejected": -1.0117337703704834,
      "eval_logps/chosen": -424.8492431640625,
      "eval_logps/rejected": -512.7779541015625,
      "eval_loss": 0.512640118598938,
      "eval_rewards/accuracies": 0.6539999842643738,
      "eval_rewards/chosen": -1.9284428358078003,
      "eval_rewards/margins": 1.0832173824310303,
      "eval_rewards/rejected": -3.01166033744812,
      "eval_runtime": 734.3129,
      "eval_samples_per_second": 2.724,
      "eval_steps_per_second": 1.362,
      "step": 4800
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.330297724324933e-06,
      "logits/chosen": -1.4888598918914795,
      "logits/rejected": -1.028956651687622,
      "logps/chosen": -497.24462890625,
      "logps/rejected": -483.7391052246094,
      "loss": 0.5424,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8883317708969116,
      "rewards/margins": 0.8769477605819702,
      "rewards/rejected": -2.765279531478882,
      "step": 4810
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.326403496221999e-06,
      "logits/chosen": -1.2531269788742065,
      "logits/rejected": -1.2295105457305908,
      "logps/chosen": -298.4060974121094,
      "logps/rejected": -327.56494140625,
      "loss": 0.6945,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3697706460952759,
      "rewards/margins": 0.5593878626823425,
      "rewards/rejected": -1.9291585683822632,
      "step": 4820
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.322499740703755e-06,
      "logits/chosen": -1.433562994003296,
      "logits/rejected": -1.5193655490875244,
      "logps/chosen": -271.50836181640625,
      "logps/rejected": -365.73895263671875,
      "loss": 0.4978,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7940917015075684,
      "rewards/margins": 0.6970726251602173,
      "rewards/rejected": -1.4911643266677856,
      "step": 4830
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.318586478134101e-06,
      "logits/chosen": -1.5995076894760132,
      "logits/rejected": -1.447758674621582,
      "logps/chosen": -257.02606201171875,
      "logps/rejected": -259.845703125,
      "loss": 0.6046,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.643585741519928,
      "rewards/margins": 0.4334011971950531,
      "rewards/rejected": -1.0769869089126587,
      "step": 4840
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.314663728926534e-06,
      "logits/chosen": -1.8331336975097656,
      "logits/rejected": -1.5674731731414795,
      "logps/chosen": -335.2432556152344,
      "logps/rejected": -360.13116455078125,
      "loss": 0.6137,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7836753129959106,
      "rewards/margins": 0.35466259717941284,
      "rewards/rejected": -1.1383378505706787,
      "step": 4850
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.310731513544033e-06,
      "logits/chosen": -1.556740403175354,
      "logits/rejected": -1.3384201526641846,
      "logps/chosen": -325.8907165527344,
      "logps/rejected": -362.38525390625,
      "loss": 0.4867,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.8303295373916626,
      "rewards/margins": 0.825741171836853,
      "rewards/rejected": -1.6560704708099365,
      "step": 4860
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.30678985249896e-06,
      "logits/chosen": -1.411836862564087,
      "logits/rejected": -1.3843629360198975,
      "logps/chosen": -257.8215637207031,
      "logps/rejected": -395.88067626953125,
      "loss": 0.3987,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.0067152976989746,
      "rewards/margins": 1.1979559659957886,
      "rewards/rejected": -2.2046713829040527,
      "step": 4870
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.302838766352952e-06,
      "logits/chosen": -1.1780803203582764,
      "logits/rejected": -1.026503324508667,
      "logps/chosen": -452.2996520996094,
      "logps/rejected": -511.9593811035156,
      "loss": 0.4642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.967066764831543,
      "rewards/margins": 0.9545124173164368,
      "rewards/rejected": -2.921579360961914,
      "step": 4880
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.298878275716806e-06,
      "logits/chosen": -0.8606294393539429,
      "logits/rejected": -0.8033844232559204,
      "logps/chosen": -475.10150146484375,
      "logps/rejected": -563.3385620117188,
      "loss": 0.5716,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.8322205543518066,
      "rewards/margins": 0.9103401303291321,
      "rewards/rejected": -3.742560863494873,
      "step": 4890
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.294908401250386e-06,
      "logits/chosen": -0.9198969006538391,
      "logits/rejected": -0.7157684564590454,
      "logps/chosen": -516.4268798828125,
      "logps/rejected": -558.6986083984375,
      "loss": 0.5004,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -3.016838312149048,
      "rewards/margins": 0.8680410385131836,
      "rewards/rejected": -3.8848788738250732,
      "step": 4900
    },
    {
      "epoch": 0.32,
      "eval_logits/chosen": -0.8907901048660278,
      "eval_logits/rejected": -0.7704294323921204,
      "eval_logps/chosen": -526.6473388671875,
      "eval_logps/rejected": -583.9198608398438,
      "eval_loss": 0.5151007771492004,
      "eval_rewards/accuracies": 0.6614999771118164,
      "eval_rewards/chosen": -2.9464235305786133,
      "eval_rewards/margins": 0.776655912399292,
      "eval_rewards/rejected": -3.7230799198150635,
      "eval_runtime": 732.5685,
      "eval_samples_per_second": 2.73,
      "eval_steps_per_second": 1.365,
      "step": 4900
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.290929163662498e-06,
      "logits/chosen": -0.6321516633033752,
      "logits/rejected": -0.6172500848770142,
      "logps/chosen": -548.6419677734375,
      "logps/rejected": -562.7332153320312,
      "loss": 0.5349,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.7890663146972656,
      "rewards/margins": 0.7180782556533813,
      "rewards/rejected": -3.5071444511413574,
      "step": 4910
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.286940583710796e-06,
      "logits/chosen": -1.0743346214294434,
      "logits/rejected": -0.8789170384407043,
      "logps/chosen": -624.6532592773438,
      "logps/rejected": -672.2891845703125,
      "loss": 0.4332,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.3190460205078125,
      "rewards/margins": 1.042657494544983,
      "rewards/rejected": -4.361703872680664,
      "step": 4920
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.282942682201667e-06,
      "logits/chosen": -0.9150835871696472,
      "logits/rejected": -0.7225080728530884,
      "logps/chosen": -558.1234741210938,
      "logps/rejected": -603.1954956054688,
      "loss": 0.5136,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -3.06015682220459,
      "rewards/margins": 0.8465850949287415,
      "rewards/rejected": -3.9067416191101074,
      "step": 4930
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.278935479990123e-06,
      "logits/chosen": -1.1814619302749634,
      "logits/rejected": -0.840973973274231,
      "logps/chosen": -482.06982421875,
      "logps/rejected": -508.01141357421875,
      "loss": 0.5504,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.774064540863037,
      "rewards/margins": 0.6891308426856995,
      "rewards/rejected": -3.463195323944092,
      "step": 4940
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.274918997979695e-06,
      "logits/chosen": -0.9456547498703003,
      "logits/rejected": -0.9622589945793152,
      "logps/chosen": -505.5489196777344,
      "logps/rejected": -578.9075927734375,
      "loss": 0.6068,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -3.1191391944885254,
      "rewards/margins": 0.7162333130836487,
      "rewards/rejected": -3.8353724479675293,
      "step": 4950
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.270893257122319e-06,
      "logits/chosen": -0.8949559330940247,
      "logits/rejected": -0.742863118648529,
      "logps/chosen": -508.76806640625,
      "logps/rejected": -651.4818115234375,
      "loss": 0.4558,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.8004612922668457,
      "rewards/margins": 1.1320440769195557,
      "rewards/rejected": -3.9325053691864014,
      "step": 4960
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.266858278418232e-06,
      "logits/chosen": -0.7636032104492188,
      "logits/rejected": -0.7520279884338379,
      "logps/chosen": -473.548583984375,
      "logps/rejected": -536.2864379882812,
      "loss": 0.5227,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.369443416595459,
      "rewards/margins": 0.8308904767036438,
      "rewards/rejected": -3.200334072113037,
      "step": 4970
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.26281408291586e-06,
      "logits/chosen": -1.1534435749053955,
      "logits/rejected": -0.9177444577217102,
      "logps/chosen": -476.447021484375,
      "logps/rejected": -567.8323974609375,
      "loss": 0.4335,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.341097116470337,
      "rewards/margins": 1.209177017211914,
      "rewards/rejected": -3.55027437210083,
      "step": 4980
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.258760691711706e-06,
      "logits/chosen": -1.1102027893066406,
      "logits/rejected": -0.977056622505188,
      "logps/chosen": -428.1145935058594,
      "logps/rejected": -505.6068420410156,
      "loss": 0.5251,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.306530475616455,
      "rewards/margins": 0.8208501935005188,
      "rewards/rejected": -3.12738037109375,
      "step": 4990
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.254698125950247e-06,
      "logits/chosen": -1.346663475036621,
      "logits/rejected": -1.1366852521896362,
      "logps/chosen": -513.910400390625,
      "logps/rejected": -552.2686767578125,
      "loss": 0.465,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.1289587020874023,
      "rewards/margins": 0.8502386808395386,
      "rewards/rejected": -2.9791972637176514,
      "step": 5000
    },
    {
      "epoch": 0.33,
      "eval_logits/chosen": -1.0639259815216064,
      "eval_logits/rejected": -0.934317409992218,
      "eval_logps/chosen": -465.99224853515625,
      "eval_logps/rejected": -532.8919677734375,
      "eval_loss": 0.5096431374549866,
      "eval_rewards/accuracies": 0.6675000190734863,
      "eval_rewards/chosen": -2.3398725986480713,
      "eval_rewards/margins": 0.8729275465011597,
      "eval_rewards/rejected": -3.2128000259399414,
      "eval_runtime": 734.3385,
      "eval_samples_per_second": 2.724,
      "eval_steps_per_second": 1.362,
      "step": 5000
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.250626406823815e-06,
      "logits/chosen": -1.1543405055999756,
      "logits/rejected": -0.9085921049118042,
      "logps/chosen": -472.118896484375,
      "logps/rejected": -643.9099731445312,
      "loss": 0.4521,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.5525622367858887,
      "rewards/margins": 1.5561296939849854,
      "rewards/rejected": -4.108692169189453,
      "step": 5010
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.246545555572489e-06,
      "logits/chosen": -0.8793859481811523,
      "logits/rejected": -0.8848443031311035,
      "logps/chosen": -409.1986999511719,
      "logps/rejected": -546.0990600585938,
      "loss": 0.5188,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.550025463104248,
      "rewards/margins": 1.1194829940795898,
      "rewards/rejected": -3.669508457183838,
      "step": 5020
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.242455593483992e-06,
      "logits/chosen": -1.1510926485061646,
      "logits/rejected": -0.9685741662979126,
      "logps/chosen": -441.26348876953125,
      "logps/rejected": -480.549560546875,
      "loss": 0.4735,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.2210960388183594,
      "rewards/margins": 0.8899548649787903,
      "rewards/rejected": -3.111051082611084,
      "step": 5030
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.238356541893567e-06,
      "logits/chosen": -1.0632343292236328,
      "logits/rejected": -1.0304065942764282,
      "logps/chosen": -393.16845703125,
      "logps/rejected": -484.4117736816406,
      "loss": 0.5064,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.03035306930542,
      "rewards/margins": 1.0678585767745972,
      "rewards/rejected": -3.0982117652893066,
      "step": 5040
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.234248422183876e-06,
      "logits/chosen": -1.16147780418396,
      "logits/rejected": -1.3117892742156982,
      "logps/chosen": -407.19354248046875,
      "logps/rejected": -506.6943359375,
      "loss": 0.4877,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6425886154174805,
      "rewards/margins": 1.0405709743499756,
      "rewards/rejected": -2.683159589767456,
      "step": 5050
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.230131255784884e-06,
      "logits/chosen": -1.5039904117584229,
      "logits/rejected": -1.2198777198791504,
      "logps/chosen": -476.0791015625,
      "logps/rejected": -579.59375,
      "loss": 0.5034,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.181959629058838,
      "rewards/margins": 1.2231109142303467,
      "rewards/rejected": -3.4050700664520264,
      "step": 5060
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.226005064173748e-06,
      "logits/chosen": -1.1899694204330444,
      "logits/rejected": -1.0733693838119507,
      "logps/chosen": -484.65301513671875,
      "logps/rejected": -579.5365600585938,
      "loss": 0.5832,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.200087785720825,
      "rewards/margins": 0.7642645239830017,
      "rewards/rejected": -2.9643523693084717,
      "step": 5070
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2218698688747035e-06,
      "logits/chosen": -0.830842137336731,
      "logits/rejected": -0.6599709987640381,
      "logps/chosen": -473.8846130371094,
      "logps/rejected": -529.7459716796875,
      "loss": 0.5309,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.411752939224243,
      "rewards/margins": 1.0099151134490967,
      "rewards/rejected": -3.4216678142547607,
      "step": 5080
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.217725691458957e-06,
      "logits/chosen": -1.3030509948730469,
      "logits/rejected": -1.1121243238449097,
      "logps/chosen": -401.26605224609375,
      "logps/rejected": -516.9102783203125,
      "loss": 0.5194,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1704189777374268,
      "rewards/margins": 0.8869827389717102,
      "rewards/rejected": -3.057401418685913,
      "step": 5090
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.213572553544565e-06,
      "logits/chosen": -0.968209445476532,
      "logits/rejected": -0.9317210912704468,
      "logps/chosen": -464.62896728515625,
      "logps/rejected": -547.8236083984375,
      "loss": 0.4609,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.2365708351135254,
      "rewards/margins": 0.9055919647216797,
      "rewards/rejected": -3.142162799835205,
      "step": 5100
    },
    {
      "epoch": 0.33,
      "eval_logits/chosen": -1.0512932538986206,
      "eval_logits/rejected": -0.9174529910087585,
      "eval_logps/chosen": -430.6408996582031,
      "eval_logps/rejected": -500.292236328125,
      "eval_loss": 0.5073493123054504,
      "eval_rewards/accuracies": 0.6654999852180481,
      "eval_rewards/chosen": -1.9863591194152832,
      "eval_rewards/margins": 0.900443971157074,
      "eval_rewards/rejected": -2.886803388595581,
      "eval_runtime": 733.5989,
      "eval_samples_per_second": 2.726,
      "eval_steps_per_second": 1.363,
      "step": 5100
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.209410476796331e-06,
      "logits/chosen": -0.9567044973373413,
      "logits/rejected": -0.9498974084854126,
      "logps/chosen": -364.0614318847656,
      "logps/rejected": -427.20098876953125,
      "loss": 0.5464,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9297001361846924,
      "rewards/margins": 0.7312873005867004,
      "rewards/rejected": -2.660987377166748,
      "step": 5110
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.205239482925686e-06,
      "logits/chosen": -0.9890548586845398,
      "logits/rejected": -0.9276937246322632,
      "logps/chosen": -364.5030212402344,
      "logps/rejected": -465.77105712890625,
      "loss": 0.5325,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7668765783309937,
      "rewards/margins": 0.7914160490036011,
      "rewards/rejected": -2.558292865753174,
      "step": 5120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.201059593690577e-06,
      "logits/chosen": -1.2910683155059814,
      "logits/rejected": -1.229804277420044,
      "logps/chosen": -417.73046875,
      "logps/rejected": -458.9871520996094,
      "loss": 0.5824,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9214271306991577,
      "rewards/margins": 0.6668572425842285,
      "rewards/rejected": -2.588284492492676,
      "step": 5130
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.196870830895354e-06,
      "logits/chosen": -1.0787732601165771,
      "logits/rejected": -0.9965429306030273,
      "logps/chosen": -433.9598693847656,
      "logps/rejected": -561.2127685546875,
      "loss": 0.5479,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7621228694915771,
      "rewards/margins": 0.7774554491043091,
      "rewards/rejected": -2.5395781993865967,
      "step": 5140
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.192673216390657e-06,
      "logits/chosen": -1.1716907024383545,
      "logits/rejected": -0.9472560882568359,
      "logps/chosen": -420.86053466796875,
      "logps/rejected": -453.2992248535156,
      "loss": 0.5648,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8788764476776123,
      "rewards/margins": 0.7076621055603027,
      "rewards/rejected": -2.586538553237915,
      "step": 5150
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.188466772073296e-06,
      "logits/chosen": -1.2703173160552979,
      "logits/rejected": -1.1680748462677002,
      "logps/chosen": -376.0206298828125,
      "logps/rejected": -413.83392333984375,
      "loss": 0.5771,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.55496084690094,
      "rewards/margins": 0.5857495069503784,
      "rewards/rejected": -2.1407103538513184,
      "step": 5160
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.184251519886148e-06,
      "logits/chosen": -1.160733938217163,
      "logits/rejected": -1.102087378501892,
      "logps/chosen": -395.3398132324219,
      "logps/rejected": -477.61346435546875,
      "loss": 0.612,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9923226833343506,
      "rewards/margins": 0.5361863970756531,
      "rewards/rejected": -2.5285091400146484,
      "step": 5170
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.180027481818033e-06,
      "logits/chosen": -1.25356125831604,
      "logits/rejected": -1.3256597518920898,
      "logps/chosen": -432.5494079589844,
      "logps/rejected": -464.7425231933594,
      "loss": 0.5836,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6585134267807007,
      "rewards/margins": 0.7002665996551514,
      "rewards/rejected": -2.3587799072265625,
      "step": 5180
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.175794679903602e-06,
      "logits/chosen": -1.1973209381103516,
      "logits/rejected": -1.0242760181427002,
      "logps/chosen": -373.4833984375,
      "logps/rejected": -385.53668212890625,
      "loss": 0.5217,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4584256410598755,
      "rewards/margins": 0.8441031575202942,
      "rewards/rejected": -2.3025288581848145,
      "step": 5190
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.171553136223222e-06,
      "logits/chosen": -1.3787262439727783,
      "logits/rejected": -1.2474148273468018,
      "logps/chosen": -437.20831298828125,
      "logps/rejected": -550.6790771484375,
      "loss": 0.4666,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6988741159439087,
      "rewards/margins": 1.0276092290878296,
      "rewards/rejected": -2.7264835834503174,
      "step": 5200
    },
    {
      "epoch": 0.34,
      "eval_logits/chosen": -1.1704281568527222,
      "eval_logits/rejected": -1.0363733768463135,
      "eval_logps/chosen": -391.68426513671875,
      "eval_logps/rejected": -446.65252685546875,
      "eval_loss": 0.5153765678405762,
      "eval_rewards/accuracies": 0.6600000262260437,
      "eval_rewards/chosen": -1.5967929363250732,
      "eval_rewards/margins": 0.7536134123802185,
      "eval_rewards/rejected": -2.3504061698913574,
      "eval_runtime": 733.295,
      "eval_samples_per_second": 2.727,
      "eval_steps_per_second": 1.364,
      "step": 5200
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.167302872902865e-06,
      "logits/chosen": -1.2382739782333374,
      "logits/rejected": -0.9230550527572632,
      "logps/chosen": -414.4139709472656,
      "logps/rejected": -526.5465698242188,
      "loss": 0.4884,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6294580698013306,
      "rewards/margins": 1.2407068014144897,
      "rewards/rejected": -2.8701653480529785,
      "step": 5210
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.163043912113985e-06,
      "logits/chosen": -1.1802736520767212,
      "logits/rejected": -0.9871931076049805,
      "logps/chosen": -414.599609375,
      "logps/rejected": -441.56396484375,
      "loss": 0.5555,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.555594801902771,
      "rewards/margins": 0.6095894575119019,
      "rewards/rejected": -2.165184259414673,
      "step": 5220
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.15877627607341e-06,
      "logits/chosen": -0.9541276097297668,
      "logits/rejected": -0.7061694860458374,
      "logps/chosen": -375.0079345703125,
      "logps/rejected": -419.3385314941406,
      "loss": 0.5109,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5841313600540161,
      "rewards/margins": 0.6855292916297913,
      "rewards/rejected": -2.269660711288452,
      "step": 5230
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.154499987043217e-06,
      "logits/chosen": -1.1777527332305908,
      "logits/rejected": -0.9939281344413757,
      "logps/chosen": -361.29815673828125,
      "logps/rejected": -444.8980407714844,
      "loss": 0.4545,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3709460496902466,
      "rewards/margins": 1.0577964782714844,
      "rewards/rejected": -2.4287424087524414,
      "step": 5240
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.150215067330625e-06,
      "logits/chosen": -1.099338173866272,
      "logits/rejected": -0.8791137933731079,
      "logps/chosen": -346.3368225097656,
      "logps/rejected": -462.539794921875,
      "loss": 0.4998,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.372879981994629,
      "rewards/margins": 1.0216567516326904,
      "rewards/rejected": -2.3945367336273193,
      "step": 5250
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.145921539287876e-06,
      "logits/chosen": -1.0073109865188599,
      "logits/rejected": -0.9385402798652649,
      "logps/chosen": -321.3467102050781,
      "logps/rejected": -424.44293212890625,
      "loss": 0.4308,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3075411319732666,
      "rewards/margins": 1.2309949398040771,
      "rewards/rejected": -2.5385358333587646,
      "step": 5260
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.141619425312115e-06,
      "logits/chosen": -1.014169454574585,
      "logits/rejected": -0.7141678333282471,
      "logps/chosen": -381.5502624511719,
      "logps/rejected": -399.2060241699219,
      "loss": 0.6727,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -1.7289901971817017,
      "rewards/margins": 0.420976459980011,
      "rewards/rejected": -2.1499664783477783,
      "step": 5270
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.1373087478452735e-06,
      "logits/chosen": -0.9243456125259399,
      "logits/rejected": -0.8291029930114746,
      "logps/chosen": -372.74444580078125,
      "logps/rejected": -466.50909423828125,
      "loss": 0.3357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4750388860702515,
      "rewards/margins": 1.4837852716445923,
      "rewards/rejected": -2.9588239192962646,
      "step": 5280
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.132989529373959e-06,
      "logits/chosen": -0.9148101806640625,
      "logits/rejected": -0.732371985912323,
      "logps/chosen": -496.69793701171875,
      "logps/rejected": -481.1084899902344,
      "loss": 0.5406,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.407851219177246,
      "rewards/margins": 0.634090781211853,
      "rewards/rejected": -3.0419418811798096,
      "step": 5290
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.128661792429331e-06,
      "logits/chosen": -0.8414371609687805,
      "logits/rejected": -0.7446430325508118,
      "logps/chosen": -479.20733642578125,
      "logps/rejected": -548.0040893554688,
      "loss": 0.6107,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.2543675899505615,
      "rewards/margins": 0.6390036344528198,
      "rewards/rejected": -2.893371105194092,
      "step": 5300
    },
    {
      "epoch": 0.35,
      "eval_logits/chosen": -0.9356719851493835,
      "eval_logits/rejected": -0.8068389892578125,
      "eval_logps/chosen": -456.3209228515625,
      "eval_logps/rejected": -521.69482421875,
      "eval_loss": 0.5146012902259827,
      "eval_rewards/accuracies": 0.6570000052452087,
      "eval_rewards/chosen": -2.2431600093841553,
      "eval_rewards/margins": 0.8576699495315552,
      "eval_rewards/rejected": -3.100829601287842,
      "eval_runtime": 732.9797,
      "eval_samples_per_second": 2.729,
      "eval_steps_per_second": 1.364,
      "step": 5300
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.124325559586985e-06,
      "logits/chosen": -0.9261112213134766,
      "logits/rejected": -0.7737448811531067,
      "logps/chosen": -415.0210876464844,
      "logps/rejected": -464.01275634765625,
      "loss": 0.8043,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.269643545150757,
      "rewards/margins": 0.3606763482093811,
      "rewards/rejected": -2.630319833755493,
      "step": 5310
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.119980853466835e-06,
      "logits/chosen": -0.700569212436676,
      "logits/rejected": -0.3479693830013275,
      "logps/chosen": -467.34918212890625,
      "logps/rejected": -507.01568603515625,
      "loss": 0.4996,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.5819480419158936,
      "rewards/margins": 0.6678058505058289,
      "rewards/rejected": -3.249753952026367,
      "step": 5320
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.115627696732997e-06,
      "logits/chosen": -0.7397909760475159,
      "logits/rejected": -0.6655017733573914,
      "logps/chosen": -402.98309326171875,
      "logps/rejected": -443.57208251953125,
      "loss": 0.5533,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.1156888008117676,
      "rewards/margins": 0.5990991592407227,
      "rewards/rejected": -2.714787721633911,
      "step": 5330
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.111266112093668e-06,
      "logits/chosen": -0.9121619462966919,
      "logits/rejected": -0.7928305268287659,
      "logps/chosen": -454.974609375,
      "logps/rejected": -538.3798217773438,
      "loss": 0.5456,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.5150694847106934,
      "rewards/margins": 0.6500739455223083,
      "rewards/rejected": -3.1651434898376465,
      "step": 5340
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.1068961223010115e-06,
      "logits/chosen": -1.0712369680404663,
      "logits/rejected": -0.616116464138031,
      "logps/chosen": -550.204345703125,
      "logps/rejected": -586.81591796875,
      "loss": 0.6113,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.7950360774993896,
      "rewards/margins": 0.5928794145584106,
      "rewards/rejected": -3.387915849685669,
      "step": 5350
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.102517750151034e-06,
      "logits/chosen": -1.1240382194519043,
      "logits/rejected": -0.8812620043754578,
      "logps/chosen": -509.80694580078125,
      "logps/rejected": -496.1722106933594,
      "loss": 0.5562,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1476213932037354,
      "rewards/margins": 0.5767813920974731,
      "rewards/rejected": -2.724402666091919,
      "step": 5360
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.09813101848347e-06,
      "logits/chosen": -1.180262804031372,
      "logits/rejected": -0.9633585810661316,
      "logps/chosen": -409.8777770996094,
      "logps/rejected": -507.8788146972656,
      "loss": 0.5133,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9089186191558838,
      "rewards/margins": 0.8084686398506165,
      "rewards/rejected": -2.7173869609832764,
      "step": 5370
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.093735950181659e-06,
      "logits/chosen": -1.004286527633667,
      "logits/rejected": -0.8687463998794556,
      "logps/chosen": -397.5352783203125,
      "logps/rejected": -523.0494384765625,
      "loss": 0.4518,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7206192016601562,
      "rewards/margins": 1.0097081661224365,
      "rewards/rejected": -2.7303271293640137,
      "step": 5380
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.0893325681724326e-06,
      "logits/chosen": -1.2412950992584229,
      "logits/rejected": -1.1470402479171753,
      "logps/chosen": -440.41473388671875,
      "logps/rejected": -540.3668212890625,
      "loss": 0.4247,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8199307918548584,
      "rewards/margins": 1.1223140954971313,
      "rewards/rejected": -2.9422447681427,
      "step": 5390
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.084920895425988e-06,
      "logits/chosen": -1.1176074743270874,
      "logits/rejected": -0.9556187391281128,
      "logps/chosen": -450.7571716308594,
      "logps/rejected": -545.2280883789062,
      "loss": 0.5853,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.133924722671509,
      "rewards/margins": 0.8082602620124817,
      "rewards/rejected": -2.9421849250793457,
      "step": 5400
    },
    {
      "epoch": 0.35,
      "eval_logits/chosen": -1.0984172821044922,
      "eval_logits/rejected": -0.9615691900253296,
      "eval_logps/chosen": -401.5628662109375,
      "eval_logps/rejected": -471.24493408203125,
      "eval_loss": 0.5090234875679016,
      "eval_rewards/accuracies": 0.6625000238418579,
      "eval_rewards/chosen": -1.695579171180725,
      "eval_rewards/margins": 0.9007511734962463,
      "eval_rewards/rejected": -2.596330404281616,
      "eval_runtime": 733.7513,
      "eval_samples_per_second": 2.726,
      "eval_steps_per_second": 1.363,
      "step": 5400
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.080500954955769e-06,
      "logits/chosen": -1.0121065378189087,
      "logits/rejected": -0.8766088485717773,
      "logps/chosen": -451.9566955566406,
      "logps/rejected": -547.6082153320312,
      "loss": 0.5004,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9608478546142578,
      "rewards/margins": 1.1393764019012451,
      "rewards/rejected": -3.100224256515503,
      "step": 5410
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.076072769818354e-06,
      "logits/chosen": -1.373978853225708,
      "logits/rejected": -1.0941734313964844,
      "logps/chosen": -385.83087158203125,
      "logps/rejected": -430.9171447753906,
      "loss": 0.4115,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.421086072921753,
      "rewards/margins": 1.0705190896987915,
      "rewards/rejected": -2.491605043411255,
      "step": 5420
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.071636363113323e-06,
      "logits/chosen": -0.8405832052230835,
      "logits/rejected": -0.7371835708618164,
      "logps/chosen": -422.33856201171875,
      "logps/rejected": -424.07147216796875,
      "loss": 0.659,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.5420846939086914,
      "rewards/margins": 0.5577548742294312,
      "rewards/rejected": -2.099839687347412,
      "step": 5430
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.067191757983146e-06,
      "logits/chosen": -0.9356991648674011,
      "logits/rejected": -0.8254088163375854,
      "logps/chosen": -376.23724365234375,
      "logps/rejected": -477.99822998046875,
      "loss": 0.4982,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.466615915298462,
      "rewards/margins": 1.0497499704360962,
      "rewards/rejected": -2.5163657665252686,
      "step": 5440
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.062738977613063e-06,
      "logits/chosen": -0.7855893969535828,
      "logits/rejected": -0.8330990672111511,
      "logps/chosen": -376.4490966796875,
      "logps/rejected": -393.9264831542969,
      "loss": 0.5883,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4197274446487427,
      "rewards/margins": 0.6587068438529968,
      "rewards/rejected": -2.078434467315674,
      "step": 5450
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.058278045230957e-06,
      "logits/chosen": -1.189239501953125,
      "logits/rejected": -1.1592838764190674,
      "logps/chosen": -382.54998779296875,
      "logps/rejected": -449.02276611328125,
      "loss": 0.601,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.641831398010254,
      "rewards/margins": 0.6488510370254517,
      "rewards/rejected": -2.290682315826416,
      "step": 5460
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.053808984107235e-06,
      "logits/chosen": -1.1778719425201416,
      "logits/rejected": -1.030869960784912,
      "logps/chosen": -386.10565185546875,
      "logps/rejected": -389.18975830078125,
      "loss": 0.6179,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.5323729515075684,
      "rewards/margins": 0.43377256393432617,
      "rewards/rejected": -1.9661457538604736,
      "step": 5470
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.04933181755471e-06,
      "logits/chosen": -1.2040650844573975,
      "logits/rejected": -1.190525770187378,
      "logps/chosen": -352.6644287109375,
      "logps/rejected": -414.3978576660156,
      "loss": 0.5128,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.460837960243225,
      "rewards/margins": 0.6894599199295044,
      "rewards/rejected": -2.1502978801727295,
      "step": 5480
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.044846568928477e-06,
      "logits/chosen": -1.382424235343933,
      "logits/rejected": -1.3112528324127197,
      "logps/chosen": -436.9786071777344,
      "logps/rejected": -498.826171875,
      "loss": 0.5284,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7545440196990967,
      "rewards/margins": 0.7070547938346863,
      "rewards/rejected": -2.461599111557007,
      "step": 5490
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.040353261625788e-06,
      "logits/chosen": -1.389473557472229,
      "logits/rejected": -0.8910207748413086,
      "logps/chosen": -446.0039978027344,
      "logps/rejected": -486.54071044921875,
      "loss": 0.5086,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6945879459381104,
      "rewards/margins": 0.7916721701622009,
      "rewards/rejected": -2.486259937286377,
      "step": 5500
    },
    {
      "epoch": 0.36,
      "eval_logits/chosen": -1.1007320880889893,
      "eval_logits/rejected": -0.9732699990272522,
      "eval_logps/chosen": -405.7402648925781,
      "eval_logps/rejected": -457.79937744140625,
      "eval_loss": 0.5213733911514282,
      "eval_rewards/accuracies": 0.659500002861023,
      "eval_rewards/chosen": -1.7373523712158203,
      "eval_rewards/margins": 0.7245224118232727,
      "eval_rewards/rejected": -2.4618749618530273,
      "eval_runtime": 733.2339,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 5500
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.035851919085936e-06,
      "logits/chosen": -1.179037094116211,
      "logits/rejected": -0.9055557250976562,
      "logps/chosen": -443.47998046875,
      "logps/rejected": -447.66864013671875,
      "loss": 0.5885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7608587741851807,
      "rewards/margins": 0.8290047645568848,
      "rewards/rejected": -2.5898633003234863,
      "step": 5510
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.031342564790128e-06,
      "logits/chosen": -1.1599061489105225,
      "logits/rejected": -0.9323163032531738,
      "logps/chosen": -360.9304504394531,
      "logps/rejected": -445.9639587402344,
      "loss": 0.4878,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.5495872497558594,
      "rewards/margins": 0.8980630040168762,
      "rewards/rejected": -2.44765043258667,
      "step": 5520
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.026825222261367e-06,
      "logits/chosen": -0.9034293293952942,
      "logits/rejected": -0.714095413684845,
      "logps/chosen": -382.3951110839844,
      "logps/rejected": -428.35223388671875,
      "loss": 0.6408,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.093200922012329,
      "rewards/margins": 0.5310032963752747,
      "rewards/rejected": -2.624203681945801,
      "step": 5530
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.022299915064321e-06,
      "logits/chosen": -1.158623218536377,
      "logits/rejected": -0.9449752569198608,
      "logps/chosen": -453.1712341308594,
      "logps/rejected": -479.60687255859375,
      "loss": 0.4749,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4732328653335571,
      "rewards/margins": 0.795411229133606,
      "rewards/rejected": -2.268643856048584,
      "step": 5540
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.017766666805213e-06,
      "logits/chosen": -1.106878638267517,
      "logits/rejected": -0.8825507164001465,
      "logps/chosen": -366.5860595703125,
      "logps/rejected": -397.0874328613281,
      "loss": 0.6049,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5306566953659058,
      "rewards/margins": 0.6586295366287231,
      "rewards/rejected": -2.189286470413208,
      "step": 5550
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.013225501131684e-06,
      "logits/chosen": -1.176776647567749,
      "logits/rejected": -0.9218536615371704,
      "logps/chosen": -389.47491455078125,
      "logps/rejected": -412.9156799316406,
      "loss": 0.5809,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.720973253250122,
      "rewards/margins": 0.5041357278823853,
      "rewards/rejected": -2.225109100341797,
      "step": 5560
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.008676441732679e-06,
      "logits/chosen": -0.9157624244689941,
      "logits/rejected": -0.7230433821678162,
      "logps/chosen": -374.0820007324219,
      "logps/rejected": -383.73028564453125,
      "loss": 0.5511,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.6778764724731445,
      "rewards/margins": 0.5764918327331543,
      "rewards/rejected": -2.254368305206299,
      "step": 5570
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.00411951233832e-06,
      "logits/chosen": -1.112180471420288,
      "logits/rejected": -0.9340667724609375,
      "logps/chosen": -386.84442138671875,
      "logps/rejected": -427.77093505859375,
      "loss": 0.4714,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.6711927652359009,
      "rewards/margins": 0.8560009002685547,
      "rewards/rejected": -2.527193307876587,
      "step": 5580
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.999554736719785e-06,
      "logits/chosen": -0.8898111581802368,
      "logits/rejected": -0.8584083318710327,
      "logps/chosen": -453.03466796875,
      "logps/rejected": -473.88739013671875,
      "loss": 0.5457,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6105778217315674,
      "rewards/margins": 0.6437070369720459,
      "rewards/rejected": -2.2542850971221924,
      "step": 5590
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.994982138689177e-06,
      "logits/chosen": -1.4541324377059937,
      "logits/rejected": -1.138900637626648,
      "logps/chosen": -391.6158752441406,
      "logps/rejected": -467.52142333984375,
      "loss": 0.4764,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5421569347381592,
      "rewards/margins": 0.7775677442550659,
      "rewards/rejected": -2.3197245597839355,
      "step": 5600
    },
    {
      "epoch": 0.37,
      "eval_logits/chosen": -1.0609233379364014,
      "eval_logits/rejected": -0.9316677451133728,
      "eval_logps/chosen": -393.97259521484375,
      "eval_logps/rejected": -452.8468322753906,
      "eval_loss": 0.5123740434646606,
      "eval_rewards/accuracies": 0.6625000238418579,
      "eval_rewards/chosen": -1.6196763515472412,
      "eval_rewards/margins": 0.792672872543335,
      "eval_rewards/rejected": -2.412349224090576,
      "eval_runtime": 732.7002,
      "eval_samples_per_second": 2.73,
      "eval_steps_per_second": 1.365,
      "step": 5600
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.990401742099408e-06,
      "logits/chosen": -0.8740717768669128,
      "logits/rejected": -0.8899892568588257,
      "logps/chosen": -323.3037414550781,
      "logps/rejected": -371.63824462890625,
      "loss": 0.6185,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -1.4444063901901245,
      "rewards/margins": 0.556832492351532,
      "rewards/rejected": -2.0012388229370117,
      "step": 5610
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.985813570844072e-06,
      "logits/chosen": -1.2019202709197998,
      "logits/rejected": -1.0983178615570068,
      "logps/chosen": -469.497314453125,
      "logps/rejected": -519.1656494140625,
      "loss": 0.6347,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7511056661605835,
      "rewards/margins": 0.7721987962722778,
      "rewards/rejected": -2.5233044624328613,
      "step": 5620
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.981217648857316e-06,
      "logits/chosen": -1.1407134532928467,
      "logits/rejected": -1.063767433166504,
      "logps/chosen": -300.8212890625,
      "logps/rejected": -393.89117431640625,
      "loss": 0.4415,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.2828083038330078,
      "rewards/margins": 0.9128808975219727,
      "rewards/rejected": -2.1956894397735596,
      "step": 5630
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.97661400011372e-06,
      "logits/chosen": -1.126928448677063,
      "logits/rejected": -1.2266345024108887,
      "logps/chosen": -359.68841552734375,
      "logps/rejected": -416.9710388183594,
      "loss": 0.5515,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1666107177734375,
      "rewards/margins": 0.6870548725128174,
      "rewards/rejected": -1.8536655902862549,
      "step": 5640
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.972002648628174e-06,
      "logits/chosen": -1.192249059677124,
      "logits/rejected": -1.0249332189559937,
      "logps/chosen": -435.70001220703125,
      "logps/rejected": -451.20928955078125,
      "loss": 0.5513,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.5866916179656982,
      "rewards/margins": 0.6320112943649292,
      "rewards/rejected": -2.218702793121338,
      "step": 5650
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.967383618455743e-06,
      "logits/chosen": -1.167616605758667,
      "logits/rejected": -1.2211428880691528,
      "logps/chosen": -407.02044677734375,
      "logps/rejected": -497.1388244628906,
      "loss": 0.5616,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7990070581436157,
      "rewards/margins": 0.719279408454895,
      "rewards/rejected": -2.5182864665985107,
      "step": 5660
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9627569336915515e-06,
      "logits/chosen": -1.4768421649932861,
      "logits/rejected": -1.2465412616729736,
      "logps/chosen": -409.7810974121094,
      "logps/rejected": -429.510009765625,
      "loss": 0.5254,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.623183250427246,
      "rewards/margins": 0.7640554308891296,
      "rewards/rejected": -2.3872387409210205,
      "step": 5670
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9581226184706555e-06,
      "logits/chosen": -1.3483765125274658,
      "logits/rejected": -1.5304193496704102,
      "logps/chosen": -384.47637939453125,
      "logps/rejected": -524.4840698242188,
      "loss": 0.5158,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8981096744537354,
      "rewards/margins": 0.6931003332138062,
      "rewards/rejected": -2.591209888458252,
      "step": 5680
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.953480696967912e-06,
      "logits/chosen": -1.0088798999786377,
      "logits/rejected": -1.1363639831542969,
      "logps/chosen": -370.19451904296875,
      "logps/rejected": -457.1572265625,
      "loss": 0.5491,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.608229398727417,
      "rewards/margins": 0.5051697492599487,
      "rewards/rejected": -2.113399028778076,
      "step": 5690
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.948831193397857e-06,
      "logits/chosen": -1.0741056203842163,
      "logits/rejected": -1.0835530757904053,
      "logps/chosen": -304.1203308105469,
      "logps/rejected": -343.5845642089844,
      "loss": 0.6562,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.3810492753982544,
      "rewards/margins": 0.3363833725452423,
      "rewards/rejected": -1.7174327373504639,
      "step": 5700
    },
    {
      "epoch": 0.37,
      "eval_logits/chosen": -1.2059582471847534,
      "eval_logits/rejected": -1.0711474418640137,
      "eval_logps/chosen": -369.1748962402344,
      "eval_logps/rejected": -425.8072509765625,
      "eval_loss": 0.5096937417984009,
      "eval_rewards/accuracies": 0.6710000038146973,
      "eval_rewards/chosen": -1.371699333190918,
      "eval_rewards/margins": 0.7702544331550598,
      "eval_rewards/rejected": -2.141953706741333,
      "eval_runtime": 733.1703,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 5700
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.94417413201458e-06,
      "logits/chosen": -1.284786343574524,
      "logits/rejected": -1.0199940204620361,
      "logps/chosen": -344.146240234375,
      "logps/rejected": -395.18060302734375,
      "loss": 0.5787,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.3257472515106201,
      "rewards/margins": 0.6880120038986206,
      "rewards/rejected": -2.013759136199951,
      "step": 5710
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9395095371115935e-06,
      "logits/chosen": -1.1523475646972656,
      "logits/rejected": -1.065656065940857,
      "logps/chosen": -355.04437255859375,
      "logps/rejected": -424.20001220703125,
      "loss": 0.5606,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.423872947692871,
      "rewards/margins": 0.7976680994033813,
      "rewards/rejected": -2.221540927886963,
      "step": 5720
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.93483743302171e-06,
      "logits/chosen": -1.2104334831237793,
      "logits/rejected": -0.918979823589325,
      "logps/chosen": -350.27996826171875,
      "logps/rejected": -402.14666748046875,
      "loss": 0.5375,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.3748830556869507,
      "rewards/margins": 0.7560206055641174,
      "rewards/rejected": -2.130903720855713,
      "step": 5730
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.930157844116913e-06,
      "logits/chosen": -1.1220810413360596,
      "logits/rejected": -0.8528586626052856,
      "logps/chosen": -365.50885009765625,
      "logps/rejected": -416.3897399902344,
      "loss": 0.6294,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.5575157403945923,
      "rewards/margins": 0.6559960246086121,
      "rewards/rejected": -2.2135117053985596,
      "step": 5740
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.925470794808229e-06,
      "logits/chosen": -1.0437357425689697,
      "logits/rejected": -0.9060714840888977,
      "logps/chosen": -393.3031005859375,
      "logps/rejected": -465.1321716308594,
      "loss": 0.4304,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5431537628173828,
      "rewards/margins": 1.046464443206787,
      "rewards/rejected": -2.58961820602417,
      "step": 5750
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.920776309545606e-06,
      "logits/chosen": -1.3418052196502686,
      "logits/rejected": -1.122631311416626,
      "logps/chosen": -249.40090942382812,
      "logps/rejected": -329.80426025390625,
      "loss": 0.4853,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.9669389724731445,
      "rewards/margins": 0.8723586797714233,
      "rewards/rejected": -1.8392976522445679,
      "step": 5760
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.916074412817778e-06,
      "logits/chosen": -1.1772065162658691,
      "logits/rejected": -0.8717998266220093,
      "logps/chosen": -393.0423278808594,
      "logps/rejected": -518.3782958984375,
      "loss": 0.3867,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.497083067893982,
      "rewards/margins": 1.2481367588043213,
      "rewards/rejected": -2.7452197074890137,
      "step": 5770
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.911365129152139e-06,
      "logits/chosen": -1.2302935123443604,
      "logits/rejected": -1.1185319423675537,
      "logps/chosen": -387.8710632324219,
      "logps/rejected": -487.1651916503906,
      "loss": 0.507,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5543718338012695,
      "rewards/margins": 1.0483394861221313,
      "rewards/rejected": -2.6027112007141113,
      "step": 5780
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.906648483114623e-06,
      "logits/chosen": -1.241497278213501,
      "logits/rejected": -1.0616934299468994,
      "logps/chosen": -333.7834777832031,
      "logps/rejected": -418.350830078125,
      "loss": 0.5069,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.364598274230957,
      "rewards/margins": 1.171523928642273,
      "rewards/rejected": -2.5361220836639404,
      "step": 5790
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.901924499309564e-06,
      "logits/chosen": -1.1808868646621704,
      "logits/rejected": -1.052524209022522,
      "logps/chosen": -348.7895202636719,
      "logps/rejected": -419.6104431152344,
      "loss": 0.5178,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2008459568023682,
      "rewards/margins": 1.0342692136764526,
      "rewards/rejected": -2.2351150512695312,
      "step": 5800
    },
    {
      "epoch": 0.38,
      "eval_logits/chosen": -1.2822357416152954,
      "eval_logits/rejected": -1.1354150772094727,
      "eval_logps/chosen": -367.5433044433594,
      "eval_logps/rejected": -447.6251220703125,
      "eval_loss": 0.5039393305778503,
      "eval_rewards/accuracies": 0.6614999771118164,
      "eval_rewards/chosen": -1.3553833961486816,
      "eval_rewards/margins": 1.0047485828399658,
      "eval_rewards/rejected": -2.3601317405700684,
      "eval_runtime": 733.8198,
      "eval_samples_per_second": 2.725,
      "eval_steps_per_second": 1.363,
      "step": 5800
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.897193202379575e-06,
      "logits/chosen": -1.3546619415283203,
      "logits/rejected": -1.1554213762283325,
      "logps/chosen": -323.00579833984375,
      "logps/rejected": -405.39202880859375,
      "loss": 0.547,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.225281000137329,
      "rewards/margins": 0.9935903549194336,
      "rewards/rejected": -2.2188713550567627,
      "step": 5810
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.8924546170054215e-06,
      "logits/chosen": -1.2916877269744873,
      "logits/rejected": -1.132554054260254,
      "logps/chosen": -355.0088806152344,
      "logps/rejected": -439.87890625,
      "loss": 0.5571,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3910125494003296,
      "rewards/margins": 1.0434343814849854,
      "rewards/rejected": -2.4344468116760254,
      "step": 5820
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.887708767905883e-06,
      "logits/chosen": -1.4169222116470337,
      "logits/rejected": -1.1427775621414185,
      "logps/chosen": -397.4792175292969,
      "logps/rejected": -421.56280517578125,
      "loss": 0.4589,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5371202230453491,
      "rewards/margins": 0.8992649912834167,
      "rewards/rejected": -2.4363853931427,
      "step": 5830
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.882955679837636e-06,
      "logits/chosen": -1.1456177234649658,
      "logits/rejected": -1.1761143207550049,
      "logps/chosen": -452.54736328125,
      "logps/rejected": -531.5821533203125,
      "loss": 0.5629,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.1317031383514404,
      "rewards/margins": 0.7665945887565613,
      "rewards/rejected": -2.8982975482940674,
      "step": 5840
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.878195377595113e-06,
      "logits/chosen": -1.1690410375595093,
      "logits/rejected": -1.02626371383667,
      "logps/chosen": -395.9454650878906,
      "logps/rejected": -514.1718139648438,
      "loss": 0.5532,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.600396752357483,
      "rewards/margins": 1.2378990650177002,
      "rewards/rejected": -2.8382959365844727,
      "step": 5850
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.873427886010384e-06,
      "logits/chosen": -1.234686255455017,
      "logits/rejected": -0.9635084867477417,
      "logps/chosen": -356.0743408203125,
      "logps/rejected": -423.4266052246094,
      "loss": 0.5384,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.656224012374878,
      "rewards/margins": 0.8644134402275085,
      "rewards/rejected": -2.5206375122070312,
      "step": 5860
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.868653229953021e-06,
      "logits/chosen": -1.2322075366973877,
      "logits/rejected": -1.0185177326202393,
      "logps/chosen": -443.06982421875,
      "logps/rejected": -576.76123046875,
      "loss": 0.4052,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.0585498809814453,
      "rewards/margins": 1.357552170753479,
      "rewards/rejected": -3.416102170944214,
      "step": 5870
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.8638714343299675e-06,
      "logits/chosen": -1.262803316116333,
      "logits/rejected": -1.1337450742721558,
      "logps/chosen": -364.6521911621094,
      "logps/rejected": -484.88433837890625,
      "loss": 0.4626,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.463094711303711,
      "rewards/margins": 1.0511635541915894,
      "rewards/rejected": -2.5142581462860107,
      "step": 5880
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.859082524085414e-06,
      "logits/chosen": -1.118187665939331,
      "logits/rejected": -0.9261728525161743,
      "logps/chosen": -433.3507385253906,
      "logps/rejected": -452.5609436035156,
      "loss": 0.5653,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.61007559299469,
      "rewards/margins": 0.7944930791854858,
      "rewards/rejected": -2.404569149017334,
      "step": 5890
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.854286524200659e-06,
      "logits/chosen": -1.5227950811386108,
      "logits/rejected": -1.1201550960540771,
      "logps/chosen": -422.0400390625,
      "logps/rejected": -453.4849548339844,
      "loss": 0.5391,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4562262296676636,
      "rewards/margins": 0.8208130598068237,
      "rewards/rejected": -2.2770392894744873,
      "step": 5900
    },
    {
      "epoch": 0.39,
      "eval_logits/chosen": -1.2484019994735718,
      "eval_logits/rejected": -1.1068450212478638,
      "eval_logps/chosen": -369.74603271484375,
      "eval_logps/rejected": -439.0063171386719,
      "eval_loss": 0.503896176815033,
      "eval_rewards/accuracies": 0.6614999771118164,
      "eval_rewards/chosen": -1.3774104118347168,
      "eval_rewards/margins": 0.8965333700180054,
      "eval_rewards/rejected": -2.273944139480591,
      "eval_runtime": 733.8267,
      "eval_samples_per_second": 2.725,
      "eval_steps_per_second": 1.363,
      "step": 5900
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.849483459693991e-06,
      "logits/chosen": -1.343681812286377,
      "logits/rejected": -1.1354031562805176,
      "logps/chosen": -343.40374755859375,
      "logps/rejected": -400.17694091796875,
      "loss": 0.4648,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3372704982757568,
      "rewards/margins": 0.9544233083724976,
      "rewards/rejected": -2.291693687438965,
      "step": 5910
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.844673355620544e-06,
      "logits/chosen": -1.2277530431747437,
      "logits/rejected": -0.9738208651542664,
      "logps/chosen": -422.9916076660156,
      "logps/rejected": -475.74822998046875,
      "loss": 0.5377,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.7641870975494385,
      "rewards/margins": 0.9104213714599609,
      "rewards/rejected": -2.6746084690093994,
      "step": 5920
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.839856237072178e-06,
      "logits/chosen": -0.9866417646408081,
      "logits/rejected": -0.9235474467277527,
      "logps/chosen": -350.86114501953125,
      "logps/rejected": -475.2818298339844,
      "loss": 0.5043,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6828702688217163,
      "rewards/margins": 1.072285771369934,
      "rewards/rejected": -2.7551560401916504,
      "step": 5930
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8350321291773455e-06,
      "logits/chosen": -0.9643562436103821,
      "logits/rejected": -0.8080571293830872,
      "logps/chosen": -331.8020324707031,
      "logps/rejected": -392.1284484863281,
      "loss": 0.5131,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.2928006649017334,
      "rewards/margins": 1.04522705078125,
      "rewards/rejected": -2.3380279541015625,
      "step": 5940
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.830201057100953e-06,
      "logits/chosen": -1.3062032461166382,
      "logits/rejected": -1.205256462097168,
      "logps/chosen": -359.8435974121094,
      "logps/rejected": -475.44189453125,
      "loss": 0.4919,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6807063817977905,
      "rewards/margins": 1.0010545253753662,
      "rewards/rejected": -2.681760787963867,
      "step": 5950
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.82536304604424e-06,
      "logits/chosen": -1.07782781124115,
      "logits/rejected": -0.9527738690376282,
      "logps/chosen": -372.5465087890625,
      "logps/rejected": -422.93499755859375,
      "loss": 0.531,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.3638226985931396,
      "rewards/margins": 0.8212461471557617,
      "rewards/rejected": -2.1850686073303223,
      "step": 5960
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8205181212446435e-06,
      "logits/chosen": -1.4037820100784302,
      "logits/rejected": -1.2214797735214233,
      "logps/chosen": -376.4328918457031,
      "logps/rejected": -438.6416931152344,
      "loss": 0.4591,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.0587395429611206,
      "rewards/margins": 1.0261236429214478,
      "rewards/rejected": -2.0848631858825684,
      "step": 5970
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.815666307975664e-06,
      "logits/chosen": -1.19920015335083,
      "logits/rejected": -1.1759016513824463,
      "logps/chosen": -379.8518981933594,
      "logps/rejected": -470.53643798828125,
      "loss": 0.4468,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.425341010093689,
      "rewards/margins": 1.09866464138031,
      "rewards/rejected": -2.524005651473999,
      "step": 5980
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8108076315467346e-06,
      "logits/chosen": -1.3615524768829346,
      "logits/rejected": -1.2437620162963867,
      "logps/chosen": -436.19671630859375,
      "logps/rejected": -463.3844299316406,
      "loss": 0.548,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.789139747619629,
      "rewards/margins": 1.0191426277160645,
      "rewards/rejected": -2.8082823753356934,
      "step": 5990
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.805942117303093e-06,
      "logits/chosen": -1.3936203718185425,
      "logits/rejected": -1.2214877605438232,
      "logps/chosen": -477.30487060546875,
      "logps/rejected": -533.3314819335938,
      "loss": 0.4757,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.612693190574646,
      "rewards/margins": 1.0255014896392822,
      "rewards/rejected": -2.6381945610046387,
      "step": 6000
    },
    {
      "epoch": 0.39,
      "eval_logits/chosen": -1.0945810079574585,
      "eval_logits/rejected": -0.9610912799835205,
      "eval_logps/chosen": -386.2829284667969,
      "eval_logps/rejected": -458.74658203125,
      "eval_loss": 0.5028457045555115,
      "eval_rewards/accuracies": 0.6654999852180481,
      "eval_rewards/chosen": -1.5427796840667725,
      "eval_rewards/margins": 0.9285673499107361,
      "eval_rewards/rejected": -2.4713470935821533,
      "eval_runtime": 734.4828,
      "eval_samples_per_second": 2.723,
      "eval_steps_per_second": 1.362,
      "step": 6000
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8010697906256446e-06,
      "logits/chosen": -1.1743717193603516,
      "logits/rejected": -0.9436849355697632,
      "logps/chosen": -404.7149658203125,
      "logps/rejected": -437.8125,
      "loss": 0.6255,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.9801855087280273,
      "rewards/margins": 0.6092766523361206,
      "rewards/rejected": -2.5894620418548584,
      "step": 6010
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7961906769308323e-06,
      "logits/chosen": -0.7950822114944458,
      "logits/rejected": -0.8160813450813293,
      "logps/chosen": -389.021484375,
      "logps/rejected": -467.6617126464844,
      "loss": 0.5355,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8278629779815674,
      "rewards/margins": 0.6856533288955688,
      "rewards/rejected": -2.5135159492492676,
      "step": 6020
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7913048016705028e-06,
      "logits/chosen": -1.1174968481063843,
      "logits/rejected": -0.9798613786697388,
      "logps/chosen": -443.1385803222656,
      "logps/rejected": -494.66705322265625,
      "loss": 0.6275,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.8319900035858154,
      "rewards/margins": 0.5357648730278015,
      "rewards/rejected": -2.367755174636841,
      "step": 6030
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.786412190331775e-06,
      "logits/chosen": -1.1643774509429932,
      "logits/rejected": -0.7012365460395813,
      "logps/chosen": -336.55694580078125,
      "logps/rejected": -370.9845275878906,
      "loss": 0.5543,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.3369648456573486,
      "rewards/margins": 0.7152687311172485,
      "rewards/rejected": -2.0522334575653076,
      "step": 6040
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.781512868436906e-06,
      "logits/chosen": -1.1843137741088867,
      "logits/rejected": -1.1895091533660889,
      "logps/chosen": -248.87582397460938,
      "logps/rejected": -328.52557373046875,
      "loss": 0.4893,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1626508235931396,
      "rewards/margins": 0.7186079025268555,
      "rewards/rejected": -1.8812586069107056,
      "step": 6050
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7766068615431605e-06,
      "logits/chosen": -1.017377495765686,
      "logits/rejected": -0.9144327044487,
      "logps/chosen": -408.78265380859375,
      "logps/rejected": -416.5787048339844,
      "loss": 0.5734,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.5169557332992554,
      "rewards/margins": 0.5643107295036316,
      "rewards/rejected": -2.081266403198242,
      "step": 6060
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.771694195242671e-06,
      "logits/chosen": -1.2855536937713623,
      "logits/rejected": -0.7142298817634583,
      "logps/chosen": -455.6666564941406,
      "logps/rejected": -439.2303771972656,
      "loss": 0.4649,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.564879059791565,
      "rewards/margins": 0.9529463648796082,
      "rewards/rejected": -2.5178253650665283,
      "step": 6070
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.766774895162314e-06,
      "logits/chosen": -0.928503155708313,
      "logits/rejected": -0.9303508996963501,
      "logps/chosen": -422.6429138183594,
      "logps/rejected": -412.329833984375,
      "loss": 0.6427,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.7610315084457397,
      "rewards/margins": 0.45822644233703613,
      "rewards/rejected": -2.2192580699920654,
      "step": 6080
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7618489869635666e-06,
      "logits/chosen": -0.9766615629196167,
      "logits/rejected": -0.8009330630302429,
      "logps/chosen": -423.50152587890625,
      "logps/rejected": -477.24444580078125,
      "loss": 0.6055,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7647453546524048,
      "rewards/margins": 0.7348583936691284,
      "rewards/rejected": -2.499603748321533,
      "step": 6090
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.756916496342379e-06,
      "logits/chosen": -1.1199396848678589,
      "logits/rejected": -1.1375951766967773,
      "logps/chosen": -329.74737548828125,
      "logps/rejected": -424.89373779296875,
      "loss": 0.5633,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4287549257278442,
      "rewards/margins": 0.8669357299804688,
      "rewards/rejected": -2.2956907749176025,
      "step": 6100
    },
    {
      "epoch": 0.4,
      "eval_logits/chosen": -1.014034390449524,
      "eval_logits/rejected": -0.8871217370033264,
      "eval_logps/chosen": -376.6841125488281,
      "eval_logps/rejected": -444.1477355957031,
      "eval_loss": 0.5061184763908386,
      "eval_rewards/accuracies": 0.6604999899864197,
      "eval_rewards/chosen": -1.4467917680740356,
      "eval_rewards/margins": 0.8785668015480042,
      "eval_rewards/rejected": -2.3253581523895264,
      "eval_runtime": 736.7553,
      "eval_samples_per_second": 2.715,
      "eval_steps_per_second": 1.357,
      "step": 6100
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.751977449029039e-06,
      "logits/chosen": -0.8152976036071777,
      "logits/rejected": -0.7364732623100281,
      "logps/chosen": -415.014404296875,
      "logps/rejected": -477.4894104003906,
      "loss": 0.4901,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5493793487548828,
      "rewards/margins": 0.9987069368362427,
      "rewards/rejected": -2.548086643218994,
      "step": 6110
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.747031870788037e-06,
      "logits/chosen": -1.1710433959960938,
      "logits/rejected": -1.0110199451446533,
      "logps/chosen": -455.12298583984375,
      "logps/rejected": -462.619140625,
      "loss": 0.4913,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3836971521377563,
      "rewards/margins": 0.8991546630859375,
      "rewards/rejected": -2.2828516960144043,
      "step": 6120
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7420797874179326e-06,
      "logits/chosen": -0.899192214012146,
      "logits/rejected": -0.8290202021598816,
      "logps/chosen": -364.8775329589844,
      "logps/rejected": -379.16156005859375,
      "loss": 0.5909,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.2776801586151123,
      "rewards/margins": 0.7317312359809875,
      "rewards/rejected": -2.009411334991455,
      "step": 6130
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7371212247512167e-06,
      "logits/chosen": -1.3570548295974731,
      "logits/rejected": -1.0728057622909546,
      "logps/chosen": -430.4407653808594,
      "logps/rejected": -455.63336181640625,
      "loss": 0.4932,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.040462613105774,
      "rewards/margins": 0.9030081629753113,
      "rewards/rejected": -1.9434705972671509,
      "step": 6140
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7321562086541817e-06,
      "logits/chosen": -1.1366050243377686,
      "logits/rejected": -1.0764566659927368,
      "logps/chosen": -367.8480224609375,
      "logps/rejected": -450.5508728027344,
      "loss": 0.506,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1486790180206299,
      "rewards/margins": 0.8191258311271667,
      "rewards/rejected": -1.9678049087524414,
      "step": 6150
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7271847650267834e-06,
      "logits/chosen": -0.965534508228302,
      "logits/rejected": -0.8177278637886047,
      "logps/chosen": -331.1851806640625,
      "logps/rejected": -384.86279296875,
      "loss": 0.5897,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.3125053644180298,
      "rewards/margins": 0.545678973197937,
      "rewards/rejected": -1.8581840991973877,
      "step": 6160
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7222069198025086e-06,
      "logits/chosen": -0.8877931833267212,
      "logits/rejected": -0.7562873959541321,
      "logps/chosen": -348.67169189453125,
      "logps/rejected": -421.09088134765625,
      "loss": 0.4725,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.401292324066162,
      "rewards/margins": 0.9301143884658813,
      "rewards/rejected": -2.331406354904175,
      "step": 6170
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7172226989482353e-06,
      "logits/chosen": -0.8825893402099609,
      "logits/rejected": -0.8212112188339233,
      "logps/chosen": -349.2421875,
      "logps/rejected": -415.50311279296875,
      "loss": 0.571,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.392693281173706,
      "rewards/margins": 0.6840615272521973,
      "rewards/rejected": -2.0767548084259033,
      "step": 6180
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7122321284641007e-06,
      "logits/chosen": -1.1571756601333618,
      "logits/rejected": -0.9974996447563171,
      "logps/chosen": -506.38433837890625,
      "logps/rejected": -488.990234375,
      "loss": 0.4851,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4359768629074097,
      "rewards/margins": 0.9309576749801636,
      "rewards/rejected": -2.366934299468994,
      "step": 6190
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.707235234383365e-06,
      "logits/chosen": -0.8960892558097839,
      "logits/rejected": -0.8411703109741211,
      "logps/chosen": -375.9416198730469,
      "logps/rejected": -374.1454772949219,
      "loss": 0.4512,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.146876573562622,
      "rewards/margins": 0.8498269319534302,
      "rewards/rejected": -1.9967035055160522,
      "step": 6200
    },
    {
      "epoch": 0.41,
      "eval_logits/chosen": -1.0898276567459106,
      "eval_logits/rejected": -0.9586160182952881,
      "eval_logps/chosen": -351.6016845703125,
      "eval_logps/rejected": -419.078857421875,
      "eval_loss": 0.5026857256889343,
      "eval_rewards/accuracies": 0.6589999794960022,
      "eval_rewards/chosen": -1.195967197418213,
      "eval_rewards/margins": 0.8787018656730652,
      "eval_rewards/rejected": -2.0746688842773438,
      "eval_runtime": 732.4564,
      "eval_samples_per_second": 2.731,
      "eval_steps_per_second": 1.365,
      "step": 6200
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.702232042772277e-06,
      "logits/chosen": -1.134568214416504,
      "logits/rejected": -1.0888020992279053,
      "logps/chosen": -348.2904968261719,
      "logps/rejected": -419.7118225097656,
      "loss": 0.455,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4574432373046875,
      "rewards/margins": 0.9377595782279968,
      "rewards/rejected": -2.395203113555908,
      "step": 6210
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6972225797299325e-06,
      "logits/chosen": -1.1086461544036865,
      "logits/rejected": -1.122828483581543,
      "logps/chosen": -408.348876953125,
      "logps/rejected": -480.79833984375,
      "loss": 0.5259,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5784021615982056,
      "rewards/margins": 0.8838506937026978,
      "rewards/rejected": -2.4622528553009033,
      "step": 6220
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.692206871388147e-06,
      "logits/chosen": -1.1035563945770264,
      "logits/rejected": -0.7370472550392151,
      "logps/chosen": -365.0359191894531,
      "logps/rejected": -439.5252990722656,
      "loss": 0.414,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3264720439910889,
      "rewards/margins": 1.157185435295105,
      "rewards/rejected": -2.4836573600769043,
      "step": 6230
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6871849439113115e-06,
      "logits/chosen": -0.5176571011543274,
      "logits/rejected": -0.7378994226455688,
      "logps/chosen": -387.4356994628906,
      "logps/rejected": -446.71826171875,
      "loss": 0.5859,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.5752484798431396,
      "rewards/margins": 0.688303530216217,
      "rewards/rejected": -2.263552188873291,
      "step": 6240
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.682156823496259e-06,
      "logits/chosen": -0.99120032787323,
      "logits/rejected": -0.6283798217773438,
      "logps/chosen": -352.1692810058594,
      "logps/rejected": -434.5711975097656,
      "loss": 0.4444,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3825536966323853,
      "rewards/margins": 1.1476740837097168,
      "rewards/rejected": -2.5302276611328125,
      "step": 6250
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.67712253637213e-06,
      "logits/chosen": -0.9281431436538696,
      "logits/rejected": -0.7577448487281799,
      "logps/chosen": -429.355712890625,
      "logps/rejected": -434.19647216796875,
      "loss": 0.5062,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3706127405166626,
      "rewards/margins": 0.9475326538085938,
      "rewards/rejected": -2.318145275115967,
      "step": 6260
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.672082108800231e-06,
      "logits/chosen": -0.6212013363838196,
      "logits/rejected": -0.7015591859817505,
      "logps/chosen": -430.39727783203125,
      "logps/rejected": -477.31982421875,
      "loss": 0.5381,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.1658148765563965,
      "rewards/margins": 0.8096197247505188,
      "rewards/rejected": -2.9754345417022705,
      "step": 6270
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6670355670739012e-06,
      "logits/chosen": -0.7404322624206543,
      "logits/rejected": -0.6079100370407104,
      "logps/chosen": -349.10504150390625,
      "logps/rejected": -431.7937927246094,
      "loss": 0.4855,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.871516227722168,
      "rewards/margins": 0.8532747030258179,
      "rewards/rejected": -2.7247908115386963,
      "step": 6280
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6619829375183745e-06,
      "logits/chosen": -0.8828223347663879,
      "logits/rejected": -0.7857164144515991,
      "logps/chosen": -449.6969299316406,
      "logps/rejected": -528.0665283203125,
      "loss": 0.5199,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2369437217712402,
      "rewards/margins": 0.9444055557250977,
      "rewards/rejected": -3.181349515914917,
      "step": 6290
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6569242464906427e-06,
      "logits/chosen": -0.7781280279159546,
      "logits/rejected": -0.7232946157455444,
      "logps/chosen": -389.30474853515625,
      "logps/rejected": -515.1699829101562,
      "loss": 0.4765,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.815757393836975,
      "rewards/margins": 1.0481603145599365,
      "rewards/rejected": -2.863917350769043,
      "step": 6300
    },
    {
      "epoch": 0.41,
      "eval_logits/chosen": -0.8425333499908447,
      "eval_logits/rejected": -0.7242295145988464,
      "eval_logps/chosen": -450.2899169921875,
      "eval_logps/rejected": -523.9769897460938,
      "eval_loss": 0.5007634162902832,
      "eval_rewards/accuracies": 0.6654999852180481,
      "eval_rewards/chosen": -2.182849645614624,
      "eval_rewards/margins": 0.9408012628555298,
      "eval_rewards/rejected": -3.1236507892608643,
      "eval_runtime": 733.0976,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 6300
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6518595203793156e-06,
      "logits/chosen": -0.8151119351387024,
      "logits/rejected": -0.6735297441482544,
      "logps/chosen": -491.1490173339844,
      "logps/rejected": -613.0597534179688,
      "loss": 0.4785,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.324347496032715,
      "rewards/margins": 1.0843600034713745,
      "rewards/rejected": -3.4087073802948,
      "step": 6310
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.646788785604485e-06,
      "logits/chosen": -0.9957554936408997,
      "logits/rejected": -1.0441629886627197,
      "logps/chosen": -402.04180908203125,
      "logps/rejected": -468.99884033203125,
      "loss": 0.6281,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.9638341665267944,
      "rewards/margins": 0.692020058631897,
      "rewards/rejected": -2.6558539867401123,
      "step": 6320
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.641712068617588e-06,
      "logits/chosen": -0.8832969665527344,
      "logits/rejected": -0.76265549659729,
      "logps/chosen": -461.4491271972656,
      "logps/rejected": -495.14862060546875,
      "loss": 0.4246,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.0008909702301025,
      "rewards/margins": 0.8724395036697388,
      "rewards/rejected": -2.873330593109131,
      "step": 6330
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6366293959012673e-06,
      "logits/chosen": -0.7737056016921997,
      "logits/rejected": -0.6211354732513428,
      "logps/chosen": -356.0001220703125,
      "logps/rejected": -422.6172790527344,
      "loss": 0.4797,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.747981309890747,
      "rewards/margins": 0.8614038228988647,
      "rewards/rejected": -2.6093852519989014,
      "step": 6340
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.631540793969233e-06,
      "logits/chosen": -1.336551308631897,
      "logits/rejected": -1.1524276733398438,
      "logps/chosen": -355.12652587890625,
      "logps/rejected": -429.01641845703125,
      "loss": 0.4994,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.6650876998901367,
      "rewards/margins": 0.7221162915229797,
      "rewards/rejected": -2.3872039318084717,
      "step": 6350
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.626446289366127e-06,
      "logits/chosen": -1.2687801122665405,
      "logits/rejected": -1.0421772003173828,
      "logps/chosen": -415.6885681152344,
      "logps/rejected": -421.1853942871094,
      "loss": 0.5333,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9553935527801514,
      "rewards/margins": 0.7701565027236938,
      "rewards/rejected": -2.7255501747131348,
      "step": 6360
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6213459086673786e-06,
      "logits/chosen": -1.239116907119751,
      "logits/rejected": -1.2933433055877686,
      "logps/chosen": -342.5483703613281,
      "logps/rejected": -441.9959411621094,
      "loss": 0.5679,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7161080837249756,
      "rewards/margins": 0.9007428884506226,
      "rewards/rejected": -2.6168508529663086,
      "step": 6370
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6162396784790737e-06,
      "logits/chosen": -1.0483715534210205,
      "logits/rejected": -0.9412117004394531,
      "logps/chosen": -366.8360900878906,
      "logps/rejected": -427.07275390625,
      "loss": 0.5743,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4382898807525635,
      "rewards/margins": 0.6039630174636841,
      "rewards/rejected": -2.042252779006958,
      "step": 6380
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6111276254378095e-06,
      "logits/chosen": -1.2586199045181274,
      "logits/rejected": -1.1685311794281006,
      "logps/chosen": -372.86676025390625,
      "logps/rejected": -455.90179443359375,
      "loss": 0.5039,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4490987062454224,
      "rewards/margins": 0.8744858503341675,
      "rewards/rejected": -2.32358455657959,
      "step": 6390
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.606009776210559e-06,
      "logits/chosen": -1.1810599565505981,
      "logits/rejected": -1.1550124883651733,
      "logps/chosen": -443.35546875,
      "logps/rejected": -485.1634216308594,
      "loss": 0.5056,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.027317762374878,
      "rewards/margins": 0.8546678423881531,
      "rewards/rejected": -2.8819851875305176,
      "step": 6400
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": -1.1095459461212158,
      "eval_logits/rejected": -0.981111466884613,
      "eval_logps/chosen": -404.58245849609375,
      "eval_logps/rejected": -472.86614990234375,
      "eval_loss": 0.5051407217979431,
      "eval_rewards/accuracies": 0.6589999794960022,
      "eval_rewards/chosen": -1.7257753610610962,
      "eval_rewards/margins": 0.8867669701576233,
      "eval_rewards/rejected": -2.6125423908233643,
      "eval_runtime": 735.605,
      "eval_samples_per_second": 2.719,
      "eval_steps_per_second": 1.359,
      "step": 6400
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.600886157494531e-06,
      "logits/chosen": -1.3280621767044067,
      "logits/rejected": -1.2239741086959839,
      "logps/chosen": -441.26220703125,
      "logps/rejected": -522.0778198242188,
      "loss": 0.473,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7840566635131836,
      "rewards/margins": 0.9259775280952454,
      "rewards/rejected": -2.710033893585205,
      "step": 6410
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5957567960170304e-06,
      "logits/chosen": -1.319632887840271,
      "logits/rejected": -0.8672634959220886,
      "logps/chosen": -453.4800720214844,
      "logps/rejected": -438.85455322265625,
      "loss": 0.4254,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6229498386383057,
      "rewards/margins": 1.040321946144104,
      "rewards/rejected": -2.663271427154541,
      "step": 6420
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.590621718535319e-06,
      "logits/chosen": -0.9140608906745911,
      "logits/rejected": -0.8867174983024597,
      "logps/chosen": -370.87628173828125,
      "logps/rejected": -481.146484375,
      "loss": 0.5087,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7415390014648438,
      "rewards/margins": 1.1116279363632202,
      "rewards/rejected": -2.8531670570373535,
      "step": 6430
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5854809518364775e-06,
      "logits/chosen": -1.211717963218689,
      "logits/rejected": -1.0965769290924072,
      "logps/chosen": -388.0635681152344,
      "logps/rejected": -460.0933532714844,
      "loss": 0.4406,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4191679954528809,
      "rewards/margins": 1.1654255390167236,
      "rewards/rejected": -2.5845935344696045,
      "step": 6440
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.580334522737262e-06,
      "logits/chosen": -1.0971286296844482,
      "logits/rejected": -1.0055302381515503,
      "logps/chosen": -364.49560546875,
      "logps/rejected": -415.90679931640625,
      "loss": 0.5878,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6255121231079102,
      "rewards/margins": 0.8506077527999878,
      "rewards/rejected": -2.4761199951171875,
      "step": 6450
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.575182458083968e-06,
      "logits/chosen": -1.1140817403793335,
      "logits/rejected": -1.0681989192962646,
      "logps/chosen": -379.82159423828125,
      "logps/rejected": -475.5859375,
      "loss": 0.4182,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3901653289794922,
      "rewards/margins": 1.237693190574646,
      "rewards/rejected": -2.6278586387634277,
      "step": 6460
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5700247847522883e-06,
      "logits/chosen": -1.2970528602600098,
      "logits/rejected": -1.2003545761108398,
      "logps/chosen": -349.5565185546875,
      "logps/rejected": -463.45306396484375,
      "loss": 0.5227,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4675853252410889,
      "rewards/margins": 1.1391479969024658,
      "rewards/rejected": -2.6067328453063965,
      "step": 6470
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5648615296471743e-06,
      "logits/chosen": -1.103219985961914,
      "logits/rejected": -1.0351829528808594,
      "logps/chosen": -392.39373779296875,
      "logps/rejected": -552.31494140625,
      "loss": 0.4326,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9503090381622314,
      "rewards/margins": 1.2184739112854004,
      "rewards/rejected": -3.168782949447632,
      "step": 6480
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.559692719702693e-06,
      "logits/chosen": -0.9401780366897583,
      "logits/rejected": -0.7638431191444397,
      "logps/chosen": -493.917236328125,
      "logps/rejected": -564.3541259765625,
      "loss": 0.481,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.0777204036712646,
      "rewards/margins": 1.2716641426086426,
      "rewards/rejected": -3.349384307861328,
      "step": 6490
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.55451838188189e-06,
      "logits/chosen": -1.2014122009277344,
      "logits/rejected": -1.2170169353485107,
      "logps/chosen": -447.1665954589844,
      "logps/rejected": -580.5740356445312,
      "loss": 0.5037,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8587976694107056,
      "rewards/margins": 1.1147427558898926,
      "rewards/rejected": -2.9735400676727295,
      "step": 6500
    },
    {
      "epoch": 0.43,
      "eval_logits/chosen": -1.077275276184082,
      "eval_logits/rejected": -0.9466902613639832,
      "eval_logps/chosen": -469.41241455078125,
      "eval_logps/rejected": -541.4144897460938,
      "eval_loss": 0.5052820444107056,
      "eval_rewards/accuracies": 0.6644999980926514,
      "eval_rewards/chosen": -2.3740742206573486,
      "eval_rewards/margins": 0.9239515066146851,
      "eval_rewards/rejected": -3.2980258464813232,
      "eval_runtime": 734.7702,
      "eval_samples_per_second": 2.722,
      "eval_steps_per_second": 1.361,
      "step": 6500
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.549338543176645e-06,
      "logits/chosen": -1.1520912647247314,
      "logits/rejected": -1.0121804475784302,
      "logps/chosen": -548.9495849609375,
      "logps/rejected": -615.66064453125,
      "loss": 0.5094,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.4246106147766113,
      "rewards/margins": 0.9906423687934875,
      "rewards/rejected": -3.415252685546875,
      "step": 6510
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5441532306075342e-06,
      "logits/chosen": -1.2366782426834106,
      "logits/rejected": -1.1612181663513184,
      "logps/chosen": -460.84844970703125,
      "logps/rejected": -562.7225952148438,
      "loss": 0.6087,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.294292449951172,
      "rewards/margins": 0.5530889630317688,
      "rewards/rejected": -2.847381114959717,
      "step": 6520
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5389624712236894e-06,
      "logits/chosen": -1.060490369796753,
      "logits/rejected": -0.873035728931427,
      "logps/chosen": -434.99249267578125,
      "logps/rejected": -811.9586181640625,
      "loss": 0.6352,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.209566831588745,
      "rewards/margins": 4.038938999176025,
      "rewards/rejected": -6.248506546020508,
      "step": 6530
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.533766292102653e-06,
      "logits/chosen": -0.9152925610542297,
      "logits/rejected": -1.015721082687378,
      "logps/chosen": -447.19976806640625,
      "logps/rejected": -503.60736083984375,
      "loss": 0.549,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.373551607131958,
      "rewards/margins": 0.6809632182121277,
      "rewards/rejected": -3.0545151233673096,
      "step": 6540
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5285647203502404e-06,
      "logits/chosen": -1.4056029319763184,
      "logits/rejected": -1.2151262760162354,
      "logps/chosen": -455.80096435546875,
      "logps/rejected": -490.271484375,
      "loss": 0.5012,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9852997064590454,
      "rewards/margins": 0.7223653793334961,
      "rewards/rejected": -2.707664966583252,
      "step": 6550
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5233577831003983e-06,
      "logits/chosen": -1.2194260358810425,
      "logits/rejected": -1.0652081966400146,
      "logps/chosen": -435.7723693847656,
      "logps/rejected": -467.86370849609375,
      "loss": 0.5666,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.8359639644622803,
      "rewards/margins": 0.5576001405715942,
      "rewards/rejected": -2.393564462661743,
      "step": 6560
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5181455075150628e-06,
      "logits/chosen": -1.1756420135498047,
      "logits/rejected": -0.9271238446235657,
      "logps/chosen": -322.8921203613281,
      "logps/rejected": -347.01171875,
      "loss": 0.5241,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4094593524932861,
      "rewards/margins": 0.7074254155158997,
      "rewards/rejected": -2.116884708404541,
      "step": 6570
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.512927920784016e-06,
      "logits/chosen": -1.3403713703155518,
      "logits/rejected": -1.1953039169311523,
      "logps/chosen": -358.7806091308594,
      "logps/rejected": -436.91107177734375,
      "loss": 0.4811,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.33106529712677,
      "rewards/margins": 0.981104850769043,
      "rewards/rejected": -2.3121702671051025,
      "step": 6580
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5077050501247457e-06,
      "logits/chosen": -1.4561182260513306,
      "logits/rejected": -1.0121886730194092,
      "logps/chosen": -415.90350341796875,
      "logps/rejected": -451.1192321777344,
      "loss": 0.4661,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3488860130310059,
      "rewards/margins": 1.0260428190231323,
      "rewards/rejected": -2.3749289512634277,
      "step": 6590
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5024769227823042e-06,
      "logits/chosen": -1.4400004148483276,
      "logits/rejected": -1.24789559841156,
      "logps/chosen": -304.3248291015625,
      "logps/rejected": -340.4151916503906,
      "loss": 0.5839,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.4145349264144897,
      "rewards/margins": 0.7688801288604736,
      "rewards/rejected": -2.183415174484253,
      "step": 6600
    },
    {
      "epoch": 0.43,
      "eval_logits/chosen": -1.3891085386276245,
      "eval_logits/rejected": -1.2408503293991089,
      "eval_logps/chosen": -375.1404724121094,
      "eval_logps/rejected": -446.2347412109375,
      "eval_loss": 0.5008581876754761,
      "eval_rewards/accuracies": 0.6710000038146973,
      "eval_rewards/chosen": -1.4313548803329468,
      "eval_rewards/margins": 0.914873480796814,
      "eval_rewards/rejected": -2.34622859954834,
      "eval_runtime": 730.2882,
      "eval_samples_per_second": 2.739,
      "eval_steps_per_second": 1.369,
      "step": 6600
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.4972435660291646e-06,
      "logits/chosen": -1.5686732530593872,
      "logits/rejected": -1.5094807147979736,
      "logps/chosen": -399.5616149902344,
      "logps/rejected": -459.87786865234375,
      "loss": 0.5037,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5641734600067139,
      "rewards/margins": 0.8525595664978027,
      "rewards/rejected": -2.4167332649230957,
      "step": 6610
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.492005007165079e-06,
      "logits/chosen": -1.266118049621582,
      "logits/rejected": -1.173451542854309,
      "logps/chosen": -376.46588134765625,
      "logps/rejected": -453.89404296875,
      "loss": 0.5533,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4692888259887695,
      "rewards/margins": 0.712715744972229,
      "rewards/rejected": -2.182004451751709,
      "step": 6620
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.4867612735169377e-06,
      "logits/chosen": -1.5591437816619873,
      "logits/rejected": -1.1772428750991821,
      "logps/chosen": -383.3150329589844,
      "logps/rejected": -404.107666015625,
      "loss": 0.459,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5828063488006592,
      "rewards/margins": 0.9902087450027466,
      "rewards/rejected": -2.573014974594116,
      "step": 6630
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.4815123924386226e-06,
      "logits/chosen": -1.6907329559326172,
      "logits/rejected": -1.3803856372833252,
      "logps/chosen": -476.39520263671875,
      "logps/rejected": -472.42462158203125,
      "loss": 0.5546,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.6985385417938232,
      "rewards/margins": 0.6570184230804443,
      "rewards/rejected": -2.3555569648742676,
      "step": 6640
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4762583913108696e-06,
      "logits/chosen": -1.1576273441314697,
      "logits/rejected": -1.02208411693573,
      "logps/chosen": -449.2685546875,
      "logps/rejected": -480.30426025390625,
      "loss": 0.61,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7951021194458008,
      "rewards/margins": 0.719508707523346,
      "rewards/rejected": -2.514610767364502,
      "step": 6650
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4709992975411217e-06,
      "logits/chosen": -1.2916300296783447,
      "logits/rejected": -0.974404513835907,
      "logps/chosen": -461.3033142089844,
      "logps/rejected": -503.5069885253906,
      "loss": 0.4928,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.063438892364502,
      "rewards/margins": 0.9447065591812134,
      "rewards/rejected": -3.008145570755005,
      "step": 6660
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4657351385633886e-06,
      "logits/chosen": -1.3369910717010498,
      "logits/rejected": -1.203258752822876,
      "logps/chosen": -363.0143737792969,
      "logps/rejected": -436.9532165527344,
      "loss": 0.4847,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7772363424301147,
      "rewards/margins": 0.8613905906677246,
      "rewards/rejected": -2.638627052307129,
      "step": 6670
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4604659418381024e-06,
      "logits/chosen": -1.439807653427124,
      "logits/rejected": -1.0867663621902466,
      "logps/chosen": -436.64892578125,
      "logps/rejected": -499.2110290527344,
      "loss": 0.5199,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.185945987701416,
      "rewards/margins": 0.9209274053573608,
      "rewards/rejected": -3.1068735122680664,
      "step": 6680
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4551917348519744e-06,
      "logits/chosen": -1.4486407041549683,
      "logits/rejected": -1.2376219034194946,
      "logps/chosen": -458.61328125,
      "logps/rejected": -488.44256591796875,
      "loss": 0.4949,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7711458206176758,
      "rewards/margins": 0.770298182964325,
      "rewards/rejected": -2.5414438247680664,
      "step": 6690
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4499125451178505e-06,
      "logits/chosen": -1.0467464923858643,
      "logits/rejected": -1.0982073545455933,
      "logps/chosen": -404.7933654785156,
      "logps/rejected": -490.70892333984375,
      "loss": 0.6173,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.0050206184387207,
      "rewards/margins": 0.6130286455154419,
      "rewards/rejected": -2.618049144744873,
      "step": 6700
    },
    {
      "epoch": 0.44,
      "eval_logits/chosen": -1.3957716226577759,
      "eval_logits/rejected": -1.2478077411651611,
      "eval_logps/chosen": -415.9501953125,
      "eval_logps/rejected": -482.2916259765625,
      "eval_loss": 0.5004130601882935,
      "eval_rewards/accuracies": 0.6694999933242798,
      "eval_rewards/chosen": -1.8394523859024048,
      "eval_rewards/margins": 0.8673450946807861,
      "eval_rewards/rejected": -2.7067973613739014,
      "eval_runtime": 733.2017,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 6700
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4446284001745723e-06,
      "logits/chosen": -1.0857712030410767,
      "logits/rejected": -0.9940765500068665,
      "logps/chosen": -417.3369140625,
      "logps/rejected": -508.9224548339844,
      "loss": 0.5722,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.111032485961914,
      "rewards/margins": 0.7483859658241272,
      "rewards/rejected": -2.8594183921813965,
      "step": 6710
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.439339327586827e-06,
      "logits/chosen": -1.339019536972046,
      "logits/rejected": -1.3519829511642456,
      "logps/chosen": -337.61767578125,
      "logps/rejected": -425.98663330078125,
      "loss": 0.4921,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4759998321533203,
      "rewards/margins": 0.9373260736465454,
      "rewards/rejected": -2.413325786590576,
      "step": 6720
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.434045354945008e-06,
      "logits/chosen": -1.425730586051941,
      "logits/rejected": -1.2801324129104614,
      "logps/chosen": -512.8795166015625,
      "logps/rejected": -581.2686157226562,
      "loss": 0.5462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.388549327850342,
      "rewards/margins": 0.7312488555908203,
      "rewards/rejected": -3.119798183441162,
      "step": 6730
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4287465098650713e-06,
      "logits/chosen": -1.6729755401611328,
      "logits/rejected": -1.5117638111114502,
      "logps/chosen": -432.8279724121094,
      "logps/rejected": -494.1000061035156,
      "loss": 0.5091,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8485244512557983,
      "rewards/margins": 0.7611358761787415,
      "rewards/rejected": -2.6096603870391846,
      "step": 6740
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.423442819988387e-06,
      "logits/chosen": -1.42362380027771,
      "logits/rejected": -1.2848873138427734,
      "logps/chosen": -374.7146301269531,
      "logps/rejected": -472.390869140625,
      "loss": 0.3997,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.9132734537124634,
      "rewards/margins": 1.0486565828323364,
      "rewards/rejected": -2.961930274963379,
      "step": 6750
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4181343129816e-06,
      "logits/chosen": -1.4292211532592773,
      "logits/rejected": -1.2701166868209839,
      "logps/chosen": -337.1582946777344,
      "logps/rejected": -387.37823486328125,
      "loss": 0.6429,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.6079658269882202,
      "rewards/margins": 0.6451005339622498,
      "rewards/rejected": -2.2530665397644043,
      "step": 6760
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4128210165364837e-06,
      "logits/chosen": -1.2692002058029175,
      "logits/rejected": -1.123100996017456,
      "logps/chosen": -384.36895751953125,
      "logps/rejected": -498.984619140625,
      "loss": 0.4937,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.92959725856781,
      "rewards/margins": 1.060922384262085,
      "rewards/rejected": -2.9905197620391846,
      "step": 6770
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.407502958369795e-06,
      "logits/chosen": -1.3931758403778076,
      "logits/rejected": -1.2321561574935913,
      "logps/chosen": -426.935546875,
      "logps/rejected": -491.1451110839844,
      "loss": 0.5404,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.844877004623413,
      "rewards/margins": 0.9342195391654968,
      "rewards/rejected": -2.779096841812134,
      "step": 6780
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4021801662231297e-06,
      "logits/chosen": -1.4039561748504639,
      "logits/rejected": -1.240965485572815,
      "logps/chosen": -422.8265075683594,
      "logps/rejected": -477.4052734375,
      "loss": 0.5433,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7394969463348389,
      "rewards/margins": 0.768541693687439,
      "rewards/rejected": -2.5080389976501465,
      "step": 6790
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.3968526678627793e-06,
      "logits/chosen": -1.2355637550354004,
      "logits/rejected": -1.001617193222046,
      "logps/chosen": -424.4659729003906,
      "logps/rejected": -476.471923828125,
      "loss": 0.4917,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5943549871444702,
      "rewards/margins": 0.9610947370529175,
      "rewards/rejected": -2.5554497241973877,
      "step": 6800
    },
    {
      "epoch": 0.44,
      "eval_logits/chosen": -1.3386211395263672,
      "eval_logits/rejected": -1.1951630115509033,
      "eval_logps/chosen": -412.70941162109375,
      "eval_logps/rejected": -478.1150207519531,
      "eval_loss": 0.4987241327762604,
      "eval_rewards/accuracies": 0.6669999957084656,
      "eval_rewards/chosen": -1.8070447444915771,
      "eval_rewards/margins": 0.8579861521720886,
      "eval_rewards/rejected": -2.6650311946868896,
      "eval_runtime": 730.6242,
      "eval_samples_per_second": 2.737,
      "eval_steps_per_second": 1.369,
      "step": 6800
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.391520491079586e-06,
      "logits/chosen": -1.6253728866577148,
      "logits/rejected": -1.391944408416748,
      "logps/chosen": -357.5389099121094,
      "logps/rejected": -396.3597717285156,
      "loss": 0.5152,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6157629489898682,
      "rewards/margins": 0.6935049891471863,
      "rewards/rejected": -2.309267997741699,
      "step": 6810
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3861836636887936e-06,
      "logits/chosen": -1.349249005317688,
      "logits/rejected": -1.1007931232452393,
      "logps/chosen": -462.9054260253906,
      "logps/rejected": -492.6299743652344,
      "loss": 0.5324,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9608653783798218,
      "rewards/margins": 0.762030303478241,
      "rewards/rejected": -2.722895860671997,
      "step": 6820
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3808422135299106e-06,
      "logits/chosen": -1.238844633102417,
      "logits/rejected": -1.213136911392212,
      "logps/chosen": -504.2168884277344,
      "logps/rejected": -639.5765380859375,
      "loss": 0.4731,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1853911876678467,
      "rewards/margins": 0.8228403925895691,
      "rewards/rejected": -3.0082316398620605,
      "step": 6830
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.375496168466556e-06,
      "logits/chosen": -1.2250182628631592,
      "logits/rejected": -0.9360244870185852,
      "logps/chosen": -422.16705322265625,
      "logps/rejected": -448.4495544433594,
      "loss": 0.5549,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.072289228439331,
      "rewards/margins": 0.8220914006233215,
      "rewards/rejected": -2.894380569458008,
      "step": 6840
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3701455563863205e-06,
      "logits/chosen": -1.42123544216156,
      "logits/rejected": -1.2015262842178345,
      "logps/chosen": -553.3240966796875,
      "logps/rejected": -624.6856689453125,
      "loss": 0.5084,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.6144938468933105,
      "rewards/margins": 1.0768070220947266,
      "rewards/rejected": -3.691300630569458,
      "step": 6850
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3647904052006174e-06,
      "logits/chosen": -1.174742341041565,
      "logits/rejected": -1.0763640403747559,
      "logps/chosen": -518.8255615234375,
      "logps/rejected": -612.1888427734375,
      "loss": 0.4688,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.5352730751037598,
      "rewards/margins": 0.9035671949386597,
      "rewards/rejected": -3.43884015083313,
      "step": 6860
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3594307428445383e-06,
      "logits/chosen": -1.319272756576538,
      "logits/rejected": -0.9488700032234192,
      "logps/chosen": -597.383544921875,
      "logps/rejected": -680.8193969726562,
      "loss": 0.4074,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.641327381134033,
      "rewards/margins": 1.1485236883163452,
      "rewards/rejected": -3.789851427078247,
      "step": 6870
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.354066597276707e-06,
      "logits/chosen": -0.8110232353210449,
      "logits/rejected": -0.757287323474884,
      "logps/chosen": -478.07330322265625,
      "logps/rejected": -609.6417236328125,
      "loss": 0.517,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.5450901985168457,
      "rewards/margins": 0.9620795249938965,
      "rewards/rejected": -3.507169723510742,
      "step": 6880
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.348697996479136e-06,
      "logits/chosen": -0.9675430059432983,
      "logits/rejected": -0.79949551820755,
      "logps/chosen": -529.7903442382812,
      "logps/rejected": -559.5513305664062,
      "loss": 0.5838,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -3.0198137760162354,
      "rewards/margins": 0.8323270082473755,
      "rewards/rejected": -3.8521409034729004,
      "step": 6890
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3433249684570757e-06,
      "logits/chosen": -0.7183124423027039,
      "logits/rejected": -0.524259090423584,
      "logps/chosen": -416.25115966796875,
      "logps/rejected": -479.09478759765625,
      "loss": 0.4834,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.384216070175171,
      "rewards/margins": 1.0856199264526367,
      "rewards/rejected": -3.469836473464966,
      "step": 6900
    },
    {
      "epoch": 0.45,
      "eval_logits/chosen": -0.9490465521812439,
      "eval_logits/rejected": -0.8229533433914185,
      "eval_logps/chosen": -473.6738586425781,
      "eval_logps/rejected": -550.5955200195312,
      "eval_loss": 0.4964263439178467,
      "eval_rewards/accuracies": 0.6679999828338623,
      "eval_rewards/chosen": -2.4166886806488037,
      "eval_rewards/margins": 0.9731472730636597,
      "eval_rewards/rejected": -3.389836072921753,
      "eval_runtime": 733.502,
      "eval_samples_per_second": 2.727,
      "eval_steps_per_second": 1.363,
      "step": 6900
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3379475412388724e-06,
      "logits/chosen": -1.0790557861328125,
      "logits/rejected": -0.9136208295822144,
      "logps/chosen": -475.306640625,
      "logps/rejected": -564.6021728515625,
      "loss": 0.3662,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.3335165977478027,
      "rewards/margins": 1.22994065284729,
      "rewards/rejected": -3.563457489013672,
      "step": 6910
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3325657428758207e-06,
      "logits/chosen": -0.9100483059883118,
      "logits/rejected": -0.9243875741958618,
      "logps/chosen": -459.96026611328125,
      "logps/rejected": -571.3048095703125,
      "loss": 0.5337,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1628475189208984,
      "rewards/margins": 1.1051790714263916,
      "rewards/rejected": -3.268026351928711,
      "step": 6920
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3271796014420175e-06,
      "logits/chosen": -1.1402623653411865,
      "logits/rejected": -0.8654245138168335,
      "logps/chosen": -442.6941833496094,
      "logps/rejected": -542.1939697265625,
      "loss": 0.4406,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.302438735961914,
      "rewards/margins": 1.1720813512802124,
      "rewards/rejected": -3.474520206451416,
      "step": 6930
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3217891450342142e-06,
      "logits/chosen": -0.9778724908828735,
      "logits/rejected": -0.8319625854492188,
      "logps/chosen": -456.13482666015625,
      "logps/rejected": -527.78759765625,
      "loss": 0.4142,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9753639698028564,
      "rewards/margins": 1.4064559936523438,
      "rewards/rejected": -3.3818199634552,
      "step": 6940
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3163944017716733e-06,
      "logits/chosen": -1.2695777416229248,
      "logits/rejected": -1.0632556676864624,
      "logps/chosen": -419.833740234375,
      "logps/rejected": -480.20281982421875,
      "loss": 0.5368,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.0346288681030273,
      "rewards/margins": 0.9346679449081421,
      "rewards/rejected": -2.96929669380188,
      "step": 6950
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.310995399796017e-06,
      "logits/chosen": -1.276155710220337,
      "logits/rejected": -1.2048354148864746,
      "logps/chosen": -510.0116271972656,
      "logps/rejected": -541.8460083007812,
      "loss": 0.8078,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.3602535724639893,
      "rewards/margins": 0.38636505603790283,
      "rewards/rejected": -2.7466189861297607,
      "step": 6960
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.305592167271085e-06,
      "logits/chosen": -1.2273608446121216,
      "logits/rejected": -1.0127642154693604,
      "logps/chosen": -371.83209228515625,
      "logps/rejected": -457.48052978515625,
      "loss": 0.4819,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.745859146118164,
      "rewards/margins": 0.9656050801277161,
      "rewards/rejected": -2.7114641666412354,
      "step": 6970
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.3001847323827846e-06,
      "logits/chosen": -1.2883787155151367,
      "logits/rejected": -1.3649321794509888,
      "logps/chosen": -454.60198974609375,
      "logps/rejected": -536.0913696289062,
      "loss": 0.5006,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8650429248809814,
      "rewards/margins": 0.8490816354751587,
      "rewards/rejected": -2.7141242027282715,
      "step": 6980
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2947731233389447e-06,
      "logits/chosen": -1.218878984451294,
      "logits/rejected": -0.999015212059021,
      "logps/chosen": -427.6170959472656,
      "logps/rejected": -480.9115295410156,
      "loss": 0.4415,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.801103949546814,
      "rewards/margins": 1.0872182846069336,
      "rewards/rejected": -2.888322353363037,
      "step": 6990
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2893573683691706e-06,
      "logits/chosen": -1.1505297422409058,
      "logits/rejected": -1.0928208827972412,
      "logps/chosen": -368.4134521484375,
      "logps/rejected": -443.3638610839844,
      "loss": 0.4668,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.6371997594833374,
      "rewards/margins": 0.9284623861312866,
      "rewards/rejected": -2.565662384033203,
      "step": 7000
    },
    {
      "epoch": 0.46,
      "eval_logits/chosen": -1.265947699546814,
      "eval_logits/rejected": -1.1271545886993408,
      "eval_logps/chosen": -399.3541259765625,
      "eval_logps/rejected": -466.1047058105469,
      "eval_loss": 0.50328129529953,
      "eval_rewards/accuracies": 0.6700000166893005,
      "eval_rewards/chosen": -1.6734919548034668,
      "eval_rewards/margins": 0.8714357614517212,
      "eval_rewards/rejected": -2.5449280738830566,
      "eval_runtime": 733.2514,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 7000
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2839374957246915e-06,
      "logits/chosen": -1.3625187873840332,
      "logits/rejected": -1.1006382703781128,
      "logps/chosen": -453.134033203125,
      "logps/rejected": -413.80291748046875,
      "loss": 0.6148,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8704373836517334,
      "rewards/margins": 0.5220705270767212,
      "rewards/rejected": -2.392508029937744,
      "step": 7010
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2785135336782187e-06,
      "logits/chosen": -1.184256911277771,
      "logits/rejected": -1.0579241514205933,
      "logps/chosen": -420.9827575683594,
      "logps/rejected": -543.4996948242188,
      "loss": 0.4591,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8706735372543335,
      "rewards/margins": 1.0020452737808228,
      "rewards/rejected": -2.872718572616577,
      "step": 7020
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2730855105237952e-06,
      "logits/chosen": -1.2827327251434326,
      "logits/rejected": -1.1696199178695679,
      "logps/chosen": -378.5633544921875,
      "logps/rejected": -528.2980346679688,
      "loss": 0.4283,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5828688144683838,
      "rewards/margins": 1.055541753768921,
      "rewards/rejected": -2.638411045074463,
      "step": 7030
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2676534545766486e-06,
      "logits/chosen": -1.1060223579406738,
      "logits/rejected": -0.9897655248641968,
      "logps/chosen": -374.9141540527344,
      "logps/rejected": -429.4541015625,
      "loss": 0.5823,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.624446153640747,
      "rewards/margins": 0.6443280577659607,
      "rewards/rejected": -2.2687745094299316,
      "step": 7040
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.262217394173043e-06,
      "logits/chosen": -1.0637996196746826,
      "logits/rejected": -0.9355257749557495,
      "logps/chosen": -403.6563415527344,
      "logps/rejected": -503.54949951171875,
      "loss": 0.5271,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.6280568838119507,
      "rewards/margins": 1.0597625970840454,
      "rewards/rejected": -2.687819719314575,
      "step": 7050
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2567773576701333e-06,
      "logits/chosen": -1.0611984729766846,
      "logits/rejected": -0.9134718179702759,
      "logps/chosen": -395.8488464355469,
      "logps/rejected": -480.60406494140625,
      "loss": 0.4829,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4009135961532593,
      "rewards/margins": 1.1507080793380737,
      "rewards/rejected": -2.551621675491333,
      "step": 7060
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2513333734458154e-06,
      "logits/chosen": -1.145353078842163,
      "logits/rejected": -0.9989140629768372,
      "logps/chosen": -377.0892639160156,
      "logps/rejected": -417.6884765625,
      "loss": 0.5898,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.702479600906372,
      "rewards/margins": 0.5720685720443726,
      "rewards/rejected": -2.274547815322876,
      "step": 7070
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.245885469898576e-06,
      "logits/chosen": -1.0995618104934692,
      "logits/rejected": -1.0788781642913818,
      "logps/chosen": -447.03076171875,
      "logps/rejected": -490.30157470703125,
      "loss": 0.4897,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.464755892753601,
      "rewards/margins": 1.069588303565979,
      "rewards/rejected": -2.534344434738159,
      "step": 7080
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2404336754473497e-06,
      "logits/chosen": -1.1354840993881226,
      "logits/rejected": -0.9494547843933105,
      "logps/chosen": -395.81268310546875,
      "logps/rejected": -423.93695068359375,
      "loss": 0.4502,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2830572128295898,
      "rewards/margins": 0.9289452433586121,
      "rewards/rejected": -2.2120022773742676,
      "step": 7090
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.234978018531367e-06,
      "logits/chosen": -1.5512938499450684,
      "logits/rejected": -1.0924882888793945,
      "logps/chosen": -404.32403564453125,
      "logps/rejected": -448.44610595703125,
      "loss": 0.4544,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.458237886428833,
      "rewards/margins": 1.089217185974121,
      "rewards/rejected": -2.547455310821533,
      "step": 7100
    },
    {
      "epoch": 0.46,
      "eval_logits/chosen": -1.0685359239578247,
      "eval_logits/rejected": -0.9393064379692078,
      "eval_logps/chosen": -391.1265563964844,
      "eval_logps/rejected": -470.7079772949219,
      "eval_loss": 0.4963315725326538,
      "eval_rewards/accuracies": 0.671500027179718,
      "eval_rewards/chosen": -1.591215968132019,
      "eval_rewards/margins": 0.9997445940971375,
      "eval_rewards/rejected": -2.5909605026245117,
      "eval_runtime": 730.6695,
      "eval_samples_per_second": 2.737,
      "eval_steps_per_second": 1.369,
      "step": 7100
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.229518527610006e-06,
      "logits/chosen": -1.2886658906936646,
      "logits/rejected": -1.1227843761444092,
      "logps/chosen": -484.5293884277344,
      "logps/rejected": -517.5733642578125,
      "loss": 0.5212,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9173930883407593,
      "rewards/margins": 0.7821990251541138,
      "rewards/rejected": -2.699592113494873,
      "step": 7110
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2240552311626465e-06,
      "logits/chosen": -1.097480058670044,
      "logits/rejected": -0.9205149412155151,
      "logps/chosen": -396.7735900878906,
      "logps/rejected": -465.60107421875,
      "loss": 0.5388,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5555986166000366,
      "rewards/margins": 0.837724506855011,
      "rewards/rejected": -2.3933229446411133,
      "step": 7120
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2185881576885193e-06,
      "logits/chosen": -1.0562323331832886,
      "logits/rejected": -0.9030343890190125,
      "logps/chosen": -423.342041015625,
      "logps/rejected": -465.4990234375,
      "loss": 0.5439,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.116192102432251,
      "rewards/margins": 0.8082444071769714,
      "rewards/rejected": -2.924436569213867,
      "step": 7130
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.213117335706557e-06,
      "logits/chosen": -1.1086636781692505,
      "logits/rejected": -1.045616865158081,
      "logps/chosen": -455.88922119140625,
      "logps/rejected": -570.3858642578125,
      "loss": 0.5255,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9984424114227295,
      "rewards/margins": 1.0529043674468994,
      "rewards/rejected": -3.05134654045105,
      "step": 7140
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2076427937552473e-06,
      "logits/chosen": -1.0971708297729492,
      "logits/rejected": -0.7860563397407532,
      "logps/chosen": -406.85443115234375,
      "logps/rejected": -514.7252197265625,
      "loss": 0.4371,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6136468648910522,
      "rewards/margins": 1.2699447870254517,
      "rewards/rejected": -2.883591651916504,
      "step": 7150
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2021645603924827e-06,
      "logits/chosen": -0.8555929064750671,
      "logits/rejected": -0.8638184666633606,
      "logps/chosen": -283.649658203125,
      "logps/rejected": -406.4692077636719,
      "loss": 0.5197,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4519553184509277,
      "rewards/margins": 1.1345512866973877,
      "rewards/rejected": -2.5865063667297363,
      "step": 7160
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.196682664195412e-06,
      "logits/chosen": -0.9364141225814819,
      "logits/rejected": -0.8269764184951782,
      "logps/chosen": -361.3995056152344,
      "logps/rejected": -384.99273681640625,
      "loss": 0.6921,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.5735127925872803,
      "rewards/margins": 0.5848428010940552,
      "rewards/rejected": -2.158355712890625,
      "step": 7170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.191197133760291e-06,
      "logits/chosen": -1.6076021194458008,
      "logits/rejected": -1.082669973373413,
      "logps/chosen": -394.1782531738281,
      "logps/rejected": -455.48870849609375,
      "loss": 0.424,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.2677477598190308,
      "rewards/margins": 1.3555927276611328,
      "rewards/rejected": -2.623340606689453,
      "step": 7180
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.185707997702334e-06,
      "logits/chosen": -1.2360800504684448,
      "logits/rejected": -0.9020748138427734,
      "logps/chosen": -411.65325927734375,
      "logps/rejected": -461.48162841796875,
      "loss": 0.5452,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7088842391967773,
      "rewards/margins": 0.8767988085746765,
      "rewards/rejected": -2.5856828689575195,
      "step": 7190
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1802152846555624e-06,
      "logits/chosen": -1.1463711261749268,
      "logits/rejected": -0.976833701133728,
      "logps/chosen": -398.6136474609375,
      "logps/rejected": -483.20458984375,
      "loss": 0.5048,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.743706464767456,
      "rewards/margins": 1.001102328300476,
      "rewards/rejected": -2.7448086738586426,
      "step": 7200
    },
    {
      "epoch": 0.47,
      "eval_logits/chosen": -1.1263185739517212,
      "eval_logits/rejected": -0.9988316297531128,
      "eval_logps/chosen": -396.180419921875,
      "eval_logps/rejected": -459.2229309082031,
      "eval_loss": 0.5000500679016113,
      "eval_rewards/accuracies": 0.6675000190734863,
      "eval_rewards/chosen": -1.641754150390625,
      "eval_rewards/margins": 0.8343563079833984,
      "eval_rewards/rejected": -2.4761104583740234,
      "eval_runtime": 731.8554,
      "eval_samples_per_second": 2.733,
      "eval_steps_per_second": 1.366,
      "step": 7200
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.174719023272659e-06,
      "logits/chosen": -1.390661597251892,
      "logits/rejected": -1.3155701160430908,
      "logps/chosen": -373.8555908203125,
      "logps/rejected": -508.87786865234375,
      "loss": 0.5011,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6016145944595337,
      "rewards/margins": 0.8867307901382446,
      "rewards/rejected": -2.4883456230163574,
      "step": 7210
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.169219242224816e-06,
      "logits/chosen": -1.3606770038604736,
      "logits/rejected": -1.1562891006469727,
      "logps/chosen": -411.99658203125,
      "logps/rejected": -466.5486755371094,
      "loss": 0.611,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7206933498382568,
      "rewards/margins": 0.6064735651016235,
      "rewards/rejected": -2.327166795730591,
      "step": 7220
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1637159702015837e-06,
      "logits/chosen": -1.2122749090194702,
      "logits/rejected": -1.0905072689056396,
      "logps/chosen": -350.120849609375,
      "logps/rejected": -435.25323486328125,
      "loss": 0.4162,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4422870874404907,
      "rewards/margins": 1.0445516109466553,
      "rewards/rejected": -2.4868385791778564,
      "step": 7230
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1582092359107263e-06,
      "logits/chosen": -1.2009532451629639,
      "logits/rejected": -0.9746109843254089,
      "logps/chosen": -454.6007385253906,
      "logps/rejected": -489.78302001953125,
      "loss": 0.5496,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.763230323791504,
      "rewards/margins": 0.8101302981376648,
      "rewards/rejected": -2.5733604431152344,
      "step": 7240
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.152699068078067e-06,
      "logits/chosen": -1.0341651439666748,
      "logits/rejected": -0.9404195547103882,
      "logps/chosen": -481.7880859375,
      "logps/rejected": -568.8102416992188,
      "loss": 0.4662,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0149779319763184,
      "rewards/margins": 1.0756332874298096,
      "rewards/rejected": -3.090610980987549,
      "step": 7250
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1471854954473415e-06,
      "logits/chosen": -1.2517516613006592,
      "logits/rejected": -1.1824498176574707,
      "logps/chosen": -404.09130859375,
      "logps/rejected": -505.6543884277344,
      "loss": 0.4441,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5332467555999756,
      "rewards/margins": 1.053310513496399,
      "rewards/rejected": -2.586557149887085,
      "step": 7260
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1416685467800436e-06,
      "logits/chosen": -0.916897177696228,
      "logits/rejected": -0.5630122423171997,
      "logps/chosen": -382.2656555175781,
      "logps/rejected": -469.0830078125,
      "loss": 0.4253,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9668285846710205,
      "rewards/margins": 1.0934703350067139,
      "rewards/rejected": -3.0602986812591553,
      "step": 7270
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1361482508552803e-06,
      "logits/chosen": -1.0097547769546509,
      "logits/rejected": -0.8572899699211121,
      "logps/chosen": -464.41973876953125,
      "logps/rejected": -529.024658203125,
      "loss": 0.4495,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2246882915496826,
      "rewards/margins": 1.0799113512039185,
      "rewards/rejected": -3.3045997619628906,
      "step": 7280
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1306246364696198e-06,
      "logits/chosen": -1.2491976022720337,
      "logits/rejected": -1.0045207738876343,
      "logps/chosen": -477.87872314453125,
      "logps/rejected": -540.9088134765625,
      "loss": 0.5529,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.2540440559387207,
      "rewards/margins": 0.8053514361381531,
      "rewards/rejected": -3.0593953132629395,
      "step": 7290
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1250977324369413e-06,
      "logits/chosen": -0.8631597757339478,
      "logits/rejected": -0.7546287775039673,
      "logps/chosen": -353.9215393066406,
      "logps/rejected": -460.38671875,
      "loss": 0.5141,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.949690818786621,
      "rewards/margins": 0.9975317120552063,
      "rewards/rejected": -2.9472222328186035,
      "step": 7300
    },
    {
      "epoch": 0.48,
      "eval_logits/chosen": -0.9431403875350952,
      "eval_logits/rejected": -0.8168740272521973,
      "eval_logps/chosen": -440.5569763183594,
      "eval_logps/rejected": -534.328125,
      "eval_loss": 0.4976823031902313,
      "eval_rewards/accuracies": 0.6679999828338623,
      "eval_rewards/chosen": -2.0855202674865723,
      "eval_rewards/margins": 1.1416418552398682,
      "eval_rewards/rejected": -3.2271616458892822,
      "eval_runtime": 734.1583,
      "eval_samples_per_second": 2.724,
      "eval_steps_per_second": 1.362,
      "step": 7300
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1195675675882825e-06,
      "logits/chosen": -1.0076789855957031,
      "logits/rejected": -0.7726067900657654,
      "logps/chosen": -480.07440185546875,
      "logps/rejected": -547.6993408203125,
      "loss": 0.6865,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.44883394241333,
      "rewards/margins": 0.9822977185249329,
      "rewards/rejected": -3.431131362915039,
      "step": 7310
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1140341707716926e-06,
      "logits/chosen": -0.6309999227523804,
      "logits/rejected": -0.5980799794197083,
      "logps/chosen": -380.33355712890625,
      "logps/rejected": -449.6524353027344,
      "loss": 0.5185,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8452589511871338,
      "rewards/margins": 1.123349905014038,
      "rewards/rejected": -2.9686086177825928,
      "step": 7320
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1084975708520803e-06,
      "logits/chosen": -1.2348999977111816,
      "logits/rejected": -0.9366706013679504,
      "logps/chosen": -439.20257568359375,
      "logps/rejected": -488.87420654296875,
      "loss": 0.3986,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7747869491577148,
      "rewards/margins": 1.2041819095611572,
      "rewards/rejected": -2.978969097137451,
      "step": 7330
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1029577967110625e-06,
      "logits/chosen": -1.1992151737213135,
      "logits/rejected": -0.9312963485717773,
      "logps/chosen": -398.64178466796875,
      "logps/rejected": -415.91705322265625,
      "loss": 0.5562,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.8224748373031616,
      "rewards/margins": 0.6633359789848328,
      "rewards/rejected": -2.4858107566833496,
      "step": 7340
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.097414877246814e-06,
      "logits/chosen": -1.1034296751022339,
      "logits/rejected": -0.8228904008865356,
      "logps/chosen": -375.7052307128906,
      "logps/rejected": -454.7461853027344,
      "loss": 0.5736,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.7706174850463867,
      "rewards/margins": 1.085538387298584,
      "rewards/rejected": -2.8561558723449707,
      "step": 7350
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0918688413739197e-06,
      "logits/chosen": -1.0494706630706787,
      "logits/rejected": -0.7607582211494446,
      "logps/chosen": -391.17401123046875,
      "logps/rejected": -433.62841796875,
      "loss": 0.5225,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.5849109888076782,
      "rewards/margins": 1.0672708749771118,
      "rewards/rejected": -2.65218186378479,
      "step": 7360
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0863197180232178e-06,
      "logits/chosen": -1.0603563785552979,
      "logits/rejected": -0.9168928861618042,
      "logps/chosen": -383.76544189453125,
      "logps/rejected": -453.3426818847656,
      "loss": 0.5822,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8402124643325806,
      "rewards/margins": 0.8179795145988464,
      "rewards/rejected": -2.6581919193267822,
      "step": 7370
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0807675361416554e-06,
      "logits/chosen": -0.9228275418281555,
      "logits/rejected": -0.8080077171325684,
      "logps/chosen": -340.25274658203125,
      "logps/rejected": -358.236083984375,
      "loss": 0.4654,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.513724684715271,
      "rewards/margins": 0.9701772928237915,
      "rewards/rejected": -2.4839017391204834,
      "step": 7380
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0752123246921327e-06,
      "logits/chosen": -1.0758297443389893,
      "logits/rejected": -0.7389814853668213,
      "logps/chosen": -491.60467529296875,
      "logps/rejected": -515.2469482421875,
      "loss": 0.5163,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.1369142532348633,
      "rewards/margins": 0.9057363271713257,
      "rewards/rejected": -3.0426506996154785,
      "step": 7390
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.069654112653353e-06,
      "logits/chosen": -1.2086726427078247,
      "logits/rejected": -0.9895984530448914,
      "logps/chosen": -429.166259765625,
      "logps/rejected": -445.71527099609375,
      "loss": 0.646,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.1268205642700195,
      "rewards/margins": 0.49101099371910095,
      "rewards/rejected": -2.6178317070007324,
      "step": 7400
    },
    {
      "epoch": 0.48,
      "eval_logits/chosen": -1.0571123361587524,
      "eval_logits/rejected": -0.9287133812904358,
      "eval_logps/chosen": -424.531494140625,
      "eval_logps/rejected": -497.04150390625,
      "eval_loss": 0.49757885932922363,
      "eval_rewards/accuracies": 0.6679999828338623,
      "eval_rewards/chosen": -1.9252653121948242,
      "eval_rewards/margins": 0.9290300011634827,
      "eval_rewards/rejected": -2.854295492172241,
      "eval_runtime": 732.9108,
      "eval_samples_per_second": 2.729,
      "eval_steps_per_second": 1.364,
      "step": 7400
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.064092929019673e-06,
      "logits/chosen": -1.0198802947998047,
      "logits/rejected": -1.211188554763794,
      "logps/chosen": -459.1338806152344,
      "logps/rejected": -546.1898193359375,
      "loss": 0.5362,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.015903949737549,
      "rewards/margins": 0.6718121767044067,
      "rewards/rejected": -2.687716007232666,
      "step": 7410
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.058528802800952e-06,
      "logits/chosen": -1.2751950025558472,
      "logits/rejected": -1.0569217205047607,
      "logps/chosen": -469.09527587890625,
      "logps/rejected": -543.2975463867188,
      "loss": 0.4846,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7646152973175049,
      "rewards/margins": 1.107751727104187,
      "rewards/rejected": -2.8723669052124023,
      "step": 7420
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.052961763022397e-06,
      "logits/chosen": -1.4869779348373413,
      "logits/rejected": -0.9571384191513062,
      "logps/chosen": -338.5403747558594,
      "logps/rejected": -422.4949645996094,
      "loss": 0.4403,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5565922260284424,
      "rewards/margins": 1.2267820835113525,
      "rewards/rejected": -2.783374309539795,
      "step": 7430
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.047391838724415e-06,
      "logits/chosen": -1.393455147743225,
      "logits/rejected": -1.1742329597473145,
      "logps/chosen": -411.403564453125,
      "logps/rejected": -498.971435546875,
      "loss": 0.4829,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7588484287261963,
      "rewards/margins": 1.0363447666168213,
      "rewards/rejected": -2.7951929569244385,
      "step": 7440
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0418190589624587e-06,
      "logits/chosen": -1.0927526950836182,
      "logits/rejected": -0.9543555378913879,
      "logps/chosen": -354.21954345703125,
      "logps/rejected": -436.81353759765625,
      "loss": 0.5101,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.7659744024276733,
      "rewards/margins": 0.742416262626648,
      "rewards/rejected": -2.5083906650543213,
      "step": 7450
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0362434528068784e-06,
      "logits/chosen": -0.9671896696090698,
      "logits/rejected": -0.7721429467201233,
      "logps/chosen": -487.379150390625,
      "logps/rejected": -489.2608337402344,
      "loss": 0.5641,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.1805596351623535,
      "rewards/margins": 0.8737584948539734,
      "rewards/rejected": -3.0543181896209717,
      "step": 7460
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0306650493427657e-06,
      "logits/chosen": -1.1028976440429688,
      "logits/rejected": -0.9604462385177612,
      "logps/chosen": -431.97442626953125,
      "logps/rejected": -527.2906494140625,
      "loss": 0.4603,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.003551959991455,
      "rewards/margins": 1.0427815914154053,
      "rewards/rejected": -3.0463337898254395,
      "step": 7470
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0250838776698077e-06,
      "logits/chosen": -1.2236485481262207,
      "logits/rejected": -1.0227609872817993,
      "logps/chosen": -366.2548828125,
      "logps/rejected": -462.3900451660156,
      "loss": 0.4685,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8233562707901,
      "rewards/margins": 0.9888140559196472,
      "rewards/rejected": -2.8121702671051025,
      "step": 7480
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0194999669021275e-06,
      "logits/chosen": -0.9461000561714172,
      "logits/rejected": -0.660954475402832,
      "logps/chosen": -390.1704406738281,
      "logps/rejected": -483.80126953125,
      "loss": 0.3723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6330198049545288,
      "rewards/margins": 1.4068280458450317,
      "rewards/rejected": -3.0398478507995605,
      "step": 7490
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0139133461681403e-06,
      "logits/chosen": -1.2607884407043457,
      "logits/rejected": -1.0787745714187622,
      "logps/chosen": -435.36334228515625,
      "logps/rejected": -533.2015991210938,
      "loss": 0.3417,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.709691047668457,
      "rewards/margins": 1.5397835969924927,
      "rewards/rejected": -3.249474287033081,
      "step": 7500
    },
    {
      "epoch": 0.49,
      "eval_logits/chosen": -1.1435625553131104,
      "eval_logits/rejected": -1.0097819566726685,
      "eval_logps/chosen": -411.1138610839844,
      "eval_logps/rejected": -493.5840148925781,
      "eval_loss": 0.493692010641098,
      "eval_rewards/accuracies": 0.671500027179718,
      "eval_rewards/chosen": -1.7910891771316528,
      "eval_rewards/margins": 1.0286314487457275,
      "eval_rewards/rejected": -2.81972074508667,
      "eval_runtime": 733.7776,
      "eval_samples_per_second": 2.726,
      "eval_steps_per_second": 1.363,
      "step": 7500
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0083240446103965e-06,
      "logits/chosen": -0.9401206970214844,
      "logits/rejected": -0.7115296125411987,
      "logps/chosen": -353.5488586425781,
      "logps/rejected": -463.7230529785156,
      "loss": 0.4751,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.695142388343811,
      "rewards/margins": 1.0250508785247803,
      "rewards/rejected": -2.720193386077881,
      "step": 7510
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0027320913854306e-06,
      "logits/chosen": -1.4791367053985596,
      "logits/rejected": -1.2607471942901611,
      "logps/chosen": -449.88909912109375,
      "logps/rejected": -552.6820678710938,
      "loss": 0.4171,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5742747783660889,
      "rewards/margins": 1.6550228595733643,
      "rewards/rejected": -3.229297637939453,
      "step": 7520
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.997137515663609e-06,
      "logits/chosen": -1.2254869937896729,
      "logits/rejected": -1.0456231832504272,
      "logps/chosen": -365.78118896484375,
      "logps/rejected": -470.77374267578125,
      "loss": 0.4353,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4040377140045166,
      "rewards/margins": 1.426917314529419,
      "rewards/rejected": -2.8309547901153564,
      "step": 7530
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.991540346628981e-06,
      "logits/chosen": -1.089773178100586,
      "logits/rejected": -1.074831247329712,
      "logps/chosen": -462.69830322265625,
      "logps/rejected": -528.5394287109375,
      "loss": 0.6096,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.2309799194335938,
      "rewards/margins": 0.9194940328598022,
      "rewards/rejected": -3.1504738330841064,
      "step": 7540
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.985940613479121e-06,
      "logits/chosen": -1.2983834743499756,
      "logits/rejected": -1.1657660007476807,
      "logps/chosen": -481.27520751953125,
      "logps/rejected": -534.2381591796875,
      "loss": 0.4576,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8742554187774658,
      "rewards/margins": 1.1233479976654053,
      "rewards/rejected": -2.997603178024292,
      "step": 7550
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.980338345424981e-06,
      "logits/chosen": -1.0275734663009644,
      "logits/rejected": -0.9290380477905273,
      "logps/chosen": -459.503662109375,
      "logps/rejected": -502.5309143066406,
      "loss": 0.4863,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.1028387546539307,
      "rewards/margins": 0.9130401611328125,
      "rewards/rejected": -3.0158791542053223,
      "step": 7560
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.974733571690735e-06,
      "logits/chosen": -1.2624428272247314,
      "logits/rejected": -0.8733476400375366,
      "logps/chosen": -439.00433349609375,
      "logps/rejected": -475.9070739746094,
      "loss": 0.5729,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.015974998474121,
      "rewards/margins": 0.961951732635498,
      "rewards/rejected": -2.977926731109619,
      "step": 7570
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9691263215136274e-06,
      "logits/chosen": -1.3705028295516968,
      "logits/rejected": -1.253151535987854,
      "logps/chosen": -424.6815490722656,
      "logps/rejected": -504.9693298339844,
      "loss": 0.4176,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.576811671257019,
      "rewards/margins": 1.1194231510162354,
      "rewards/rejected": -2.696234941482544,
      "step": 7580
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.963516624143823e-06,
      "logits/chosen": -1.0707788467407227,
      "logits/rejected": -1.2443617582321167,
      "logps/chosen": -433.29901123046875,
      "logps/rejected": -480.945556640625,
      "loss": 0.5844,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.2073233127593994,
      "rewards/margins": 0.8517726063728333,
      "rewards/rejected": -3.059095859527588,
      "step": 7590
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9579045088442504e-06,
      "logits/chosen": -1.2128026485443115,
      "logits/rejected": -1.0893349647521973,
      "logps/chosen": -329.65692138671875,
      "logps/rejected": -456.46087646484375,
      "loss": 0.4662,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.4115194082260132,
      "rewards/margins": 1.0705502033233643,
      "rewards/rejected": -2.482069492340088,
      "step": 7600
    },
    {
      "epoch": 0.5,
      "eval_logits/chosen": -1.2991713285446167,
      "eval_logits/rejected": -1.1591788530349731,
      "eval_logps/chosen": -382.1550598144531,
      "eval_logps/rejected": -461.83013916015625,
      "eval_loss": 0.5000800490379333,
      "eval_rewards/accuracies": 0.6669999957084656,
      "eval_rewards/chosen": -1.5015009641647339,
      "eval_rewards/margins": 1.0006811618804932,
      "eval_rewards/rejected": -2.5021822452545166,
      "eval_runtime": 732.2161,
      "eval_samples_per_second": 2.731,
      "eval_steps_per_second": 1.366,
      "step": 7600
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9522900048904534e-06,
      "logits/chosen": -1.3992488384246826,
      "logits/rejected": -1.1895803213119507,
      "logps/chosen": -424.6064453125,
      "logps/rejected": -467.15789794921875,
      "loss": 0.5624,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8486826419830322,
      "rewards/margins": 0.725730299949646,
      "rewards/rejected": -2.5744128227233887,
      "step": 7610
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9466731415704343e-06,
      "logits/chosen": -1.306740164756775,
      "logits/rejected": -1.1790627241134644,
      "logps/chosen": -384.5773620605469,
      "logps/rejected": -492.77264404296875,
      "loss": 0.4353,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6079070568084717,
      "rewards/margins": 1.1256263256072998,
      "rewards/rejected": -2.7335333824157715,
      "step": 7620
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.941053948184503e-06,
      "logits/chosen": -1.3813174962997437,
      "logits/rejected": -1.0688410997390747,
      "logps/chosen": -429.5804748535156,
      "logps/rejected": -484.577880859375,
      "loss": 0.5808,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.4964145421981812,
      "rewards/margins": 0.8990960121154785,
      "rewards/rejected": -2.39551043510437,
      "step": 7630
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.935432454045125e-06,
      "logits/chosen": -1.0862460136413574,
      "logits/rejected": -1.1066715717315674,
      "logps/chosen": -408.181640625,
      "logps/rejected": -448.7203674316406,
      "loss": 0.5967,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.7621729373931885,
      "rewards/margins": 0.6174625158309937,
      "rewards/rejected": -2.3796355724334717,
      "step": 7640
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.929808688476768e-06,
      "logits/chosen": -1.3649405241012573,
      "logits/rejected": -1.3180668354034424,
      "logps/chosen": -405.2919921875,
      "logps/rejected": -504.7835388183594,
      "loss": 0.4175,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6592957973480225,
      "rewards/margins": 1.170940637588501,
      "rewards/rejected": -2.8302364349365234,
      "step": 7650
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.924182680815748e-06,
      "logits/chosen": -1.2287496328353882,
      "logits/rejected": -1.1857244968414307,
      "logps/chosen": -394.7705383300781,
      "logps/rejected": -527.1818237304688,
      "loss": 0.3751,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.6013991832733154,
      "rewards/margins": 1.551189661026001,
      "rewards/rejected": -3.1525888442993164,
      "step": 7660
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9185544604100765e-06,
      "logits/chosen": -0.9437403678894043,
      "logits/rejected": -0.8567582368850708,
      "logps/chosen": -395.3851013183594,
      "logps/rejected": -465.4707946777344,
      "loss": 0.6494,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.9947502613067627,
      "rewards/margins": 0.7434107661247253,
      "rewards/rejected": -2.7381606101989746,
      "step": 7670
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9129240566193083e-06,
      "logits/chosen": -1.505251169204712,
      "logits/rejected": -1.1629559993743896,
      "logps/chosen": -369.99041748046875,
      "logps/rejected": -450.4754333496094,
      "loss": 0.5025,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6791009902954102,
      "rewards/margins": 0.9377298355102539,
      "rewards/rejected": -2.616830825805664,
      "step": 7680
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9072914988143874e-06,
      "logits/chosen": -1.2632492780685425,
      "logits/rejected": -1.0924867391586304,
      "logps/chosen": -349.3762512207031,
      "logps/rejected": -469.223388671875,
      "loss": 0.3885,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4794648885726929,
      "rewards/margins": 1.2831852436065674,
      "rewards/rejected": -2.7626500129699707,
      "step": 7690
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9016568163774956e-06,
      "logits/chosen": -1.3373229503631592,
      "logits/rejected": -1.1682683229446411,
      "logps/chosen": -305.866455078125,
      "logps/rejected": -353.1499938964844,
      "loss": 0.5059,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.336371660232544,
      "rewards/margins": 0.8133636713027954,
      "rewards/rejected": -2.149735450744629,
      "step": 7700
    },
    {
      "epoch": 0.5,
      "eval_logits/chosen": -1.324633002281189,
      "eval_logits/rejected": -1.1849044561386108,
      "eval_logps/chosen": -373.3851318359375,
      "eval_logps/rejected": -449.12884521484375,
      "eval_loss": 0.4978525936603546,
      "eval_rewards/accuracies": 0.6710000038146973,
      "eval_rewards/chosen": -1.4138015508651733,
      "eval_rewards/margins": 0.961367666721344,
      "eval_rewards/rejected": -2.375169515609741,
      "eval_runtime": 735.0378,
      "eval_samples_per_second": 2.721,
      "eval_steps_per_second": 1.36,
      "step": 7700
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.8960200387018942e-06,
      "logits/chosen": -1.3530820608139038,
      "logits/rejected": -1.199158787727356,
      "logps/chosen": -462.70745849609375,
      "logps/rejected": -492.1473693847656,
      "loss": 0.5305,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.5497446060180664,
      "rewards/margins": 0.7827426195144653,
      "rewards/rejected": -2.332487106323242,
      "step": 7710
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8903811951917792e-06,
      "logits/chosen": -1.3843700885772705,
      "logits/rejected": -1.33487868309021,
      "logps/chosen": -311.75762939453125,
      "logps/rejected": -350.55328369140625,
      "loss": 0.4853,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.1347800493240356,
      "rewards/margins": 0.859378457069397,
      "rewards/rejected": -1.994158387184143,
      "step": 7720
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.88474031526212e-06,
      "logits/chosen": -1.4788033962249756,
      "logits/rejected": -1.3710378408432007,
      "logps/chosen": -347.46319580078125,
      "logps/rejected": -442.15655517578125,
      "loss": 0.4774,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4462344646453857,
      "rewards/margins": 0.8126481175422668,
      "rewards/rejected": -2.258882522583008,
      "step": 7730
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.879097428338509e-06,
      "logits/chosen": -1.3819596767425537,
      "logits/rejected": -1.0944236516952515,
      "logps/chosen": -347.5988464355469,
      "logps/rejected": -415.97509765625,
      "loss": 0.5487,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3170385360717773,
      "rewards/margins": 0.9070457220077515,
      "rewards/rejected": -2.22408390045166,
      "step": 7740
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8734525638570094e-06,
      "logits/chosen": -1.379392147064209,
      "logits/rejected": -1.3306528329849243,
      "logps/chosen": -370.0411682128906,
      "logps/rejected": -447.42645263671875,
      "loss": 0.5758,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3584692478179932,
      "rewards/margins": 0.8975862264633179,
      "rewards/rejected": -2.2560553550720215,
      "step": 7750
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8678057512639982e-06,
      "logits/chosen": -1.3406450748443604,
      "logits/rejected": -1.2567927837371826,
      "logps/chosen": -405.84051513671875,
      "logps/rejected": -522.5167236328125,
      "loss": 0.3867,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.1977407932281494,
      "rewards/margins": 1.3873558044433594,
      "rewards/rejected": -2.585096836090088,
      "step": 7760
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8621570200160172e-06,
      "logits/chosen": -0.9038798213005066,
      "logits/rejected": -0.8531060218811035,
      "logps/chosen": -302.6310729980469,
      "logps/rejected": -423.69244384765625,
      "loss": 0.4657,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.3406198024749756,
      "rewards/margins": 1.2923932075500488,
      "rewards/rejected": -2.6330132484436035,
      "step": 7770
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.856506399579615e-06,
      "logits/chosen": -1.3094675540924072,
      "logits/rejected": -1.25465989112854,
      "logps/chosen": -426.06146240234375,
      "logps/rejected": -501.49700927734375,
      "loss": 0.5147,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.0769801139831543,
      "rewards/margins": 0.9492477178573608,
      "rewards/rejected": -3.0262279510498047,
      "step": 7780
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8508539194311964e-06,
      "logits/chosen": -1.1623289585113525,
      "logits/rejected": -1.2776973247528076,
      "logps/chosen": -444.77008056640625,
      "logps/rejected": -559.510986328125,
      "loss": 0.4635,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9103660583496094,
      "rewards/margins": 1.0191086530685425,
      "rewards/rejected": -2.929474353790283,
      "step": 7790
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8451996090568656e-06,
      "logits/chosen": -1.0837061405181885,
      "logits/rejected": -0.8834303617477417,
      "logps/chosen": -380.545654296875,
      "logps/rejected": -475.828369140625,
      "loss": 0.4464,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9440603256225586,
      "rewards/margins": 1.1324434280395508,
      "rewards/rejected": -3.0765042304992676,
      "step": 7800
    },
    {
      "epoch": 0.51,
      "eval_logits/chosen": -1.0977903604507446,
      "eval_logits/rejected": -0.9724917411804199,
      "eval_logps/chosen": -452.9457702636719,
      "eval_logps/rejected": -531.2132568359375,
      "eval_loss": 0.5016809701919556,
      "eval_rewards/accuracies": 0.6740000247955322,
      "eval_rewards/chosen": -2.209408760070801,
      "eval_rewards/margins": 0.9866045117378235,
      "eval_rewards/rejected": -3.1960132122039795,
      "eval_runtime": 732.3502,
      "eval_samples_per_second": 2.731,
      "eval_steps_per_second": 1.365,
      "step": 7800
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.839543497952276e-06,
      "logits/chosen": -1.030879259109497,
      "logits/rejected": -1.0171698331832886,
      "logps/chosen": -380.8997497558594,
      "logps/rejected": -486.53656005859375,
      "loss": 0.4362,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9628963470458984,
      "rewards/margins": 1.1434067487716675,
      "rewards/rejected": -3.1063029766082764,
      "step": 7810
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.833885615622474e-06,
      "logits/chosen": -0.986414909362793,
      "logits/rejected": -0.8642165064811707,
      "logps/chosen": -474.72515869140625,
      "logps/rejected": -554.31201171875,
      "loss": 0.6366,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.7214932441711426,
      "rewards/margins": 0.7011139392852783,
      "rewards/rejected": -3.422607898712158,
      "step": 7820
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8282259915817454e-06,
      "logits/chosen": -0.789636492729187,
      "logits/rejected": -0.7618891596794128,
      "logps/chosen": -339.92449951171875,
      "logps/rejected": -497.02923583984375,
      "loss": 0.4635,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9897979497909546,
      "rewards/margins": 1.168931484222412,
      "rewards/rejected": -3.1587295532226562,
      "step": 7830
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8225646553534614e-06,
      "logits/chosen": -0.812696099281311,
      "logits/rejected": -0.7494589686393738,
      "logps/chosen": -385.42608642578125,
      "logps/rejected": -450.68621826171875,
      "loss": 0.5923,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.8611011505126953,
      "rewards/margins": 0.6855142712593079,
      "rewards/rejected": -2.5466151237487793,
      "step": 7840
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8169016364699255e-06,
      "logits/chosen": -1.135119080543518,
      "logits/rejected": -0.9809944033622742,
      "logps/chosen": -436.8231506347656,
      "logps/rejected": -525.0554809570312,
      "loss": 0.5057,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.249351978302002,
      "rewards/margins": 0.864822268486023,
      "rewards/rejected": -3.1141743659973145,
      "step": 7850
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.811236964472217e-06,
      "logits/chosen": -1.2407481670379639,
      "logits/rejected": -1.075469732284546,
      "logps/chosen": -495.60333251953125,
      "logps/rejected": -545.838134765625,
      "loss": 0.4944,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8519678115844727,
      "rewards/margins": 1.103547215461731,
      "rewards/rejected": -2.955515146255493,
      "step": 7860
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.805570668910041e-06,
      "logits/chosen": -0.8709837198257446,
      "logits/rejected": -0.9303278923034668,
      "logps/chosen": -379.8825988769531,
      "logps/rejected": -545.008544921875,
      "loss": 0.4578,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.1024327278137207,
      "rewards/margins": 1.0162558555603027,
      "rewards/rejected": -3.1186885833740234,
      "step": 7870
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7999027793415695e-06,
      "logits/chosen": -1.2885119915008545,
      "logits/rejected": -0.8333857655525208,
      "logps/chosen": -422.70928955078125,
      "logps/rejected": -477.76019287109375,
      "loss": 0.5059,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7779327630996704,
      "rewards/margins": 0.9997657537460327,
      "rewards/rejected": -2.7776987552642822,
      "step": 7880
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.794233325333293e-06,
      "logits/chosen": -1.0390526056289673,
      "logits/rejected": -0.8826634287834167,
      "logps/chosen": -427.525634765625,
      "logps/rejected": -531.1912231445312,
      "loss": 0.4356,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.6924421787261963,
      "rewards/margins": 1.273428201675415,
      "rewards/rejected": -2.9658703804016113,
      "step": 7890
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7885623364598597e-06,
      "logits/chosen": -1.298937201499939,
      "logits/rejected": -0.9922115206718445,
      "logps/chosen": -453.6202697753906,
      "logps/rejected": -565.6160888671875,
      "loss": 0.3597,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8992764949798584,
      "rewards/margins": 1.5391474962234497,
      "rewards/rejected": -3.4384243488311768,
      "step": 7900
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -1.0761736631393433,
      "eval_logits/rejected": -0.9468325972557068,
      "eval_logps/chosen": -403.9175720214844,
      "eval_logps/rejected": -494.2936706542969,
      "eval_loss": 0.495647668838501,
      "eval_rewards/accuracies": 0.6725000143051147,
      "eval_rewards/chosen": -1.7191258668899536,
      "eval_rewards/margins": 1.1076916456222534,
      "eval_rewards/rejected": -2.826817274093628,
      "eval_runtime": 733.0812,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 7900
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.782889842303926e-06,
      "logits/chosen": -1.0714633464813232,
      "logits/rejected": -1.0015760660171509,
      "logps/chosen": -360.4320983886719,
      "logps/rejected": -431.151123046875,
      "loss": 0.5618,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.0020298957824707,
      "rewards/margins": 0.7621687650680542,
      "rewards/rejected": -2.7641987800598145,
      "step": 7910
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7772158724559987e-06,
      "logits/chosen": -1.0210740566253662,
      "logits/rejected": -0.7624063491821289,
      "logps/chosen": -365.82025146484375,
      "logps/rejected": -589.4786376953125,
      "loss": 0.36,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4871119260787964,
      "rewards/margins": 1.861876130104065,
      "rewards/rejected": -3.3489882946014404,
      "step": 7920
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7715404565142856e-06,
      "logits/chosen": -0.7972566485404968,
      "logits/rejected": -0.9107509851455688,
      "logps/chosen": -411.510009765625,
      "logps/rejected": -491.3306579589844,
      "loss": 0.5595,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.1154580116271973,
      "rewards/margins": 0.8604329228401184,
      "rewards/rejected": -2.975890636444092,
      "step": 7930
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7658636240845354e-06,
      "logits/chosen": -1.2706204652786255,
      "logits/rejected": -1.1921221017837524,
      "logps/chosen": -449.60516357421875,
      "logps/rejected": -611.0333251953125,
      "loss": 0.3961,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.261568307876587,
      "rewards/margins": 1.468100905418396,
      "rewards/rejected": -3.7296690940856934,
      "step": 7940
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7601854047798872e-06,
      "logits/chosen": -0.7478612661361694,
      "logits/rejected": -0.800074577331543,
      "logps/chosen": -465.3731384277344,
      "logps/rejected": -553.3946533203125,
      "loss": 0.6341,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.4144978523254395,
      "rewards/margins": 0.6860623359680176,
      "rewards/rejected": -3.100559949874878,
      "step": 7950
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7545058282207148e-06,
      "logits/chosen": -0.6742497682571411,
      "logits/rejected": -0.6114516258239746,
      "logps/chosen": -422.21807861328125,
      "logps/rejected": -478.6226501464844,
      "loss": 0.5483,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.106762647628784,
      "rewards/margins": 0.8698352575302124,
      "rewards/rejected": -2.976598024368286,
      "step": 7960
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.748824924034471e-06,
      "logits/chosen": -0.9385892152786255,
      "logits/rejected": -0.8127745389938354,
      "logps/chosen": -480.69482421875,
      "logps/rejected": -532.44775390625,
      "loss": 0.6853,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.6128458976745605,
      "rewards/margins": 0.7064965963363647,
      "rewards/rejected": -3.319342851638794,
      "step": 7970
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.743142721855536e-06,
      "logits/chosen": -0.4976627230644226,
      "logits/rejected": -0.6767405271530151,
      "logps/chosen": -345.3768310546875,
      "logps/rejected": -402.8832702636719,
      "loss": 0.5117,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9351670742034912,
      "rewards/margins": 0.6585490107536316,
      "rewards/rejected": -2.5937161445617676,
      "step": 7980
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.737459251325058e-06,
      "logits/chosen": -0.8807660937309265,
      "logits/rejected": -0.729613184928894,
      "logps/chosen": -452.9046325683594,
      "logps/rejected": -510.9497985839844,
      "loss": 0.4951,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.829310655593872,
      "rewards/margins": 0.8057653307914734,
      "rewards/rejected": -2.6350760459899902,
      "step": 7990
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.731774542090804e-06,
      "logits/chosen": -0.5301181674003601,
      "logits/rejected": -0.5973738431930542,
      "logps/chosen": -405.48492431640625,
      "logps/rejected": -436.1067810058594,
      "loss": 0.6685,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.127089262008667,
      "rewards/margins": 0.4965682625770569,
      "rewards/rejected": -2.623657703399658,
      "step": 8000
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -0.8313817977905273,
      "eval_logits/rejected": -0.7170847058296204,
      "eval_logps/chosen": -446.3575134277344,
      "eval_logps/rejected": -524.3576049804688,
      "eval_loss": 0.4940214455127716,
      "eval_rewards/accuracies": 0.6694999933242798,
      "eval_rewards/chosen": -2.1435256004333496,
      "eval_rewards/margins": 0.9839317798614502,
      "eval_rewards/rejected": -3.127457618713379,
      "eval_runtime": 733.0072,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 8000
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7260886238070034e-06,
      "logits/chosen": -0.9801632761955261,
      "logits/rejected": -0.8519521951675415,
      "logps/chosen": -426.9603576660156,
      "logps/rejected": -517.7711181640625,
      "loss": 0.4641,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.3049042224884033,
      "rewards/margins": 0.9551888704299927,
      "rewards/rejected": -3.2600929737091064,
      "step": 8010
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.72040152613419e-06,
      "logits/chosen": -0.8683644533157349,
      "logits/rejected": -0.7820503115653992,
      "logps/chosen": -433.59735107421875,
      "logps/rejected": -472.0309143066406,
      "loss": 0.4193,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.167222261428833,
      "rewards/margins": 1.2165911197662354,
      "rewards/rejected": -3.3838133811950684,
      "step": 8020
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.7147132787390516e-06,
      "logits/chosen": -1.0270106792449951,
      "logits/rejected": -0.7757326364517212,
      "logps/chosen": -414.59613037109375,
      "logps/rejected": -508.03814697265625,
      "loss": 0.4447,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8711802959442139,
      "rewards/margins": 1.1042753458023071,
      "rewards/rejected": -2.9754557609558105,
      "step": 8030
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.709023911294273e-06,
      "logits/chosen": -1.136345624923706,
      "logits/rejected": -0.9073754549026489,
      "logps/chosen": -395.5877990722656,
      "logps/rejected": -545.9440307617188,
      "loss": 0.4058,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5326893329620361,
      "rewards/margins": 1.8125578165054321,
      "rewards/rejected": -3.3452467918395996,
      "step": 8040
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.7033334534783806e-06,
      "logits/chosen": -1.1330026388168335,
      "logits/rejected": -1.2662372589111328,
      "logps/chosen": -347.5755920410156,
      "logps/rejected": -466.73443603515625,
      "loss": 0.5626,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4842970371246338,
      "rewards/margins": 1.0295974016189575,
      "rewards/rejected": -2.5138943195343018,
      "step": 8050
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.697641934975592e-06,
      "logits/chosen": -1.2012157440185547,
      "logits/rejected": -1.0244762897491455,
      "logps/chosen": -388.85107421875,
      "logps/rejected": -484.35498046875,
      "loss": 0.5007,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6106462478637695,
      "rewards/margins": 1.280577540397644,
      "rewards/rejected": -2.891223907470703,
      "step": 8060
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.691949385475654e-06,
      "logits/chosen": -1.198803186416626,
      "logits/rejected": -1.0046731233596802,
      "logps/chosen": -436.41021728515625,
      "logps/rejected": -520.1088256835938,
      "loss": 0.5023,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9251935482025146,
      "rewards/margins": 1.0926984548568726,
      "rewards/rejected": -3.0178921222686768,
      "step": 8070
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6862558346736937e-06,
      "logits/chosen": -1.1371890306472778,
      "logits/rejected": -0.9129287600517273,
      "logps/chosen": -407.1488037109375,
      "logps/rejected": -593.5284423828125,
      "loss": 0.3355,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6879653930664062,
      "rewards/margins": 1.915813684463501,
      "rewards/rejected": -3.603778839111328,
      "step": 8080
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6805613122700617e-06,
      "logits/chosen": -1.0117746591567993,
      "logits/rejected": -0.9848273992538452,
      "logps/chosen": -440.2730407714844,
      "logps/rejected": -555.3220825195312,
      "loss": 0.5118,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.1670055389404297,
      "rewards/margins": 1.1410691738128662,
      "rewards/rejected": -3.308074474334717,
      "step": 8090
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.674865847970176e-06,
      "logits/chosen": -1.0662400722503662,
      "logits/rejected": -0.8855364918708801,
      "logps/chosen": -409.4464416503906,
      "logps/rejected": -533.459228515625,
      "loss": 0.5494,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.0183000564575195,
      "rewards/margins": 1.015644907951355,
      "rewards/rejected": -3.033945322036743,
      "step": 8100
    },
    {
      "epoch": 0.53,
      "eval_logits/chosen": -1.0209530591964722,
      "eval_logits/rejected": -0.8909984827041626,
      "eval_logps/chosen": -445.6345520019531,
      "eval_logps/rejected": -532.8621826171875,
      "eval_loss": 0.49142247438430786,
      "eval_rewards/accuracies": 0.6654999852180481,
      "eval_rewards/chosen": -2.136295795440674,
      "eval_rewards/margins": 1.076207160949707,
      "eval_rewards/rejected": -3.212502956390381,
      "eval_runtime": 731.4903,
      "eval_samples_per_second": 2.734,
      "eval_steps_per_second": 1.367,
      "step": 8100
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.669169471484368e-06,
      "logits/chosen": -0.7829369306564331,
      "logits/rejected": -0.8367104530334473,
      "logps/chosen": -400.9513244628906,
      "logps/rejected": -442.726806640625,
      "loss": 0.6904,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.3589324951171875,
      "rewards/margins": 0.44415682554244995,
      "rewards/rejected": -2.8030896186828613,
      "step": 8110
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6634722125277278e-06,
      "logits/chosen": -1.1393558979034424,
      "logits/rejected": -0.8891308903694153,
      "logps/chosen": -481.6310119628906,
      "logps/rejected": -586.5345458984375,
      "loss": 0.5243,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.4951107501983643,
      "rewards/margins": 0.9670877456665039,
      "rewards/rejected": -3.4621987342834473,
      "step": 8120
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6577741008199498e-06,
      "logits/chosen": -0.939818263053894,
      "logits/rejected": -0.8135367631912231,
      "logps/chosen": -490.2794494628906,
      "logps/rejected": -577.6835327148438,
      "loss": 0.409,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.341212511062622,
      "rewards/margins": 1.474454402923584,
      "rewards/rejected": -3.815666913986206,
      "step": 8130
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.652075166085175e-06,
      "logits/chosen": -0.8981655240058899,
      "logits/rejected": -0.9567068815231323,
      "logps/chosen": -460.9468688964844,
      "logps/rejected": -618.2373046875,
      "loss": 0.491,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.2636752128601074,
      "rewards/margins": 1.3374521732330322,
      "rewards/rejected": -3.6011271476745605,
      "step": 8140
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6463754380518395e-06,
      "logits/chosen": -0.8287490010261536,
      "logits/rejected": -0.7122872471809387,
      "logps/chosen": -475.3302307128906,
      "logps/rejected": -512.5914306640625,
      "loss": 0.5309,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.443725347518921,
      "rewards/margins": 0.9208828806877136,
      "rewards/rejected": -3.3646087646484375,
      "step": 8150
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6406749464525167e-06,
      "logits/chosen": -1.278389811515808,
      "logits/rejected": -0.9623058438301086,
      "logps/chosen": -380.12249755859375,
      "logps/rejected": -452.919921875,
      "loss": 0.4098,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4656339883804321,
      "rewards/margins": 1.200292706489563,
      "rewards/rejected": -2.665926694869995,
      "step": 8160
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.634973721023762e-06,
      "logits/chosen": -1.3094470500946045,
      "logits/rejected": -1.1664118766784668,
      "logps/chosen": -468.9601135253906,
      "logps/rejected": -489.9962463378906,
      "loss": 0.5648,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1449787616729736,
      "rewards/margins": 0.7047473192214966,
      "rewards/rejected": -2.849726438522339,
      "step": 8170
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6292717915059605e-06,
      "logits/chosen": -1.3345282077789307,
      "logits/rejected": -1.1698658466339111,
      "logps/chosen": -482.92279052734375,
      "logps/rejected": -533.3839111328125,
      "loss": 0.4298,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.0729269981384277,
      "rewards/margins": 1.0665416717529297,
      "rewards/rejected": -3.1394686698913574,
      "step": 8180
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6235691876431706e-06,
      "logits/chosen": -1.2751821279525757,
      "logits/rejected": -1.2013987302780151,
      "logps/chosen": -439.2770080566406,
      "logps/rejected": -522.8220825195312,
      "loss": 0.5507,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.2066140174865723,
      "rewards/margins": 0.7794591784477234,
      "rewards/rejected": -2.9860730171203613,
      "step": 8190
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6178659391829673e-06,
      "logits/chosen": -1.326781988143921,
      "logits/rejected": -1.0384869575500488,
      "logps/chosen": -414.6600646972656,
      "logps/rejected": -479.1424255371094,
      "loss": 0.4703,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7718207836151123,
      "rewards/margins": 1.0532562732696533,
      "rewards/rejected": -2.8250768184661865,
      "step": 8200
    },
    {
      "epoch": 0.54,
      "eval_logits/chosen": -1.1886258125305176,
      "eval_logits/rejected": -1.055037498474121,
      "eval_logps/chosen": -433.6510314941406,
      "eval_logps/rejected": -508.3775634765625,
      "eval_loss": 0.49489960074424744,
      "eval_rewards/accuracies": 0.6660000085830688,
      "eval_rewards/chosen": -2.016460657119751,
      "eval_rewards/margins": 0.9511955380439758,
      "eval_rewards/rejected": -2.9676566123962402,
      "eval_runtime": 734.8366,
      "eval_samples_per_second": 2.722,
      "eval_steps_per_second": 1.361,
      "step": 8200
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6121620758762877e-06,
      "logits/chosen": -1.2698452472686768,
      "logits/rejected": -0.9875261187553406,
      "logps/chosen": -390.28704833984375,
      "logps/rejected": -475.6351013183594,
      "loss": 0.4836,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9820274114608765,
      "rewards/margins": 0.8614705801010132,
      "rewards/rejected": -2.8434982299804688,
      "step": 8210
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.606457627477277e-06,
      "logits/chosen": -1.0521068572998047,
      "logits/rejected": -0.9721614122390747,
      "logps/chosen": -345.18341064453125,
      "logps/rejected": -463.9532165527344,
      "loss": 0.4758,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.6955410242080688,
      "rewards/margins": 1.1541770696640015,
      "rewards/rejected": -2.8497185707092285,
      "step": 8220
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6007526237431324e-06,
      "logits/chosen": -1.3272507190704346,
      "logits/rejected": -1.214674472808838,
      "logps/chosen": -360.69989013671875,
      "logps/rejected": -493.56854248046875,
      "loss": 0.4782,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7867038249969482,
      "rewards/margins": 1.1875064373016357,
      "rewards/rejected": -2.974210023880005,
      "step": 8230
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5950470944339478e-06,
      "logits/chosen": -1.1821718215942383,
      "logits/rejected": -1.211446762084961,
      "logps/chosen": -405.30413818359375,
      "logps/rejected": -467.179931640625,
      "loss": 0.5559,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.8350213766098022,
      "rewards/margins": 0.6537161469459534,
      "rewards/rejected": -2.4887375831604004,
      "step": 8240
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.58934106931256e-06,
      "logits/chosen": -1.1724306344985962,
      "logits/rejected": -1.018654704093933,
      "logps/chosen": -437.35595703125,
      "logps/rejected": -499.43572998046875,
      "loss": 0.5731,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1658389568328857,
      "rewards/margins": 0.8125737905502319,
      "rewards/rejected": -2.978412628173828,
      "step": 8250
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.58363457814439e-06,
      "logits/chosen": -1.2482324838638306,
      "logits/rejected": -0.9831002354621887,
      "logps/chosen": -450.75244140625,
      "logps/rejected": -523.3827514648438,
      "loss": 0.6366,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.4008052349090576,
      "rewards/margins": 0.8635374903678894,
      "rewards/rejected": -3.26434326171875,
      "step": 8260
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5779276506972924e-06,
      "logits/chosen": -1.1220371723175049,
      "logits/rejected": -1.1320124864578247,
      "logps/chosen": -439.0234375,
      "logps/rejected": -486.7811584472656,
      "loss": 0.554,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.0707054138183594,
      "rewards/margins": 0.8713995218276978,
      "rewards/rejected": -2.9421050548553467,
      "step": 8270
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5722203167413945e-06,
      "logits/chosen": -1.2530734539031982,
      "logits/rejected": -1.1008774042129517,
      "logps/chosen": -472.61737060546875,
      "logps/rejected": -483.79278564453125,
      "loss": 0.495,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8931972980499268,
      "rewards/margins": 0.9309617280960083,
      "rewards/rejected": -2.8241589069366455,
      "step": 8280
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5665126060489476e-06,
      "logits/chosen": -1.3096177577972412,
      "logits/rejected": -1.2289804220199585,
      "logps/chosen": -399.1604919433594,
      "logps/rejected": -502.7049255371094,
      "loss": 0.5053,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1105847358703613,
      "rewards/margins": 0.7469784021377563,
      "rewards/rejected": -2.857563018798828,
      "step": 8290
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.560804548394165e-06,
      "logits/chosen": -1.165805459022522,
      "logits/rejected": -0.8079797625541687,
      "logps/chosen": -437.4986877441406,
      "logps/rejected": -492.98358154296875,
      "loss": 0.4901,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.902182936668396,
      "rewards/margins": 1.007571816444397,
      "rewards/rejected": -2.909754991531372,
      "step": 8300
    },
    {
      "epoch": 0.54,
      "eval_logits/chosen": -1.2041014432907104,
      "eval_logits/rejected": -1.072398066520691,
      "eval_logps/chosen": -416.77789306640625,
      "eval_logps/rejected": -487.3052978515625,
      "eval_loss": 0.4975956082344055,
      "eval_rewards/accuracies": 0.6635000109672546,
      "eval_rewards/chosen": -1.8477294445037842,
      "eval_rewards/margins": 0.9092043042182922,
      "eval_rewards/rejected": -2.75693416595459,
      "eval_runtime": 731.7183,
      "eval_samples_per_second": 2.733,
      "eval_steps_per_second": 1.367,
      "step": 8300
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5550961735530734e-06,
      "logits/chosen": -0.9673964381217957,
      "logits/rejected": -1.094525933265686,
      "logps/chosen": -307.0442810058594,
      "logps/rejected": -424.9918518066406,
      "loss": 0.4957,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4400845766067505,
      "rewards/margins": 0.8735688924789429,
      "rewards/rejected": -2.3136534690856934,
      "step": 8310
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.549387511303351e-06,
      "logits/chosen": -1.1674532890319824,
      "logits/rejected": -1.2690563201904297,
      "logps/chosen": -357.15203857421875,
      "logps/rejected": -461.42706298828125,
      "loss": 0.5702,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.874151587486267,
      "rewards/margins": 0.5999444127082825,
      "rewards/rejected": -2.4740958213806152,
      "step": 8320
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5436785914241774e-06,
      "logits/chosen": -1.0933245420455933,
      "logits/rejected": -1.0153937339782715,
      "logps/chosen": -414.09619140625,
      "logps/rejected": -490.25152587890625,
      "loss": 0.4778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.151369571685791,
      "rewards/margins": 1.081249475479126,
      "rewards/rejected": -3.232619524002075,
      "step": 8330
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5379694436960746e-06,
      "logits/chosen": -1.2745726108551025,
      "logits/rejected": -1.200372338294983,
      "logps/chosen": -446.69580078125,
      "logps/rejected": -548.1271362304688,
      "loss": 0.5295,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.0068485736846924,
      "rewards/margins": 0.9086639285087585,
      "rewards/rejected": -2.9155125617980957,
      "step": 8340
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5322600979007533e-06,
      "logits/chosen": -1.3381099700927734,
      "logits/rejected": -1.088611364364624,
      "logps/chosen": -390.39862060546875,
      "logps/rejected": -444.66998291015625,
      "loss": 0.5258,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7790319919586182,
      "rewards/margins": 0.7497051358222961,
      "rewards/rejected": -2.5287370681762695,
      "step": 8350
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5265505838209592e-06,
      "logits/chosen": -1.216956615447998,
      "logits/rejected": -1.059798240661621,
      "logps/chosen": -504.1844177246094,
      "logps/rejected": -533.2293701171875,
      "loss": 0.6299,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.4722084999084473,
      "rewards/margins": 0.7545827627182007,
      "rewards/rejected": -3.2267906665802,
      "step": 8360
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.520840931240314e-06,
      "logits/chosen": -1.2149550914764404,
      "logits/rejected": -0.8863165974617004,
      "logps/chosen": -431.2649841308594,
      "logps/rejected": -461.96856689453125,
      "loss": 0.4361,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.2145636081695557,
      "rewards/margins": 0.9442449808120728,
      "rewards/rejected": -3.158808469772339,
      "step": 8370
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.515131169943162e-06,
      "logits/chosen": -0.7880765795707703,
      "logits/rejected": -0.7995393872261047,
      "logps/chosen": -510.5859375,
      "logps/rejected": -632.1096801757812,
      "loss": 0.4487,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.5249135494232178,
      "rewards/margins": 1.2849611043930054,
      "rewards/rejected": -3.8098747730255127,
      "step": 8380
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.509421329714416e-06,
      "logits/chosen": -0.7812287211418152,
      "logits/rejected": -0.8888276219367981,
      "logps/chosen": -446.288818359375,
      "logps/rejected": -559.374267578125,
      "loss": 0.5248,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.401496171951294,
      "rewards/margins": 0.918934166431427,
      "rewards/rejected": -3.320430278778076,
      "step": 8390
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5037114403393987e-06,
      "logits/chosen": -0.9296455383300781,
      "logits/rejected": -0.7097471952438354,
      "logps/chosen": -420.11614990234375,
      "logps/rejected": -490.839111328125,
      "loss": 0.4759,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.091357707977295,
      "rewards/margins": 1.0450347661972046,
      "rewards/rejected": -3.13639235496521,
      "step": 8400
    },
    {
      "epoch": 0.55,
      "eval_logits/chosen": -1.0123143196105957,
      "eval_logits/rejected": -0.8860378861427307,
      "eval_logps/chosen": -479.3096008300781,
      "eval_logps/rejected": -566.3602905273438,
      "eval_loss": 0.4948574900627136,
      "eval_rewards/accuracies": 0.6654999852180481,
      "eval_rewards/chosen": -2.47304630279541,
      "eval_rewards/margins": 1.0744378566741943,
      "eval_rewards/rejected": -3.5474843978881836,
      "eval_runtime": 732.3944,
      "eval_samples_per_second": 2.731,
      "eval_steps_per_second": 1.365,
      "step": 8400
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4980015316036908e-06,
      "logits/chosen": -0.8672155141830444,
      "logits/rejected": -0.8484716415405273,
      "logps/chosen": -402.535888671875,
      "logps/rejected": -579.1514892578125,
      "loss": 0.404,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.2672619819641113,
      "rewards/margins": 1.5502973794937134,
      "rewards/rejected": -3.817559003829956,
      "step": 8410
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4922916332929725e-06,
      "logits/chosen": -1.1366407871246338,
      "logits/rejected": -1.1110883951187134,
      "logps/chosen": -463.5205078125,
      "logps/rejected": -523.5811767578125,
      "loss": 0.4977,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2809627056121826,
      "rewards/margins": 0.9967510104179382,
      "rewards/rejected": -3.2777132987976074,
      "step": 8420
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4865817751928716e-06,
      "logits/chosen": -1.0972706079483032,
      "logits/rejected": -1.0200917720794678,
      "logps/chosen": -428.05413818359375,
      "logps/rejected": -609.2821044921875,
      "loss": 0.3714,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.3256542682647705,
      "rewards/margins": 1.548058271408081,
      "rewards/rejected": -3.8737120628356934,
      "step": 8430
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4808719870888037e-06,
      "logits/chosen": -0.9892493486404419,
      "logits/rejected": -0.7642117738723755,
      "logps/chosen": -451.3265686035156,
      "logps/rejected": -560.0855102539062,
      "loss": 0.4537,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.3365511894226074,
      "rewards/margins": 1.420327067375183,
      "rewards/rejected": -3.75687837600708,
      "step": 8440
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4751622987658206e-06,
      "logits/chosen": -1.2597066164016724,
      "logits/rejected": -1.1139386892318726,
      "logps/chosen": -487.68646240234375,
      "logps/rejected": -580.7915649414062,
      "loss": 0.5196,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.497361898422241,
      "rewards/margins": 1.0384573936462402,
      "rewards/rejected": -3.5358192920684814,
      "step": 8450
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4694527400084546e-06,
      "logits/chosen": -1.1042139530181885,
      "logits/rejected": -0.9779818654060364,
      "logps/chosen": -441.48944091796875,
      "logps/rejected": -536.3157348632812,
      "loss": 0.4901,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1924479007720947,
      "rewards/margins": 1.0067086219787598,
      "rewards/rejected": -3.1991565227508545,
      "step": 8460
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4637433406005607e-06,
      "logits/chosen": -1.1094534397125244,
      "logits/rejected": -1.2497950792312622,
      "logps/chosen": -571.42431640625,
      "logps/rejected": -641.7213134765625,
      "loss": 0.5045,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.6169791221618652,
      "rewards/margins": 0.9834015965461731,
      "rewards/rejected": -3.6003806591033936,
      "step": 8470
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4580341303251628e-06,
      "logits/chosen": -0.862544059753418,
      "logits/rejected": -0.7224100828170776,
      "logps/chosen": -508.24090576171875,
      "logps/rejected": -622.977783203125,
      "loss": 0.3792,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.4828972816467285,
      "rewards/margins": 1.511356234550476,
      "rewards/rejected": -3.994253635406494,
      "step": 8480
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4523251389642984e-06,
      "logits/chosen": -1.0363171100616455,
      "logits/rejected": -0.7067098617553711,
      "logps/chosen": -539.4168701171875,
      "logps/rejected": -659.4783935546875,
      "loss": 0.422,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.8485755920410156,
      "rewards/margins": 1.571999430656433,
      "rewards/rejected": -4.420575141906738,
      "step": 8490
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4466163962988626e-06,
      "logits/chosen": -1.2876355648040771,
      "logits/rejected": -0.9966632723808289,
      "logps/chosen": -529.1260375976562,
      "logps/rejected": -556.1248779296875,
      "loss": 0.5511,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.485250949859619,
      "rewards/margins": 1.257182002067566,
      "rewards/rejected": -3.7424328327178955,
      "step": 8500
    },
    {
      "epoch": 0.56,
      "eval_logits/chosen": -0.9928444623947144,
      "eval_logits/rejected": -0.8652651906013489,
      "eval_logps/chosen": -498.1315612792969,
      "eval_logps/rejected": -596.16943359375,
      "eval_loss": 0.49668240547180176,
      "eval_rewards/accuracies": 0.6690000295639038,
      "eval_rewards/chosen": -2.6612660884857178,
      "eval_rewards/margins": 1.1843092441558838,
      "eval_rewards/rejected": -3.8455750942230225,
      "eval_runtime": 731.4494,
      "eval_samples_per_second": 2.734,
      "eval_steps_per_second": 1.367,
      "step": 8500
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4409079321084543e-06,
      "logits/chosen": -1.0816056728363037,
      "logits/rejected": -1.2254623174667358,
      "logps/chosen": -443.826416015625,
      "logps/rejected": -604.9515380859375,
      "loss": 0.4718,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.287067413330078,
      "rewards/margins": 1.3118616342544556,
      "rewards/rejected": -3.598928928375244,
      "step": 8510
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4351997761712184e-06,
      "logits/chosen": -1.3557398319244385,
      "logits/rejected": -0.8279944658279419,
      "logps/chosen": -487.305419921875,
      "logps/rejected": -549.6810913085938,
      "loss": 0.405,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.419170618057251,
      "rewards/margins": 1.2908157110214233,
      "rewards/rejected": -3.7099862098693848,
      "step": 8520
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4294919582636933e-06,
      "logits/chosen": -1.2039768695831299,
      "logits/rejected": -0.9999167323112488,
      "logps/chosen": -412.7548828125,
      "logps/rejected": -518.6639404296875,
      "loss": 0.4139,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.016523838043213,
      "rewards/margins": 1.177484154701233,
      "rewards/rejected": -3.1940078735351562,
      "step": 8530
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.423784508160652e-06,
      "logits/chosen": -1.1773066520690918,
      "logits/rejected": -1.0269687175750732,
      "logps/chosen": -514.6947631835938,
      "logps/rejected": -588.3440551757812,
      "loss": 0.4267,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.5946402549743652,
      "rewards/margins": 1.2148663997650146,
      "rewards/rejected": -3.8095061779022217,
      "step": 8540
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.418077455634951e-06,
      "logits/chosen": -1.0692018270492554,
      "logits/rejected": -1.0105565786361694,
      "logps/chosen": -440.36785888671875,
      "logps/rejected": -564.6529541015625,
      "loss": 0.5634,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.2379679679870605,
      "rewards/margins": 0.967279314994812,
      "rewards/rejected": -3.205247163772583,
      "step": 8550
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4123708304573714e-06,
      "logits/chosen": -1.2759517431259155,
      "logits/rejected": -1.0008412599563599,
      "logps/chosen": -499.835693359375,
      "logps/rejected": -604.622314453125,
      "loss": 0.5107,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.1006827354431152,
      "rewards/margins": 1.21485435962677,
      "rewards/rejected": -3.3155369758605957,
      "step": 8560
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.406664662396465e-06,
      "logits/chosen": -0.8560956120491028,
      "logits/rejected": -0.8265968561172485,
      "logps/chosen": -431.65948486328125,
      "logps/rejected": -494.79718017578125,
      "loss": 0.619,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.472776174545288,
      "rewards/margins": 0.7881981730461121,
      "rewards/rejected": -3.260974407196045,
      "step": 8570
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4009589812184012e-06,
      "logits/chosen": -1.178475260734558,
      "logits/rejected": -0.8446548581123352,
      "logps/chosen": -381.1634216308594,
      "logps/rejected": -413.203857421875,
      "loss": 0.4929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.779435157775879,
      "rewards/margins": 0.8515142202377319,
      "rewards/rejected": -2.6309492588043213,
      "step": 8580
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3952538166868073e-06,
      "logits/chosen": -1.0692102909088135,
      "logits/rejected": -1.1181179285049438,
      "logps/chosen": -401.1435546875,
      "logps/rejected": -496.79901123046875,
      "loss": 0.4597,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8359718322753906,
      "rewards/margins": 1.1224865913391113,
      "rewards/rejected": -2.958458423614502,
      "step": 8590
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.389549198562616e-06,
      "logits/chosen": -1.2879879474639893,
      "logits/rejected": -0.9037203788757324,
      "logps/chosen": -403.9749450683594,
      "logps/rejected": -526.24658203125,
      "loss": 0.4126,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7789331674575806,
      "rewards/margins": 1.5225181579589844,
      "rewards/rejected": -3.3014514446258545,
      "step": 8600
    },
    {
      "epoch": 0.56,
      "eval_logits/chosen": -1.238702654838562,
      "eval_logits/rejected": -1.1029170751571655,
      "eval_logps/chosen": -414.6830749511719,
      "eval_logps/rejected": -496.90240478515625,
      "eval_loss": 0.4945288598537445,
      "eval_rewards/accuracies": 0.6664999723434448,
      "eval_rewards/chosen": -1.8267810344696045,
      "eval_rewards/margins": 1.0261238813400269,
      "eval_rewards/rejected": -2.852905035018921,
      "eval_runtime": 732.9791,
      "eval_samples_per_second": 2.729,
      "eval_steps_per_second": 1.364,
      "step": 8600
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3838451566039098e-06,
      "logits/chosen": -1.4302728176116943,
      "logits/rejected": -1.2620681524276733,
      "logps/chosen": -432.6902770996094,
      "logps/rejected": -485.573974609375,
      "loss": 0.5606,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.941802978515625,
      "rewards/margins": 0.630177915096283,
      "rewards/rejected": -2.5719809532165527,
      "step": 8610
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3781417205657662e-06,
      "logits/chosen": -1.325465440750122,
      "logits/rejected": -1.1412758827209473,
      "logps/chosen": -370.631103515625,
      "logps/rejected": -435.3427734375,
      "loss": 0.5132,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.7287461757659912,
      "rewards/margins": 1.029188871383667,
      "rewards/rejected": -2.757934808731079,
      "step": 8620
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3724389202001006e-06,
      "logits/chosen": -1.1985505819320679,
      "logits/rejected": -0.9390853643417358,
      "logps/chosen": -383.4583435058594,
      "logps/rejected": -473.318603515625,
      "loss": 0.4625,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8113489151000977,
      "rewards/margins": 1.1155695915222168,
      "rewards/rejected": -2.9269185066223145,
      "step": 8630
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.366736785255514e-06,
      "logits/chosen": -1.199562907218933,
      "logits/rejected": -1.242752194404602,
      "logps/chosen": -384.41534423828125,
      "logps/rejected": -458.1105041503906,
      "loss": 0.5235,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.8568652868270874,
      "rewards/margins": 0.8491514325141907,
      "rewards/rejected": -2.706017017364502,
      "step": 8640
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3610353454771355e-06,
      "logits/chosen": -1.046150803565979,
      "logits/rejected": -0.9059313535690308,
      "logps/chosen": -342.5575866699219,
      "logps/rejected": -422.2818908691406,
      "loss": 0.5636,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.5261014699935913,
      "rewards/margins": 0.960188090801239,
      "rewards/rejected": -2.4862895011901855,
      "step": 8650
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.355334630606467e-06,
      "logits/chosen": -1.5204510688781738,
      "logits/rejected": -1.2104465961456299,
      "logps/chosen": -435.4972229003906,
      "logps/rejected": -462.814453125,
      "loss": 0.5038,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9520695209503174,
      "rewards/margins": 0.9086283445358276,
      "rewards/rejected": -2.8606975078582764,
      "step": 8660
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.349634670381231e-06,
      "logits/chosen": -1.0414966344833374,
      "logits/rejected": -0.9392479658126831,
      "logps/chosen": -379.5929260253906,
      "logps/rejected": -483.87969970703125,
      "loss": 0.5311,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7239383459091187,
      "rewards/margins": 0.9448808431625366,
      "rewards/rejected": -2.668818950653076,
      "step": 8670
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3439354945352104e-06,
      "logits/chosen": -1.3489819765090942,
      "logits/rejected": -1.2225961685180664,
      "logps/chosen": -402.8128967285156,
      "logps/rejected": -424.5769958496094,
      "loss": 0.5713,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.5928926467895508,
      "rewards/margins": 0.6616193056106567,
      "rewards/rejected": -2.254511594772339,
      "step": 8680
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3382371327981e-06,
      "logits/chosen": -1.326878547668457,
      "logits/rejected": -1.2365591526031494,
      "logps/chosen": -360.7772216796875,
      "logps/rejected": -470.10174560546875,
      "loss": 0.4126,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.293009638786316,
      "rewards/margins": 1.2361931800842285,
      "rewards/rejected": -2.529202938079834,
      "step": 8690
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3325396148953456e-06,
      "logits/chosen": -1.0092710256576538,
      "logits/rejected": -1.130898356437683,
      "logps/chosen": -336.06768798828125,
      "logps/rejected": -493.4744567871094,
      "loss": 0.4881,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6476625204086304,
      "rewards/margins": 1.0327650308609009,
      "rewards/rejected": -2.6804277896881104,
      "step": 8700
    },
    {
      "epoch": 0.57,
      "eval_logits/chosen": -1.238073468208313,
      "eval_logits/rejected": -1.0995856523513794,
      "eval_logps/chosen": -391.00653076171875,
      "eval_logps/rejected": -475.3843688964844,
      "eval_loss": 0.4980272352695465,
      "eval_rewards/accuracies": 0.6620000004768372,
      "eval_rewards/chosen": -1.5900156497955322,
      "eval_rewards/margins": 1.0477092266082764,
      "eval_rewards/rejected": -2.6377248764038086,
      "eval_runtime": 732.2298,
      "eval_samples_per_second": 2.731,
      "eval_steps_per_second": 1.366,
      "step": 8700
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3268429705479915e-06,
      "logits/chosen": -1.59964120388031,
      "logits/rejected": -1.1946229934692383,
      "logps/chosen": -376.1258850097656,
      "logps/rejected": -466.59326171875,
      "loss": 0.3968,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5131326913833618,
      "rewards/margins": 1.3178640604019165,
      "rewards/rejected": -2.8309967517852783,
      "step": 8710
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3211472294725248e-06,
      "logits/chosen": -1.1597530841827393,
      "logits/rejected": -1.0330088138580322,
      "logps/chosen": -386.917724609375,
      "logps/rejected": -511.97967529296875,
      "loss": 0.4025,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7050254344940186,
      "rewards/margins": 1.377290964126587,
      "rewards/rejected": -3.0823163986206055,
      "step": 8720
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.315452421380721e-06,
      "logits/chosen": -0.9799795150756836,
      "logits/rejected": -0.7830728888511658,
      "logps/chosen": -428.2682189941406,
      "logps/rejected": -490.6246643066406,
      "loss": 0.4924,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7332391738891602,
      "rewards/margins": 1.1034952402114868,
      "rewards/rejected": -2.8367342948913574,
      "step": 8730
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3097585759794886e-06,
      "logits/chosen": -1.045185923576355,
      "logits/rejected": -0.8838065266609192,
      "logps/chosen": -425.63525390625,
      "logps/rejected": -519.7047119140625,
      "loss": 0.4465,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.7129396200180054,
      "rewards/margins": 1.5746146440505981,
      "rewards/rejected": -3.2875542640686035,
      "step": 8740
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3040657229707155e-06,
      "logits/chosen": -1.2783961296081543,
      "logits/rejected": -1.0873384475708008,
      "logps/chosen": -344.01971435546875,
      "logps/rejected": -494.1952209472656,
      "loss": 0.4509,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7197210788726807,
      "rewards/margins": 1.3861995935440063,
      "rewards/rejected": -3.1059205532073975,
      "step": 8750
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.2983738920511104e-06,
      "logits/chosen": -1.1936768293380737,
      "logits/rejected": -0.9711049795150757,
      "logps/chosen": -454.77264404296875,
      "logps/rejected": -515.8905029296875,
      "loss": 0.4698,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8497978448867798,
      "rewards/margins": 1.1141988039016724,
      "rewards/rejected": -2.963996410369873,
      "step": 8760
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.2926831129120523e-06,
      "logits/chosen": -0.8634411692619324,
      "logits/rejected": -0.7416626214981079,
      "logps/chosen": -443.77471923828125,
      "logps/rejected": -498.29913330078125,
      "loss": 0.4592,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.0798306465148926,
      "rewards/margins": 0.828801155090332,
      "rewards/rejected": -2.9086318016052246,
      "step": 8770
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.2869934152394323e-06,
      "logits/chosen": -1.1402966976165771,
      "logits/rejected": -1.0761340856552124,
      "logps/chosen": -481.0205993652344,
      "logps/rejected": -516.50146484375,
      "loss": 0.5834,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.120551109313965,
      "rewards/margins": 0.9108297228813171,
      "rewards/rejected": -3.0313808917999268,
      "step": 8780
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.281304828713501e-06,
      "logits/chosen": -1.2651450634002686,
      "logits/rejected": -1.0738184452056885,
      "logps/chosen": -463.01043701171875,
      "logps/rejected": -549.8284912109375,
      "loss": 0.5532,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.300232172012329,
      "rewards/margins": 0.9432049989700317,
      "rewards/rejected": -3.2434372901916504,
      "step": 8790
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.275617383008711e-06,
      "logits/chosen": -1.1871647834777832,
      "logits/rejected": -1.1233766078948975,
      "logps/chosen": -420.621337890625,
      "logps/rejected": -512.9608154296875,
      "loss": 0.4813,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.8513391017913818,
      "rewards/margins": 0.950072169303894,
      "rewards/rejected": -2.8014113903045654,
      "step": 8800
    },
    {
      "epoch": 0.58,
      "eval_logits/chosen": -1.149058222770691,
      "eval_logits/rejected": -1.0135620832443237,
      "eval_logps/chosen": -418.1949157714844,
      "eval_logps/rejected": -509.9336242675781,
      "eval_loss": 0.49589085578918457,
      "eval_rewards/accuracies": 0.6620000004768372,
      "eval_rewards/chosen": -1.861899495124817,
      "eval_rewards/margins": 1.121317744255066,
      "eval_rewards/rejected": -2.9832170009613037,
      "eval_runtime": 732.1104,
      "eval_samples_per_second": 2.732,
      "eval_steps_per_second": 1.366,
      "step": 8800
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.269931107793567e-06,
      "logits/chosen": -0.9232932925224304,
      "logits/rejected": -0.8948391079902649,
      "logps/chosen": -370.1566162109375,
      "logps/rejected": -467.27716064453125,
      "loss": 0.5171,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5787628889083862,
      "rewards/margins": 0.8902810215950012,
      "rewards/rejected": -2.4690439701080322,
      "step": 8810
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2642460327304655e-06,
      "logits/chosen": -1.26815927028656,
      "logits/rejected": -1.2135612964630127,
      "logps/chosen": -416.7547302246094,
      "logps/rejected": -502.16326904296875,
      "loss": 0.5601,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.721949577331543,
      "rewards/margins": 1.0125608444213867,
      "rewards/rejected": -2.734510660171509,
      "step": 8820
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.258562187475543e-06,
      "logits/chosen": -1.1809900999069214,
      "logits/rejected": -0.848108172416687,
      "logps/chosen": -405.0650634765625,
      "logps/rejected": -456.2456970214844,
      "loss": 0.5569,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7811815738677979,
      "rewards/margins": 0.8869251012802124,
      "rewards/rejected": -2.6681065559387207,
      "step": 8830
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2528796016785196e-06,
      "logits/chosen": -0.9581031799316406,
      "logits/rejected": -0.9923402667045593,
      "logps/chosen": -365.490966796875,
      "logps/rejected": -495.0604553222656,
      "loss": 0.4604,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.755730390548706,
      "rewards/margins": 1.1990342140197754,
      "rewards/rejected": -2.9547646045684814,
      "step": 8840
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.247198304982548e-06,
      "logits/chosen": -1.0048309564590454,
      "logits/rejected": -0.9457969665527344,
      "logps/chosen": -321.87359619140625,
      "logps/rejected": -413.46710205078125,
      "loss": 0.4841,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5990204811096191,
      "rewards/margins": 0.9554295539855957,
      "rewards/rejected": -2.554450273513794,
      "step": 8850
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2415183270240533e-06,
      "logits/chosen": -1.4782264232635498,
      "logits/rejected": -1.3399109840393066,
      "logps/chosen": -394.8039855957031,
      "logps/rejected": -499.06524658203125,
      "loss": 0.5049,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.9547436237335205,
      "rewards/margins": 0.9995563626289368,
      "rewards/rejected": -2.9542996883392334,
      "step": 8860
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2358396974325837e-06,
      "logits/chosen": -1.071343183517456,
      "logits/rejected": -1.0277284383773804,
      "logps/chosen": -408.5313720703125,
      "logps/rejected": -500.9320373535156,
      "loss": 0.4677,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.662637710571289,
      "rewards/margins": 1.18667733669281,
      "rewards/rejected": -2.8493151664733887,
      "step": 8870
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2301624458306525e-06,
      "logits/chosen": -1.1136915683746338,
      "logits/rejected": -1.0989829301834106,
      "logps/chosen": -490.1780700683594,
      "logps/rejected": -535.048095703125,
      "loss": 0.5163,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.308300018310547,
      "rewards/margins": 0.9621118307113647,
      "rewards/rejected": -3.270411968231201,
      "step": 8880
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2244866018335855e-06,
      "logits/chosen": -1.0626459121704102,
      "logits/rejected": -1.003996729850769,
      "logps/chosen": -422.763671875,
      "logps/rejected": -497.86285400390625,
      "loss": 0.6432,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.0570597648620605,
      "rewards/margins": 0.6069216728210449,
      "rewards/rejected": -2.6639814376831055,
      "step": 8890
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2188121950493648e-06,
      "logits/chosen": -1.2248162031173706,
      "logits/rejected": -0.8637872934341431,
      "logps/chosen": -421.6446838378906,
      "logps/rejected": -421.966552734375,
      "loss": 0.535,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.0290122032165527,
      "rewards/margins": 0.7857424020767212,
      "rewards/rejected": -2.8147549629211426,
      "step": 8900
    },
    {
      "epoch": 0.58,
      "eval_logits/chosen": -1.0819333791732788,
      "eval_logits/rejected": -0.9508689641952515,
      "eval_logps/chosen": -436.3647766113281,
      "eval_logps/rejected": -526.4249267578125,
      "eval_loss": 0.4916088879108429,
      "eval_rewards/accuracies": 0.6660000085830688,
      "eval_rewards/chosen": -2.043597936630249,
      "eval_rewards/margins": 1.1045318841934204,
      "eval_rewards/rejected": -3.148129940032959,
      "eval_runtime": 732.6366,
      "eval_samples_per_second": 2.73,
      "eval_steps_per_second": 1.365,
      "step": 8900
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2131392550784766e-06,
      "logits/chosen": -1.1720525026321411,
      "logits/rejected": -0.8352081179618835,
      "logps/chosen": -497.7474670410156,
      "logps/rejected": -506.4822692871094,
      "loss": 0.4338,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.128687620162964,
      "rewards/margins": 1.0429903268814087,
      "rewards/rejected": -3.171678066253662,
      "step": 8910
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2074678115137533e-06,
      "logits/chosen": -1.1111485958099365,
      "logits/rejected": -0.9204902648925781,
      "logps/chosen": -370.63104248046875,
      "logps/rejected": -509.78564453125,
      "loss": 0.4269,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.759141206741333,
      "rewards/margins": 1.3423799276351929,
      "rewards/rejected": -3.1015210151672363,
      "step": 8920
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.201797893940224e-06,
      "logits/chosen": -0.8966342806816101,
      "logits/rejected": -0.9461180567741394,
      "logps/chosen": -429.45947265625,
      "logps/rejected": -564.6712646484375,
      "loss": 0.4905,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.962821364402771,
      "rewards/margins": 1.148612380027771,
      "rewards/rejected": -3.111433267593384,
      "step": 8930
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.196129531934956e-06,
      "logits/chosen": -0.9999095797538757,
      "logits/rejected": -0.9462803602218628,
      "logps/chosen": -408.2229919433594,
      "logps/rejected": -495.52923583984375,
      "loss": 0.5563,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.703089714050293,
      "rewards/margins": 1.0536882877349854,
      "rewards/rejected": -2.7567780017852783,
      "step": 8940
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.190462755066902e-06,
      "logits/chosen": -1.1943506002426147,
      "logits/rejected": -0.9096410870552063,
      "logps/chosen": -478.9681091308594,
      "logps/rejected": -559.5634155273438,
      "loss": 0.4374,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1656689643859863,
      "rewards/margins": 1.0608609914779663,
      "rewards/rejected": -3.2265305519104004,
      "step": 8950
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.184797592896746e-06,
      "logits/chosen": -1.2017793655395508,
      "logits/rejected": -1.08447265625,
      "logps/chosen": -423.525146484375,
      "logps/rejected": -497.7538146972656,
      "loss": 0.5069,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8908952474594116,
      "rewards/margins": 1.0076181888580322,
      "rewards/rejected": -2.8985133171081543,
      "step": 8960
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.17913407497675e-06,
      "logits/chosen": -1.0965301990509033,
      "logits/rejected": -1.0276085138320923,
      "logps/chosen": -331.51068115234375,
      "logps/rejected": -470.09912109375,
      "loss": 0.4991,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.5312869548797607,
      "rewards/margins": 1.0463721752166748,
      "rewards/rejected": -2.5776591300964355,
      "step": 8970
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.173472230850596e-06,
      "logits/chosen": -1.2828447818756104,
      "logits/rejected": -0.9710990786552429,
      "logps/chosen": -350.43133544921875,
      "logps/rejected": -389.3118591308594,
      "loss": 0.5289,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.5016876459121704,
      "rewards/margins": 0.7932427525520325,
      "rewards/rejected": -2.2949302196502686,
      "step": 8980
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1678120900532375e-06,
      "logits/chosen": -1.0349891185760498,
      "logits/rejected": -0.8604212999343872,
      "logps/chosen": -460.71142578125,
      "logps/rejected": -554.3140869140625,
      "loss": 0.4128,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.2479658126831055,
      "rewards/margins": 1.2286293506622314,
      "rewards/rejected": -3.476595401763916,
      "step": 8990
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1621536821107412e-06,
      "logits/chosen": -1.0512446165084839,
      "logits/rejected": -0.8796029090881348,
      "logps/chosen": -385.85906982421875,
      "logps/rejected": -437.2776794433594,
      "loss": 0.5399,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8328920602798462,
      "rewards/margins": 0.9345098733901978,
      "rewards/rejected": -2.767401933670044,
      "step": 9000
    },
    {
      "epoch": 0.59,
      "eval_logits/chosen": -1.0397822856903076,
      "eval_logits/rejected": -0.9097519516944885,
      "eval_logps/chosen": -422.9480895996094,
      "eval_logps/rejected": -515.3348999023438,
      "eval_loss": 0.4937884509563446,
      "eval_rewards/accuracies": 0.6629999876022339,
      "eval_rewards/chosen": -1.9094310998916626,
      "eval_rewards/margins": 1.1277989149093628,
      "eval_rewards/rejected": -3.0372297763824463,
      "eval_runtime": 731.1211,
      "eval_samples_per_second": 2.736,
      "eval_steps_per_second": 1.368,
      "step": 9000
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1564970365401346e-06,
      "logits/chosen": -1.2500946521759033,
      "logits/rejected": -0.9484453201293945,
      "logps/chosen": -394.15130615234375,
      "logps/rejected": -437.1795349121094,
      "loss": 0.6583,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.1000022888183594,
      "rewards/margins": 0.8103631138801575,
      "rewards/rejected": -2.910365581512451,
      "step": 9010
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1508421828492527e-06,
      "logits/chosen": -1.31488037109375,
      "logits/rejected": -1.0038257837295532,
      "logps/chosen": -388.72802734375,
      "logps/rejected": -420.581298828125,
      "loss": 0.5326,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.6353528499603271,
      "rewards/margins": 1.0009357929229736,
      "rewards/rejected": -2.636288642883301,
      "step": 9020
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.145189150536582e-06,
      "logits/chosen": -1.0028223991394043,
      "logits/rejected": -0.9240818023681641,
      "logps/chosen": -405.3058776855469,
      "logps/rejected": -448.379150390625,
      "loss": 0.5105,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8313636779785156,
      "rewards/margins": 0.9247555732727051,
      "rewards/rejected": -2.7561192512512207,
      "step": 9030
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.139537969091107e-06,
      "logits/chosen": -0.8752814531326294,
      "logits/rejected": -0.8566150665283203,
      "logps/chosen": -435.94989013671875,
      "logps/rejected": -458.0609436035156,
      "loss": 0.5249,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7036616802215576,
      "rewards/margins": 0.8428160548210144,
      "rewards/rejected": -2.546477794647217,
      "step": 9040
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1338886679921603e-06,
      "logits/chosen": -1.0392322540283203,
      "logits/rejected": -0.9704221487045288,
      "logps/chosen": -401.290771484375,
      "logps/rejected": -473.9989318847656,
      "loss": 0.4822,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.5974773168563843,
      "rewards/margins": 0.9311855435371399,
      "rewards/rejected": -2.528662919998169,
      "step": 9050
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.128241276709263e-06,
      "logits/chosen": -1.2574546337127686,
      "logits/rejected": -1.3090790510177612,
      "logps/chosen": -331.599853515625,
      "logps/rejected": -454.751953125,
      "loss": 0.4899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2833807468414307,
      "rewards/margins": 1.0311470031738281,
      "rewards/rejected": -2.3145275115966797,
      "step": 9060
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1225958247019746e-06,
      "logits/chosen": -1.4055688381195068,
      "logits/rejected": -1.432252049446106,
      "logps/chosen": -336.8355407714844,
      "logps/rejected": -445.4163513183594,
      "loss": 0.4993,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4969367980957031,
      "rewards/margins": 0.8844121098518372,
      "rewards/rejected": -2.3813490867614746,
      "step": 9070
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1169523414197383e-06,
      "logits/chosen": -1.077289342880249,
      "logits/rejected": -1.0037438869476318,
      "logps/chosen": -342.13739013671875,
      "logps/rejected": -416.0020446777344,
      "loss": 0.6481,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -1.4333170652389526,
      "rewards/margins": 0.505429744720459,
      "rewards/rejected": -1.938746690750122,
      "step": 9080
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1113108563017267e-06,
      "logits/chosen": -1.0560085773468018,
      "logits/rejected": -1.0217021703720093,
      "logps/chosen": -385.6211242675781,
      "logps/rejected": -463.80078125,
      "loss": 0.5424,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7283786535263062,
      "rewards/margins": 1.0880886316299438,
      "rewards/rejected": -2.81646728515625,
      "step": 9090
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.1056713987766905e-06,
      "logits/chosen": -1.2946921586990356,
      "logits/rejected": -1.1029138565063477,
      "logps/chosen": -377.98126220703125,
      "logps/rejected": -428.6024475097656,
      "loss": 0.512,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6278095245361328,
      "rewards/margins": 0.9933661222457886,
      "rewards/rejected": -2.62117600440979,
      "step": 9100
    },
    {
      "epoch": 0.6,
      "eval_logits/chosen": -1.2002038955688477,
      "eval_logits/rejected": -1.0658358335494995,
      "eval_logps/chosen": -383.3267517089844,
      "eval_logps/rejected": -461.37103271484375,
      "eval_loss": 0.4937075674533844,
      "eval_rewards/accuracies": 0.6729999780654907,
      "eval_rewards/chosen": -1.5132176876068115,
      "eval_rewards/margins": 0.9843739867210388,
      "eval_rewards/rejected": -2.497591495513916,
      "eval_runtime": 733.2161,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 9100
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.1000339982628022e-06,
      "logits/chosen": -1.0017447471618652,
      "logits/rejected": -0.913290798664093,
      "logps/chosen": -436.9093322753906,
      "logps/rejected": -467.470947265625,
      "loss": 0.5786,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.8563644886016846,
      "rewards/margins": 0.6488175988197327,
      "rewards/rejected": -2.5051820278167725,
      "step": 9110
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0943986841675043e-06,
      "logits/chosen": -1.2511661052703857,
      "logits/rejected": -0.9566980600357056,
      "logps/chosen": -357.22930908203125,
      "logps/rejected": -419.00506591796875,
      "loss": 0.54,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.540183186531067,
      "rewards/margins": 0.8111560940742493,
      "rewards/rejected": -2.351339340209961,
      "step": 9120
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.088765485887356e-06,
      "logits/chosen": -1.2423206567764282,
      "logits/rejected": -0.9856125116348267,
      "logps/chosen": -393.86920166015625,
      "logps/rejected": -431.34765625,
      "loss": 0.5275,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.491542935371399,
      "rewards/margins": 0.7910388112068176,
      "rewards/rejected": -2.2825820446014404,
      "step": 9130
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.083134432807879e-06,
      "logits/chosen": -1.2834888696670532,
      "logits/rejected": -1.0755784511566162,
      "logps/chosen": -349.40350341796875,
      "logps/rejected": -488.70391845703125,
      "loss": 0.5207,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.551313877105713,
      "rewards/margins": 1.1778234243392944,
      "rewards/rejected": -2.729137420654297,
      "step": 9140
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.077505554303404e-06,
      "logits/chosen": -1.0973273515701294,
      "logits/rejected": -1.085224986076355,
      "logps/chosen": -307.36785888671875,
      "logps/rejected": -404.21368408203125,
      "loss": 0.4621,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3281586170196533,
      "rewards/margins": 0.9247921705245972,
      "rewards/rejected": -2.252951145172119,
      "step": 9150
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.071878879736918e-06,
      "logits/chosen": -1.23152756690979,
      "logits/rejected": -1.0925686359405518,
      "logps/chosen": -442.94647216796875,
      "logps/rejected": -604.2838134765625,
      "loss": 0.5539,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.961069107055664,
      "rewards/margins": 0.8305347561836243,
      "rewards/rejected": -2.7916038036346436,
      "step": 9160
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0662544384599136e-06,
      "logits/chosen": -1.0271787643432617,
      "logits/rejected": -0.8980680704116821,
      "logps/chosen": -358.5318908691406,
      "logps/rejected": -441.77337646484375,
      "loss": 0.5172,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5642528533935547,
      "rewards/margins": 0.9972681999206543,
      "rewards/rejected": -2.56152081489563,
      "step": 9170
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0606322598122314e-06,
      "logits/chosen": -0.9368621110916138,
      "logits/rejected": -1.0792651176452637,
      "logps/chosen": -381.1671447753906,
      "logps/rejected": -444.1542053222656,
      "loss": 0.6215,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.9465354681015015,
      "rewards/margins": 0.43326178193092346,
      "rewards/rejected": -2.3797974586486816,
      "step": 9180
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0550123731219085e-06,
      "logits/chosen": -1.480641484260559,
      "logits/rejected": -1.0889581441879272,
      "logps/chosen": -416.18988037109375,
      "logps/rejected": -478.01580810546875,
      "loss": 0.4067,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6301004886627197,
      "rewards/margins": 1.029707670211792,
      "rewards/rejected": -2.659808397293091,
      "step": 9190
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0493948077050267e-06,
      "logits/chosen": -0.734448254108429,
      "logits/rejected": -0.6183160543441772,
      "logps/chosen": -385.55157470703125,
      "logps/rejected": -468.9554748535156,
      "loss": 0.5069,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.9229555130004883,
      "rewards/margins": 1.0149352550506592,
      "rewards/rejected": -2.9378905296325684,
      "step": 9200
    },
    {
      "epoch": 0.6,
      "eval_logits/chosen": -1.0345836877822876,
      "eval_logits/rejected": -0.9101244211196899,
      "eval_logps/chosen": -411.07568359375,
      "eval_logps/rejected": -487.1391906738281,
      "eval_loss": 0.49305132031440735,
      "eval_rewards/accuracies": 0.671500027179718,
      "eval_rewards/chosen": -1.7907072305679321,
      "eval_rewards/margins": 0.9645655751228333,
      "eval_rewards/rejected": -2.755272626876831,
      "eval_runtime": 733.6293,
      "eval_samples_per_second": 2.726,
      "eval_steps_per_second": 1.363,
      "step": 9200
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0437795928655596e-06,
      "logits/chosen": -1.1340954303741455,
      "logits/rejected": -1.1976003646850586,
      "logps/chosen": -471.8667907714844,
      "logps/rejected": -520.8145751953125,
      "loss": 0.549,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.9015413522720337,
      "rewards/margins": 0.6713147759437561,
      "rewards/rejected": -2.5728561878204346,
      "step": 9210
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0381667578952184e-06,
      "logits/chosen": -1.1571053266525269,
      "logits/rejected": -1.022148609161377,
      "logps/chosen": -411.462646484375,
      "logps/rejected": -526.1029052734375,
      "loss": 0.4672,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.0058977603912354,
      "rewards/margins": 1.0902332067489624,
      "rewards/rejected": -3.0961310863494873,
      "step": 9220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0325563320732995e-06,
      "logits/chosen": -1.2341101169586182,
      "logits/rejected": -1.111156702041626,
      "logps/chosen": -445.73834228515625,
      "logps/rejected": -505.947021484375,
      "loss": 0.4702,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8240001201629639,
      "rewards/margins": 1.042265772819519,
      "rewards/rejected": -2.8662657737731934,
      "step": 9230
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.026948344666532e-06,
      "logits/chosen": -0.911036491394043,
      "logits/rejected": -1.0102976560592651,
      "logps/chosen": -382.86279296875,
      "logps/rejected": -468.21502685546875,
      "loss": 0.5338,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.837368369102478,
      "rewards/margins": 0.8579694032669067,
      "rewards/rejected": -2.6953375339508057,
      "step": 9240
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.0213428249289257e-06,
      "logits/chosen": -0.6779653429985046,
      "logits/rejected": -0.8694950938224792,
      "logps/chosen": -363.29046630859375,
      "logps/rejected": -479.02117919921875,
      "loss": 0.4412,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6474993228912354,
      "rewards/margins": 1.1392320394515991,
      "rewards/rejected": -2.786731243133545,
      "step": 9250
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.0157398021016175e-06,
      "logits/chosen": -0.9124547839164734,
      "logits/rejected": -0.9362983703613281,
      "logps/chosen": -314.9878845214844,
      "logps/rejected": -436.501220703125,
      "loss": 0.5809,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.6745256185531616,
      "rewards/margins": 0.7530856132507324,
      "rewards/rejected": -2.4276108741760254,
      "step": 9260
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.010139305412719e-06,
      "logits/chosen": -1.4748836755752563,
      "logits/rejected": -1.1832959651947021,
      "logps/chosen": -457.10833740234375,
      "logps/rejected": -528.8362426757812,
      "loss": 0.4254,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8051267862319946,
      "rewards/margins": 1.1409196853637695,
      "rewards/rejected": -2.9460463523864746,
      "step": 9270
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.0045413640771644e-06,
      "logits/chosen": -1.121006965637207,
      "logits/rejected": -0.9204033017158508,
      "logps/chosen": -465.58013916015625,
      "logps/rejected": -592.5983276367188,
      "loss": 0.4731,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1041476726531982,
      "rewards/margins": 1.2376271486282349,
      "rewards/rejected": -3.3417747020721436,
      "step": 9280
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.998946007296558e-06,
      "logits/chosen": -1.1622593402862549,
      "logits/rejected": -1.052321195602417,
      "logps/chosen": -503.66455078125,
      "logps/rejected": -544.9689331054688,
      "loss": 0.4878,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9506393671035767,
      "rewards/margins": 1.0474134683609009,
      "rewards/rejected": -2.9980528354644775,
      "step": 9290
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9933532642590215e-06,
      "logits/chosen": -0.801226794719696,
      "logits/rejected": -0.5509706735610962,
      "logps/chosen": -321.21893310546875,
      "logps/rejected": -397.5530090332031,
      "loss": 0.4272,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3206064701080322,
      "rewards/margins": 1.2649487257003784,
      "rewards/rejected": -2.5855555534362793,
      "step": 9300
    },
    {
      "epoch": 0.61,
      "eval_logits/chosen": -1.0575023889541626,
      "eval_logits/rejected": -0.9299944043159485,
      "eval_logps/chosen": -413.5278015136719,
      "eval_logps/rejected": -500.4742126464844,
      "eval_loss": 0.49185654520988464,
      "eval_rewards/accuracies": 0.6729999780654907,
      "eval_rewards/chosen": -1.8152281045913696,
      "eval_rewards/margins": 1.073394775390625,
      "eval_rewards/rejected": -2.888622999191284,
      "eval_runtime": 731.6322,
      "eval_samples_per_second": 2.734,
      "eval_steps_per_second": 1.367,
      "step": 9300
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.987763164139042e-06,
      "logits/chosen": -1.2010269165039062,
      "logits/rejected": -0.964155375957489,
      "logps/chosen": -373.0898742675781,
      "logps/rejected": -489.07989501953125,
      "loss": 0.5161,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6783478260040283,
      "rewards/margins": 1.1373569965362549,
      "rewards/rejected": -2.815704822540283,
      "step": 9310
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.982175736097321e-06,
      "logits/chosen": -0.9910721778869629,
      "logits/rejected": -0.963726818561554,
      "logps/chosen": -491.3431701660156,
      "logps/rejected": -576.5433349609375,
      "loss": 0.6689,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.168020725250244,
      "rewards/margins": 0.6818044781684875,
      "rewards/rejected": -2.849825382232666,
      "step": 9320
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9765910092806196e-06,
      "logits/chosen": -1.0442947149276733,
      "logits/rejected": -0.9146569967269897,
      "logps/chosen": -322.37860107421875,
      "logps/rejected": -365.389892578125,
      "loss": 0.5562,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4430307149887085,
      "rewards/margins": 0.6266955137252808,
      "rewards/rejected": -2.0697264671325684,
      "step": 9330
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9710090128216083e-06,
      "logits/chosen": -1.1570490598678589,
      "logits/rejected": -0.9983047246932983,
      "logps/chosen": -426.0396423339844,
      "logps/rejected": -540.4515991210938,
      "loss": 0.4606,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.0330958366394043,
      "rewards/margins": 1.305750846862793,
      "rewards/rejected": -3.3388469219207764,
      "step": 9340
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9654297758387155e-06,
      "logits/chosen": -1.0099811553955078,
      "logits/rejected": -0.8388859629631042,
      "logps/chosen": -346.5732116699219,
      "logps/rejected": -471.65362548828125,
      "loss": 0.5067,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.857031226158142,
      "rewards/margins": 1.0435500144958496,
      "rewards/rejected": -2.9005813598632812,
      "step": 9350
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9598533274359736e-06,
      "logits/chosen": -0.9493533968925476,
      "logits/rejected": -1.040316104888916,
      "logps/chosen": -452.26397705078125,
      "logps/rejected": -495.58001708984375,
      "loss": 0.7206,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.1258368492126465,
      "rewards/margins": 0.3595449924468994,
      "rewards/rejected": -2.485381603240967,
      "step": 9360
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9542796967028697e-06,
      "logits/chosen": -1.2399563789367676,
      "logits/rejected": -0.9904924631118774,
      "logps/chosen": -399.768310546875,
      "logps/rejected": -469.8306579589844,
      "loss": 0.4607,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8152774572372437,
      "rewards/margins": 0.8830522298812866,
      "rewards/rejected": -2.6983296871185303,
      "step": 9370
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.948708912714192e-06,
      "logits/chosen": -0.6782970428466797,
      "logits/rejected": -0.7653140425682068,
      "logps/chosen": -454.6788024902344,
      "logps/rejected": -513.2496337890625,
      "loss": 0.5389,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.0773351192474365,
      "rewards/margins": 0.8950663805007935,
      "rewards/rejected": -2.9724013805389404,
      "step": 9380
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9431410045298786e-06,
      "logits/chosen": -0.7093806862831116,
      "logits/rejected": -0.8312298059463501,
      "logps/chosen": -424.17620849609375,
      "logps/rejected": -498.23980712890625,
      "loss": 0.5269,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.0705084800720215,
      "rewards/margins": 0.7503930330276489,
      "rewards/rejected": -2.820901393890381,
      "step": 9390
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9375760011948654e-06,
      "logits/chosen": -1.0410220623016357,
      "logits/rejected": -1.040238857269287,
      "logps/chosen": -384.260009765625,
      "logps/rejected": -512.1881713867188,
      "loss": 0.4398,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.845820665359497,
      "rewards/margins": 1.0560771226882935,
      "rewards/rejected": -2.901897668838501,
      "step": 9400
    },
    {
      "epoch": 0.62,
      "eval_logits/chosen": -0.9658358693122864,
      "eval_logits/rejected": -0.8458749651908875,
      "eval_logps/chosen": -438.275634765625,
      "eval_logps/rejected": -514.0955810546875,
      "eval_loss": 0.49358147382736206,
      "eval_rewards/accuracies": 0.6704999804496765,
      "eval_rewards/chosen": -2.062706470489502,
      "eval_rewards/margins": 0.9621301889419556,
      "eval_rewards/rejected": -3.024836778640747,
      "eval_runtime": 730.7547,
      "eval_samples_per_second": 2.737,
      "eval_steps_per_second": 1.368,
      "step": 9400
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.932013931738937e-06,
      "logits/chosen": -0.9855095148086548,
      "logits/rejected": -0.7755752205848694,
      "logps/chosen": -410.9771423339844,
      "logps/rejected": -556.1227416992188,
      "loss": 0.3902,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.073754072189331,
      "rewards/margins": 1.322638750076294,
      "rewards/rejected": -3.396392822265625,
      "step": 9410
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9264548251765717e-06,
      "logits/chosen": -1.0908194780349731,
      "logits/rejected": -1.0226426124572754,
      "logps/chosen": -389.06085205078125,
      "logps/rejected": -518.1353759765625,
      "loss": 0.3952,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.8525817394256592,
      "rewards/margins": 1.3218623399734497,
      "rewards/rejected": -3.1744441986083984,
      "step": 9420
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9208987105067924e-06,
      "logits/chosen": -0.7906646132469177,
      "logits/rejected": -0.6241647601127625,
      "logps/chosen": -440.5582580566406,
      "logps/rejected": -484.5211486816406,
      "loss": 0.6315,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.260011911392212,
      "rewards/margins": 0.6785053610801697,
      "rewards/rejected": -2.9385170936584473,
      "step": 9430
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9153456167130154e-06,
      "logits/chosen": -0.8590573072433472,
      "logits/rejected": -0.8843520283699036,
      "logps/chosen": -428.5514221191406,
      "logps/rejected": -539.2299194335938,
      "loss": 0.4938,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2448079586029053,
      "rewards/margins": 0.8447982668876648,
      "rewards/rejected": -3.0896060466766357,
      "step": 9440
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9097955727628975e-06,
      "logits/chosen": -1.049441933631897,
      "logits/rejected": -1.0770914554595947,
      "logps/chosen": -410.7098693847656,
      "logps/rejected": -502.4935607910156,
      "loss": 0.5324,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.1472437381744385,
      "rewards/margins": 0.7984870672225952,
      "rewards/rejected": -2.945730686187744,
      "step": 9450
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.904248607608187e-06,
      "logits/chosen": -0.6168997883796692,
      "logits/rejected": -0.8384846448898315,
      "logps/chosen": -466.0945739746094,
      "logps/rejected": -519.4091186523438,
      "loss": 0.4144,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.0887458324432373,
      "rewards/margins": 1.004212737083435,
      "rewards/rejected": -3.092958927154541,
      "step": 9460
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8987047501845714e-06,
      "logits/chosen": -0.9473953247070312,
      "logits/rejected": -0.7209688425064087,
      "logps/chosen": -393.9309997558594,
      "logps/rejected": -504.14508056640625,
      "loss": 0.4709,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.2700653076171875,
      "rewards/margins": 1.1645954847335815,
      "rewards/rejected": -3.4346611499786377,
      "step": 9470
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8931640294115267e-06,
      "logits/chosen": -0.6625471115112305,
      "logits/rejected": -0.43366652727127075,
      "logps/chosen": -418.9068298339844,
      "logps/rejected": -512.6217651367188,
      "loss": 0.4559,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.249008893966675,
      "rewards/margins": 1.0924519300460815,
      "rewards/rejected": -3.341460704803467,
      "step": 9480
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8876264741921662e-06,
      "logits/chosen": -0.7319644689559937,
      "logits/rejected": -0.6956400871276855,
      "logps/chosen": -423.14947509765625,
      "logps/rejected": -560.1439208984375,
      "loss": 0.4691,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.330239772796631,
      "rewards/margins": 1.4242993593215942,
      "rewards/rejected": -3.7545394897460938,
      "step": 9490
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8820921134130912e-06,
      "logits/chosen": -0.9113744497299194,
      "logits/rejected": -0.5819921493530273,
      "logps/chosen": -476.8115234375,
      "logps/rejected": -583.10400390625,
      "loss": 0.498,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.462993621826172,
      "rewards/margins": 1.5120172500610352,
      "rewards/rejected": -3.975010633468628,
      "step": 9500
    },
    {
      "epoch": 0.62,
      "eval_logits/chosen": -0.7636699676513672,
      "eval_logits/rejected": -0.6522924304008484,
      "eval_logps/chosen": -485.16644287109375,
      "eval_logps/rejected": -572.1414184570312,
      "eval_loss": 0.49296244978904724,
      "eval_rewards/accuracies": 0.6644999980926514,
      "eval_rewards/chosen": -2.5316150188446045,
      "eval_rewards/margins": 1.07368004322052,
      "eval_rewards/rejected": -3.605294942855835,
      "eval_runtime": 733.6355,
      "eval_samples_per_second": 2.726,
      "eval_steps_per_second": 1.363,
      "step": 9500
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8765609759442378e-06,
      "logits/chosen": -0.34705209732055664,
      "logits/rejected": -0.4685601592063904,
      "logps/chosen": -509.03216552734375,
      "logps/rejected": -588.2866821289062,
      "loss": 0.4986,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.6771090030670166,
      "rewards/margins": 0.9006662368774414,
      "rewards/rejected": -3.577775239944458,
      "step": 9510
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8710330906387288e-06,
      "logits/chosen": -1.2283780574798584,
      "logits/rejected": -1.1918247938156128,
      "logps/chosen": -496.95623779296875,
      "logps/rejected": -614.5147094726562,
      "loss": 0.4733,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.584685802459717,
      "rewards/margins": 0.9396859407424927,
      "rewards/rejected": -3.52437162399292,
      "step": 9520
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8655084863327222e-06,
      "logits/chosen": -0.8194085955619812,
      "logits/rejected": -0.7681763768196106,
      "logps/chosen": -369.12579345703125,
      "logps/rejected": -436.62646484375,
      "loss": 0.6334,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.8566782474517822,
      "rewards/margins": 0.6133848428726196,
      "rewards/rejected": -2.4700629711151123,
      "step": 9530
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8599871918452603e-06,
      "logits/chosen": -0.7199943661689758,
      "logits/rejected": -0.789899468421936,
      "logps/chosen": -427.11700439453125,
      "logps/rejected": -527.3338012695312,
      "loss": 0.5339,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.0402514934539795,
      "rewards/margins": 0.8696733713150024,
      "rewards/rejected": -2.9099249839782715,
      "step": 9540
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8544692359781192e-06,
      "logits/chosen": -0.7468751072883606,
      "logits/rejected": -0.7449430227279663,
      "logps/chosen": -369.1566467285156,
      "logps/rejected": -454.22149658203125,
      "loss": 0.4088,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8157199621200562,
      "rewards/margins": 1.1601203680038452,
      "rewards/rejected": -2.9758403301239014,
      "step": 9550
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8489546475156602e-06,
      "logits/chosen": -1.0949957370758057,
      "logits/rejected": -1.0754151344299316,
      "logps/chosen": -448.9339294433594,
      "logps/rejected": -527.186279296875,
      "loss": 0.5017,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.2944109439849854,
      "rewards/margins": 0.9642025828361511,
      "rewards/rejected": -3.258613109588623,
      "step": 9560
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8434434552246778e-06,
      "logits/chosen": -0.7982507944107056,
      "logits/rejected": -0.7856664657592773,
      "logps/chosen": -393.72259521484375,
      "logps/rejected": -494.39874267578125,
      "loss": 0.4493,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.84091055393219,
      "rewards/margins": 1.1193912029266357,
      "rewards/rejected": -2.9603018760681152,
      "step": 9570
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.837935687854251e-06,
      "logits/chosen": -0.9066284894943237,
      "logits/rejected": -0.7065574526786804,
      "logps/chosen": -458.378662109375,
      "logps/rejected": -556.5958251953125,
      "loss": 0.4269,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.368319511413574,
      "rewards/margins": 1.3001726865768433,
      "rewards/rejected": -3.668492078781128,
      "step": 9580
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.832431374135592e-06,
      "logits/chosen": -0.9596296548843384,
      "logits/rejected": -1.0166507959365845,
      "logps/chosen": -492.32879638671875,
      "logps/rejected": -600.0111694335938,
      "loss": 0.4765,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.438934564590454,
      "rewards/margins": 1.2655547857284546,
      "rewards/rejected": -3.7044894695281982,
      "step": 9590
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8269305427818977e-06,
      "logits/chosen": -0.9456634521484375,
      "logits/rejected": -0.8710576891899109,
      "logps/chosen": -447.292724609375,
      "logps/rejected": -526.4971923828125,
      "loss": 0.4865,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.3867650032043457,
      "rewards/margins": 1.0686821937561035,
      "rewards/rejected": -3.455446720123291,
      "step": 9600
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -0.7693200707435608,
      "eval_logits/rejected": -0.6561520099639893,
      "eval_logps/chosen": -475.1277770996094,
      "eval_logps/rejected": -570.9478759765625,
      "eval_loss": 0.4916233420372009,
      "eval_rewards/accuracies": 0.6685000061988831,
      "eval_rewards/chosen": -2.431227922439575,
      "eval_rewards/margins": 1.1621315479278564,
      "eval_rewards/rejected": -3.5933594703674316,
      "eval_runtime": 733.5856,
      "eval_samples_per_second": 2.726,
      "eval_steps_per_second": 1.363,
      "step": 9600
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.821433222488199e-06,
      "logits/chosen": -0.4729815125465393,
      "logits/rejected": -0.5782935619354248,
      "logps/chosen": -471.65423583984375,
      "logps/rejected": -550.3768310546875,
      "loss": 0.5554,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.460395097732544,
      "rewards/margins": 1.0647014379501343,
      "rewards/rejected": -3.5250961780548096,
      "step": 9610
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8159394419312112e-06,
      "logits/chosen": -0.9872452616691589,
      "logits/rejected": -0.7170965075492859,
      "logps/chosen": -487.39447021484375,
      "logps/rejected": -613.5425415039062,
      "loss": 0.3922,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.286644458770752,
      "rewards/margins": 1.726867914199829,
      "rewards/rejected": -4.01351261138916,
      "step": 9620
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8104492297691845e-06,
      "logits/chosen": -0.9232928156852722,
      "logits/rejected": -0.7564734220504761,
      "logps/chosen": -506.34637451171875,
      "logps/rejected": -597.8238525390625,
      "loss": 0.5703,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.7982330322265625,
      "rewards/margins": 1.1558990478515625,
      "rewards/rejected": -3.954132080078125,
      "step": 9630
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8049626146417562e-06,
      "logits/chosen": -0.25446242094039917,
      "logits/rejected": -0.4590334892272949,
      "logps/chosen": -376.924560546875,
      "logps/rejected": -466.033935546875,
      "loss": 0.6335,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.2184863090515137,
      "rewards/margins": 0.9282231330871582,
      "rewards/rejected": -3.146709442138672,
      "step": 9640
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7994796251697983e-06,
      "logits/chosen": -0.6224043965339661,
      "logits/rejected": -0.4321642518043518,
      "logps/chosen": -441.83721923828125,
      "logps/rejected": -622.97216796875,
      "loss": 0.448,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.497983455657959,
      "rewards/margins": 1.4157010316848755,
      "rewards/rejected": -3.913684129714966,
      "step": 9650
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.794000289955269e-06,
      "logits/chosen": -0.7555705904960632,
      "logits/rejected": -0.8453912734985352,
      "logps/chosen": -528.7978515625,
      "logps/rejected": -593.1597290039062,
      "loss": 0.5633,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.6173758506774902,
      "rewards/margins": 0.9580874443054199,
      "rewards/rejected": -3.5754635334014893,
      "step": 9660
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7885246375810646e-06,
      "logits/chosen": -0.4721905589103699,
      "logits/rejected": -0.4663899540901184,
      "logps/chosen": -453.2267150878906,
      "logps/rejected": -554.3441162109375,
      "loss": 0.4463,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.2895278930664062,
      "rewards/margins": 1.0487935543060303,
      "rewards/rejected": -3.3383216857910156,
      "step": 9670
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7830526966108713e-06,
      "logits/chosen": -0.6492630243301392,
      "logits/rejected": -0.5194262266159058,
      "logps/chosen": -447.9353942871094,
      "logps/rejected": -573.6887817382812,
      "loss": 0.3638,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.6146247386932373,
      "rewards/margins": 1.5436484813690186,
      "rewards/rejected": -4.158273696899414,
      "step": 9680
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7775844955890129e-06,
      "logits/chosen": -0.6144987940788269,
      "logits/rejected": -0.5048569440841675,
      "logps/chosen": -426.19696044921875,
      "logps/rejected": -570.0430908203125,
      "loss": 0.3568,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.1465060710906982,
      "rewards/margins": 1.55422842502594,
      "rewards/rejected": -3.7007346153259277,
      "step": 9690
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7721200630403046e-06,
      "logits/chosen": -0.5308637022972107,
      "logits/rejected": -0.5351754426956177,
      "logps/chosen": -431.6376953125,
      "logps/rejected": -526.9102172851562,
      "loss": 0.5823,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.3587536811828613,
      "rewards/margins": 0.6972657442092896,
      "rewards/rejected": -3.0560193061828613,
      "step": 9700
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -0.7246482968330383,
      "eval_logits/rejected": -0.6136430501937866,
      "eval_logps/chosen": -491.6361389160156,
      "eval_logps/rejected": -579.4501342773438,
      "eval_loss": 0.49039050936698914,
      "eval_rewards/accuracies": 0.6704999804496765,
      "eval_rewards/chosen": -2.596311092376709,
      "eval_rewards/margins": 1.0820716619491577,
      "eval_rewards/rejected": -3.6783831119537354,
      "eval_runtime": 732.3714,
      "eval_samples_per_second": 2.731,
      "eval_steps_per_second": 1.365,
      "step": 9700
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7666594274699037e-06,
      "logits/chosen": -0.615153968334198,
      "logits/rejected": -0.585983395576477,
      "logps/chosen": -514.6607055664062,
      "logps/rejected": -614.2363891601562,
      "loss": 0.4235,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.619704246520996,
      "rewards/margins": 1.3777984380722046,
      "rewards/rejected": -3.997502565383911,
      "step": 9710
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.76120261736316e-06,
      "logits/chosen": -0.5721158981323242,
      "logits/rejected": -0.30697041749954224,
      "logps/chosen": -505.2025451660156,
      "logps/rejected": -604.8211059570312,
      "loss": 0.4677,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.784874677658081,
      "rewards/margins": 1.2676182985305786,
      "rewards/rejected": -4.052493095397949,
      "step": 9720
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.755749661185468e-06,
      "logits/chosen": -0.7573305368423462,
      "logits/rejected": -0.6128786206245422,
      "logps/chosen": -555.46484375,
      "logps/rejected": -618.6527099609375,
      "loss": 0.4732,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.6101527214050293,
      "rewards/margins": 1.2202155590057373,
      "rewards/rejected": -3.8303680419921875,
      "step": 9730
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7503005873821183e-06,
      "logits/chosen": -0.5666888952255249,
      "logits/rejected": -0.7371236085891724,
      "logps/chosen": -404.9433288574219,
      "logps/rejected": -559.25537109375,
      "loss": 0.4454,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.4451496601104736,
      "rewards/margins": 1.2804811000823975,
      "rewards/rejected": -3.725630521774292,
      "step": 9740
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.744855424378148e-06,
      "logits/chosen": -0.35799577832221985,
      "logits/rejected": -0.7274848818778992,
      "logps/chosen": -427.401611328125,
      "logps/rejected": -593.9816284179688,
      "loss": 0.3927,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.387746572494507,
      "rewards/margins": 1.4313452243804932,
      "rewards/rejected": -3.819091796875,
      "step": 9750
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7394142005781973e-06,
      "logits/chosen": -0.6731287837028503,
      "logits/rejected": -0.5532606244087219,
      "logps/chosen": -559.6663818359375,
      "logps/rejected": -683.4664306640625,
      "loss": 0.4492,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.979553699493408,
      "rewards/margins": 1.2022900581359863,
      "rewards/rejected": -4.1818437576293945,
      "step": 9760
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7339769443663528e-06,
      "logits/chosen": -0.5920579433441162,
      "logits/rejected": -0.6127692461013794,
      "logps/chosen": -395.1576232910156,
      "logps/rejected": -486.21258544921875,
      "loss": 0.5335,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.5221543312072754,
      "rewards/margins": 0.9035172462463379,
      "rewards/rejected": -3.4256718158721924,
      "step": 9770
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7285436841060078e-06,
      "logits/chosen": -0.710267186164856,
      "logits/rejected": -0.5984159708023071,
      "logps/chosen": -511.69659423828125,
      "logps/rejected": -608.3006591796875,
      "loss": 0.4099,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.355681896209717,
      "rewards/margins": 1.3955835103988647,
      "rewards/rejected": -3.751265287399292,
      "step": 9780
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7231144481397083e-06,
      "logits/chosen": -0.8003692626953125,
      "logits/rejected": -0.676064670085907,
      "logps/chosen": -445.18719482421875,
      "logps/rejected": -521.8858642578125,
      "loss": 0.5414,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.2478883266448975,
      "rewards/margins": 1.0213735103607178,
      "rewards/rejected": -3.2692618370056152,
      "step": 9790
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7176892647890092e-06,
      "logits/chosen": -0.6346590518951416,
      "logits/rejected": -0.31158536672592163,
      "logps/chosen": -501.07159423828125,
      "logps/rejected": -548.5491943359375,
      "loss": 0.5332,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.6540558338165283,
      "rewards/margins": 0.8635900616645813,
      "rewards/rejected": -3.517646074295044,
      "step": 9800
    },
    {
      "epoch": 0.64,
      "eval_logits/chosen": -0.6229720115661621,
      "eval_logits/rejected": -0.5179995894432068,
      "eval_logps/chosen": -486.57135009765625,
      "eval_logps/rejected": -579.4781494140625,
      "eval_loss": 0.49061888456344604,
      "eval_rewards/accuracies": 0.6704999804496765,
      "eval_rewards/chosen": -2.545663595199585,
      "eval_rewards/margins": 1.132997751235962,
      "eval_rewards/rejected": -3.678661584854126,
      "eval_runtime": 735.253,
      "eval_samples_per_second": 2.72,
      "eval_steps_per_second": 1.36,
      "step": 9800
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7122681623543239e-06,
      "logits/chosen": -0.755142092704773,
      "logits/rejected": -0.8409514427185059,
      "logps/chosen": -506.37664794921875,
      "logps/rejected": -651.1091918945312,
      "loss": 0.3644,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.5938804149627686,
      "rewards/margins": 1.5594637393951416,
      "rewards/rejected": -4.15334415435791,
      "step": 9810
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7068511691147788e-06,
      "logits/chosen": -0.5418431758880615,
      "logits/rejected": -0.46902474761009216,
      "logps/chosen": -425.26593017578125,
      "logps/rejected": -558.49365234375,
      "loss": 0.4844,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.3113489151000977,
      "rewards/margins": 1.2485809326171875,
      "rewards/rejected": -3.559929609298706,
      "step": 9820
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7014383133280636e-06,
      "logits/chosen": -0.7209473848342896,
      "logits/rejected": -0.3792896866798401,
      "logps/chosen": -532.0540771484375,
      "logps/rejected": -575.2877197265625,
      "loss": 0.5814,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.834196090698242,
      "rewards/margins": 0.9662758708000183,
      "rewards/rejected": -3.8004722595214844,
      "step": 9830
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.696029623230286e-06,
      "logits/chosen": -0.6430082321166992,
      "logits/rejected": -0.7621620893478394,
      "logps/chosen": -549.1597900390625,
      "logps/rejected": -683.6041870117188,
      "loss": 0.5399,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.9323744773864746,
      "rewards/margins": 1.156652808189392,
      "rewards/rejected": -4.089027404785156,
      "step": 9840
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.6906251270358229e-06,
      "logits/chosen": -0.8286713361740112,
      "logits/rejected": -0.7480670213699341,
      "logps/chosen": -508.75213623046875,
      "logps/rejected": -546.1533813476562,
      "loss": 0.4749,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.4589786529541016,
      "rewards/margins": 0.8714302182197571,
      "rewards/rejected": -3.3304085731506348,
      "step": 9850
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.685224852937174e-06,
      "logits/chosen": -0.6186366677284241,
      "logits/rejected": -0.2713349461555481,
      "logps/chosen": -400.1719055175781,
      "logps/rejected": -645.1832275390625,
      "loss": 0.3151,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.018071174621582,
      "rewards/margins": 2.0315232276916504,
      "rewards/rejected": -4.049594402313232,
      "step": 9860
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6798288291048136e-06,
      "logits/chosen": -0.5639387369155884,
      "logits/rejected": -0.5140062570571899,
      "logps/chosen": -461.98919677734375,
      "logps/rejected": -567.0389404296875,
      "loss": 0.4877,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.3874754905700684,
      "rewards/margins": 1.3375474214553833,
      "rewards/rejected": -3.725022554397583,
      "step": 9870
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6744370836870466e-06,
      "logits/chosen": -1.273271083831787,
      "logits/rejected": -0.8345822095870972,
      "logps/chosen": -565.0169067382812,
      "logps/rejected": -594.915771484375,
      "loss": 0.6041,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.3032336235046387,
      "rewards/margins": 1.1341663599014282,
      "rewards/rejected": -3.4374001026153564,
      "step": 9880
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6690496448098576e-06,
      "logits/chosen": -0.612956166267395,
      "logits/rejected": -0.46901369094848633,
      "logps/chosen": -441.619873046875,
      "logps/rejected": -527.1732177734375,
      "loss": 0.517,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.125836133956909,
      "rewards/margins": 1.0641244649887085,
      "rewards/rejected": -3.1899607181549072,
      "step": 9890
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6636665405767666e-06,
      "logits/chosen": -0.5108221769332886,
      "logits/rejected": -0.4370543360710144,
      "logps/chosen": -450.4356384277344,
      "logps/rejected": -529.3080444335938,
      "loss": 0.524,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1266462802886963,
      "rewards/margins": 0.977624773979187,
      "rewards/rejected": -3.104271173477173,
      "step": 9900
    },
    {
      "epoch": 0.65,
      "eval_logits/chosen": -0.7448288202285767,
      "eval_logits/rejected": -0.6354550719261169,
      "eval_logps/chosen": -445.274169921875,
      "eval_logps/rejected": -526.677001953125,
      "eval_loss": 0.4900985360145569,
      "eval_rewards/accuracies": 0.675000011920929,
      "eval_rewards/chosen": -2.1326918601989746,
      "eval_rewards/margins": 1.0179589986801147,
      "eval_rewards/rejected": -3.150651216506958,
      "eval_runtime": 734.6282,
      "eval_samples_per_second": 2.722,
      "eval_steps_per_second": 1.361,
      "step": 9900
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6582877990686827e-06,
      "logits/chosen": -0.7203329801559448,
      "logits/rejected": -0.8921548128128052,
      "logps/chosen": -297.42352294921875,
      "logps/rejected": -421.01983642578125,
      "loss": 0.4527,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.7612918615341187,
      "rewards/margins": 1.012649655342102,
      "rewards/rejected": -2.7739417552948,
      "step": 9910
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6529134483437562e-06,
      "logits/chosen": -0.6714494228363037,
      "logits/rejected": -0.8752210736274719,
      "logps/chosen": -390.29119873046875,
      "logps/rejected": -467.7923889160156,
      "loss": 0.3939,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.847169280052185,
      "rewards/margins": 1.1695491075515747,
      "rewards/rejected": -3.0167183876037598,
      "step": 9920
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.647543516437233e-06,
      "logits/chosen": -0.9732862710952759,
      "logits/rejected": -0.9849495887756348,
      "logps/chosen": -392.1590881347656,
      "logps/rejected": -500.3994140625,
      "loss": 0.5757,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.931208610534668,
      "rewards/margins": 0.8755546808242798,
      "rewards/rejected": -2.8067634105682373,
      "step": 9930
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6421780313613088e-06,
      "logits/chosen": -0.7765932083129883,
      "logits/rejected": -0.4009782671928406,
      "logps/chosen": -403.697509765625,
      "logps/rejected": -510.45379638671875,
      "loss": 0.4019,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.9895591735839844,
      "rewards/margins": 1.3994615077972412,
      "rewards/rejected": -3.3890204429626465,
      "step": 9940
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6368170211049816e-06,
      "logits/chosen": -0.47321072220802307,
      "logits/rejected": -0.4680631756782532,
      "logps/chosen": -521.65673828125,
      "logps/rejected": -566.8966674804688,
      "loss": 0.583,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.378565549850464,
      "rewards/margins": 1.0535138845443726,
      "rewards/rejected": -3.432079792022705,
      "step": 9950
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6314605136339074e-06,
      "logits/chosen": -0.8041669726371765,
      "logits/rejected": -0.6485110521316528,
      "logps/chosen": -404.8415832519531,
      "logps/rejected": -462.87188720703125,
      "loss": 0.5907,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.0925488471984863,
      "rewards/margins": 0.7592976689338684,
      "rewards/rejected": -2.851846218109131,
      "step": 9960
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6261085368902526e-06,
      "logits/chosen": -1.1480224132537842,
      "logits/rejected": -0.9943176507949829,
      "logps/chosen": -510.9529724121094,
      "logps/rejected": -532.6697998046875,
      "loss": 0.5558,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.460120677947998,
      "rewards/margins": 0.6534627676010132,
      "rewards/rejected": -3.11358380317688,
      "step": 9970
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6207611187925503e-06,
      "logits/chosen": -0.9005382657051086,
      "logits/rejected": -0.838882327079773,
      "logps/chosen": -399.6454772949219,
      "logps/rejected": -568.9725952148438,
      "loss": 0.3687,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.9081859588623047,
      "rewards/margins": 1.230093002319336,
      "rewards/rejected": -3.1382789611816406,
      "step": 9980
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6154182872355512e-06,
      "logits/chosen": -0.7568344473838806,
      "logits/rejected": -0.9104105234146118,
      "logps/chosen": -360.6561584472656,
      "logps/rejected": -448.56671142578125,
      "loss": 0.6373,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.9683866500854492,
      "rewards/margins": 0.7349096536636353,
      "rewards/rejected": -2.703296422958374,
      "step": 9990
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.610080070090084e-06,
      "logits/chosen": -0.9353153109550476,
      "logits/rejected": -0.8025709390640259,
      "logps/chosen": -384.53924560546875,
      "logps/rejected": -476.81915283203125,
      "loss": 0.4316,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.036179304122925,
      "rewards/margins": 1.0130820274353027,
      "rewards/rejected": -3.0492615699768066,
      "step": 10000
    },
    {
      "epoch": 0.65,
      "eval_logits/chosen": -0.8593093156814575,
      "eval_logits/rejected": -0.743216872215271,
      "eval_logps/chosen": -431.44866943359375,
      "eval_logps/rejected": -515.6310424804688,
      "eval_loss": 0.4896312355995178,
      "eval_rewards/accuracies": 0.6725000143051147,
      "eval_rewards/chosen": -1.9944367408752441,
      "eval_rewards/margins": 1.0457544326782227,
      "eval_rewards/rejected": -3.0401909351348877,
      "eval_runtime": 734.4039,
      "eval_samples_per_second": 2.723,
      "eval_steps_per_second": 1.362,
      "step": 10000
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6047464952029034e-06,
      "logits/chosen": -1.1312496662139893,
      "logits/rejected": -1.077750563621521,
      "logps/chosen": -442.98260498046875,
      "logps/rejected": -593.0875244140625,
      "loss": 0.3385,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8869520425796509,
      "rewards/margins": 1.4187886714935303,
      "rewards/rejected": -3.3057403564453125,
      "step": 10010
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5994175903965486e-06,
      "logits/chosen": -0.7303309440612793,
      "logits/rejected": -0.5155562162399292,
      "logps/chosen": -452.1363830566406,
      "logps/rejected": -560.2066650390625,
      "loss": 0.4909,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.0180599689483643,
      "rewards/margins": 1.0631579160690308,
      "rewards/rejected": -3.0812180042266846,
      "step": 10020
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5940933834691977e-06,
      "logits/chosen": -1.207631230354309,
      "logits/rejected": -0.8119298219680786,
      "logps/chosen": -488.1917419433594,
      "logps/rejected": -507.716064453125,
      "loss": 0.4227,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8292583227157593,
      "rewards/margins": 1.2742161750793457,
      "rewards/rejected": -3.1034743785858154,
      "step": 10030
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.588773902194522e-06,
      "logits/chosen": -0.8028233647346497,
      "logits/rejected": -0.5262702107429504,
      "logps/chosen": -406.1075439453125,
      "logps/rejected": -532.1007080078125,
      "loss": 0.4059,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.0202784538269043,
      "rewards/margins": 1.3023350238800049,
      "rewards/rejected": -3.322613477706909,
      "step": 10040
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.583459174321541e-06,
      "logits/chosen": -0.614817202091217,
      "logits/rejected": -0.6362075805664062,
      "logps/chosen": -402.9064636230469,
      "logps/rejected": -498.8567810058594,
      "loss": 0.4797,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.932891607284546,
      "rewards/margins": 1.2304433584213257,
      "rewards/rejected": -3.163335084915161,
      "step": 10050
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5781492275744797e-06,
      "logits/chosen": -1.2567778825759888,
      "logits/rejected": -1.0947939157485962,
      "logps/chosen": -487.99322509765625,
      "logps/rejected": -631.7654418945312,
      "loss": 0.412,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.9374876022338867,
      "rewards/margins": 1.6999012231826782,
      "rewards/rejected": -3.6373889446258545,
      "step": 10060
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5728440896526215e-06,
      "logits/chosen": -0.7466986775398254,
      "logits/rejected": -0.670479953289032,
      "logps/chosen": -459.1402282714844,
      "logps/rejected": -534.97216796875,
      "loss": 0.4596,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8573827743530273,
      "rewards/margins": 1.2203471660614014,
      "rewards/rejected": -3.077730178833008,
      "step": 10070
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5675437882301633e-06,
      "logits/chosen": -0.9598585367202759,
      "logits/rejected": -0.9147456884384155,
      "logps/chosen": -407.67547607421875,
      "logps/rejected": -435.04742431640625,
      "loss": 0.5896,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.8863128423690796,
      "rewards/margins": 0.6216771006584167,
      "rewards/rejected": -2.5079903602600098,
      "step": 10080
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5622483509560748e-06,
      "logits/chosen": -0.8460068702697754,
      "logits/rejected": -0.9365545511245728,
      "logps/chosen": -343.73724365234375,
      "logps/rejected": -474.4576721191406,
      "loss": 0.5243,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7300806045532227,
      "rewards/margins": 1.0053818225860596,
      "rewards/rejected": -2.7354624271392822,
      "step": 10090
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5569578054539506e-06,
      "logits/chosen": -1.0083024501800537,
      "logits/rejected": -0.6957250833511353,
      "logps/chosen": -470.3836364746094,
      "logps/rejected": -570.5111083984375,
      "loss": 0.3164,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9118655920028687,
      "rewards/margins": 1.8399053812026978,
      "rewards/rejected": -3.7517712116241455,
      "step": 10100
    },
    {
      "epoch": 0.66,
      "eval_logits/chosen": -0.9509793519973755,
      "eval_logits/rejected": -0.8275907635688782,
      "eval_logps/chosen": -418.5705261230469,
      "eval_logps/rejected": -511.33795166015625,
      "eval_loss": 0.4899994134902954,
      "eval_rewards/accuracies": 0.671500027179718,
      "eval_rewards/chosen": -1.865655541419983,
      "eval_rewards/margins": 1.131604790687561,
      "eval_rewards/rejected": -2.997260570526123,
      "eval_runtime": 732.5384,
      "eval_samples_per_second": 2.73,
      "eval_steps_per_second": 1.365,
      "step": 10100
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.551672179321867e-06,
      "logits/chosen": -0.9406857490539551,
      "logits/rejected": -0.9383149147033691,
      "logps/chosen": -386.16998291015625,
      "logps/rejected": -473.0233459472656,
      "loss": 0.4964,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7031307220458984,
      "rewards/margins": 1.080244541168213,
      "rewards/rejected": -2.7833752632141113,
      "step": 10110
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5463915001322398e-06,
      "logits/chosen": -0.9525520205497742,
      "logits/rejected": -0.776652455329895,
      "logps/chosen": -439.74163818359375,
      "logps/rejected": -530.9915161132812,
      "loss": 0.639,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.9408528804779053,
      "rewards/margins": 1.0561749935150146,
      "rewards/rejected": -2.997027635574341,
      "step": 10120
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5411157954316784e-06,
      "logits/chosen": -1.1934840679168701,
      "logits/rejected": -0.8447572588920593,
      "logps/chosen": -347.8656005859375,
      "logps/rejected": -438.13330078125,
      "loss": 0.4629,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4598805904388428,
      "rewards/margins": 1.0395748615264893,
      "rewards/rejected": -2.499455690383911,
      "step": 10130
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.535845092740843e-06,
      "logits/chosen": -1.065664529800415,
      "logits/rejected": -1.037608027458191,
      "logps/chosen": -403.48992919921875,
      "logps/rejected": -487.27313232421875,
      "loss": 0.5483,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.6616871356964111,
      "rewards/margins": 0.7138743996620178,
      "rewards/rejected": -2.375561475753784,
      "step": 10140
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5305794195543005e-06,
      "logits/chosen": -1.1213363409042358,
      "logits/rejected": -1.1167705059051514,
      "logps/chosen": -382.3546447753906,
      "logps/rejected": -487.45135498046875,
      "loss": 0.4169,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.7517554759979248,
      "rewards/margins": 1.2202813625335693,
      "rewards/rejected": -2.9720370769500732,
      "step": 10150
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5253188033403816e-06,
      "logits/chosen": -1.1482348442077637,
      "logits/rejected": -1.0655076503753662,
      "logps/chosen": -344.8389892578125,
      "logps/rejected": -426.26483154296875,
      "loss": 0.5629,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.7410409450531006,
      "rewards/margins": 0.6476749181747437,
      "rewards/rejected": -2.388715982437134,
      "step": 10160
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.520063271541037e-06,
      "logits/chosen": -0.9673888087272644,
      "logits/rejected": -0.8863734006881714,
      "logps/chosen": -423.4639587402344,
      "logps/rejected": -525.8157958984375,
      "loss": 0.5195,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.449592113494873,
      "rewards/margins": 1.1365474462509155,
      "rewards/rejected": -3.5861401557922363,
      "step": 10170
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5148128515716954e-06,
      "logits/chosen": -1.0516979694366455,
      "logits/rejected": -0.6746267080307007,
      "logps/chosen": -480.287109375,
      "logps/rejected": -529.2515869140625,
      "loss": 0.489,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.1375792026519775,
      "rewards/margins": 1.2133866548538208,
      "rewards/rejected": -3.350966215133667,
      "step": 10180
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5095675708211197e-06,
      "logits/chosen": -0.9321045875549316,
      "logits/rejected": -0.8480215072631836,
      "logps/chosen": -450.95098876953125,
      "logps/rejected": -511.4710388183594,
      "loss": 0.6915,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -2.597900867462158,
      "rewards/margins": 0.37176448106765747,
      "rewards/rejected": -2.96966552734375,
      "step": 10190
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.504327456651263e-06,
      "logits/chosen": -0.6907519698143005,
      "logits/rejected": -0.5835791826248169,
      "logps/chosen": -530.9114379882812,
      "logps/rejected": -577.1619873046875,
      "loss": 0.517,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.695084810256958,
      "rewards/margins": 0.7321516275405884,
      "rewards/rejected": -3.427236557006836,
      "step": 10200
    },
    {
      "epoch": 0.67,
      "eval_logits/chosen": -0.8518573641777039,
      "eval_logits/rejected": -0.7372215390205383,
      "eval_logps/chosen": -465.50921630859375,
      "eval_logps/rejected": -543.9869995117188,
      "eval_loss": 0.492562472820282,
      "eval_rewards/accuracies": 0.6679999828338623,
      "eval_rewards/chosen": -2.335042715072632,
      "eval_rewards/margins": 0.9887086153030396,
      "eval_rewards/rejected": -3.323751211166382,
      "eval_runtime": 733.657,
      "eval_samples_per_second": 2.726,
      "eval_steps_per_second": 1.363,
      "step": 10200
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4990925363971284e-06,
      "logits/chosen": -0.9815598726272583,
      "logits/rejected": -0.504173755645752,
      "logps/chosen": -545.0457763671875,
      "logps/rejected": -634.4996337890625,
      "loss": 0.3686,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.5512261390686035,
      "rewards/margins": 1.5794289112091064,
      "rewards/rejected": -4.130654811859131,
      "step": 10210
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4938628373666236e-06,
      "logits/chosen": -0.8545439839363098,
      "logits/rejected": -0.768897533416748,
      "logps/chosen": -392.7917175292969,
      "logps/rejected": -484.1936950683594,
      "loss": 0.5518,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -2.202124834060669,
      "rewards/margins": 0.8952932357788086,
      "rewards/rejected": -3.0974178314208984,
      "step": 10220
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4886383868404203e-06,
      "logits/chosen": -0.6459547281265259,
      "logits/rejected": -0.7341974973678589,
      "logps/chosen": -368.89923095703125,
      "logps/rejected": -473.0926208496094,
      "loss": 0.4036,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.076774835586548,
      "rewards/margins": 1.1422326564788818,
      "rewards/rejected": -3.2190074920654297,
      "step": 10230
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.483419212071813e-06,
      "logits/chosen": -0.5583785176277161,
      "logits/rejected": -0.3464600741863251,
      "logps/chosen": -415.6678771972656,
      "logps/rejected": -493.46759033203125,
      "loss": 0.5239,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2720372676849365,
      "rewards/margins": 0.8498104810714722,
      "rewards/rejected": -3.1218478679656982,
      "step": 10240
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.478205340286573e-06,
      "logits/chosen": -0.8108803033828735,
      "logits/rejected": -0.9261777997016907,
      "logps/chosen": -460.70050048828125,
      "logps/rejected": -527.3087158203125,
      "loss": 0.56,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.570915937423706,
      "rewards/margins": 0.8461298942565918,
      "rewards/rejected": -3.4170455932617188,
      "step": 10250
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4729967986828104e-06,
      "logits/chosen": -0.8202739953994751,
      "logits/rejected": -0.7946422696113586,
      "logps/chosen": -569.1189575195312,
      "logps/rejected": -645.8231201171875,
      "loss": 0.4275,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.4713521003723145,
      "rewards/margins": 1.2467219829559326,
      "rewards/rejected": -3.718074083328247,
      "step": 10260
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4677936144308286e-06,
      "logits/chosen": -0.8981763124465942,
      "logits/rejected": -0.6755679249763489,
      "logps/chosen": -467.2249450683594,
      "logps/rejected": -572.6213989257812,
      "loss": 0.4523,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.4298019409179688,
      "rewards/margins": 1.3345946073532104,
      "rewards/rejected": -3.7643961906433105,
      "step": 10270
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4625958146729864e-06,
      "logits/chosen": -1.0698392391204834,
      "logits/rejected": -0.7566840052604675,
      "logps/chosen": -467.49822998046875,
      "logps/rejected": -540.9188232421875,
      "loss": 0.5114,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.472343921661377,
      "rewards/margins": 0.8936858177185059,
      "rewards/rejected": -3.366029739379883,
      "step": 10280
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4574034265235523e-06,
      "logits/chosen": -0.799394428730011,
      "logits/rejected": -0.6376802325248718,
      "logps/chosen": -481.4248046875,
      "logps/rejected": -503.7127990722656,
      "loss": 0.4684,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.255962371826172,
      "rewards/margins": 1.2005085945129395,
      "rewards/rejected": -3.4564712047576904,
      "step": 10290
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.452216477068568e-06,
      "logits/chosen": -0.6714291572570801,
      "logits/rejected": -0.465345561504364,
      "logps/chosen": -427.8998107910156,
      "logps/rejected": -462.84124755859375,
      "loss": 0.4479,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9785480499267578,
      "rewards/margins": 1.278562307357788,
      "rewards/rejected": -3.257110118865967,
      "step": 10300
    },
    {
      "epoch": 0.67,
      "eval_logits/chosen": -0.8858751058578491,
      "eval_logits/rejected": -0.7681257128715515,
      "eval_logps/chosen": -471.5842590332031,
      "eval_logps/rejected": -554.7045288085938,
      "eval_loss": 0.4910919666290283,
      "eval_rewards/accuracies": 0.6639999747276306,
      "eval_rewards/chosen": -2.3957931995391846,
      "eval_rewards/margins": 1.0351324081420898,
      "eval_rewards/rejected": -3.4309258460998535,
      "eval_runtime": 732.4777,
      "eval_samples_per_second": 2.73,
      "eval_steps_per_second": 1.365,
      "step": 10300
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4470349933657004e-06,
      "logits/chosen": -1.418860673904419,
      "logits/rejected": -0.9940776824951172,
      "logps/chosen": -454.7064514160156,
      "logps/rejected": -535.0302124023438,
      "loss": 0.4524,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.339768171310425,
      "rewards/margins": 1.0430269241333008,
      "rewards/rejected": -3.3827946186065674,
      "step": 10310
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4418590024441096e-06,
      "logits/chosen": -1.18935227394104,
      "logits/rejected": -0.697831928730011,
      "logps/chosen": -467.32855224609375,
      "logps/rejected": -498.80645751953125,
      "loss": 0.5162,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1386830806732178,
      "rewards/margins": 0.9796315431594849,
      "rewards/rejected": -3.118314266204834,
      "step": 10320
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.436688531304297e-06,
      "logits/chosen": -1.0277507305145264,
      "logits/rejected": -0.8398463129997253,
      "logps/chosen": -437.97479248046875,
      "logps/rejected": -539.4046630859375,
      "loss": 0.5487,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.196472644805908,
      "rewards/margins": 1.0691899061203003,
      "rewards/rejected": -3.265662670135498,
      "step": 10330
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.431523606917974e-06,
      "logits/chosen": -0.9352237582206726,
      "logits/rejected": -0.8883055448532104,
      "logps/chosen": -485.72412109375,
      "logps/rejected": -579.2689208984375,
      "loss": 0.6787,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.806248188018799,
      "rewards/margins": 0.8693000078201294,
      "rewards/rejected": -3.675548553466797,
      "step": 10340
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4263642562279162e-06,
      "logits/chosen": -0.6453313827514648,
      "logits/rejected": -0.5077480673789978,
      "logps/chosen": -519.162353515625,
      "logps/rejected": -649.4414672851562,
      "loss": 0.431,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.6967499256134033,
      "rewards/margins": 1.2178905010223389,
      "rewards/rejected": -3.9146409034729004,
      "step": 10350
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4212105061478257e-06,
      "logits/chosen": -0.7289652228355408,
      "logits/rejected": -0.46540945768356323,
      "logps/chosen": -536.9349365234375,
      "logps/rejected": -655.5059814453125,
      "loss": 0.5523,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.070444345474243,
      "rewards/margins": 1.1074758768081665,
      "rewards/rejected": -4.177920341491699,
      "step": 10360
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4160623835621848e-06,
      "logits/chosen": -1.2176058292388916,
      "logits/rejected": -0.8479671478271484,
      "logps/chosen": -469.68682861328125,
      "logps/rejected": -566.1956787109375,
      "loss": 0.4645,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.323755979537964,
      "rewards/margins": 1.0798704624176025,
      "rewards/rejected": -3.4036262035369873,
      "step": 10370
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4109199153261249e-06,
      "logits/chosen": -0.9392701387405396,
      "logits/rejected": -0.7439551949501038,
      "logps/chosen": -543.1309204101562,
      "logps/rejected": -624.5114135742188,
      "loss": 0.4741,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.67136287689209,
      "rewards/margins": 1.1169406175613403,
      "rewards/rejected": -3.7883033752441406,
      "step": 10380
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.405783128265278e-06,
      "logits/chosen": -1.0329110622406006,
      "logits/rejected": -0.9256108999252319,
      "logps/chosen": -479.529541015625,
      "logps/rejected": -570.4544677734375,
      "loss": 0.5458,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.753960371017456,
      "rewards/margins": 0.9419110417366028,
      "rewards/rejected": -3.695871353149414,
      "step": 10390
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4006520491756427e-06,
      "logits/chosen": -0.9245737791061401,
      "logits/rejected": -0.6148309707641602,
      "logps/chosen": -415.374755859375,
      "logps/rejected": -461.10223388671875,
      "loss": 0.4663,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.191504955291748,
      "rewards/margins": 1.1036841869354248,
      "rewards/rejected": -3.295189619064331,
      "step": 10400
    },
    {
      "epoch": 0.68,
      "eval_logits/chosen": -0.9938727021217346,
      "eval_logits/rejected": -0.8684021234512329,
      "eval_logps/chosen": -437.4019470214844,
      "eval_logps/rejected": -522.1436157226562,
      "eval_loss": 0.4915103614330292,
      "eval_rewards/accuracies": 0.6675000190734863,
      "eval_rewards/chosen": -2.0539698600769043,
      "eval_rewards/margins": 1.0513461828231812,
      "eval_rewards/rejected": -3.105316400527954,
      "eval_runtime": 733.2105,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 10400
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.39552670482344e-06,
      "logits/chosen": -0.9635096788406372,
      "logits/rejected": -1.0806286334991455,
      "logps/chosen": -363.7113952636719,
      "logps/rejected": -444.799072265625,
      "loss": 0.5617,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.891117811203003,
      "rewards/margins": 0.8119624853134155,
      "rewards/rejected": -2.703080177307129,
      "step": 10410
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3904071219449776e-06,
      "logits/chosen": -0.9084879755973816,
      "logits/rejected": -0.6438118815422058,
      "logps/chosen": -374.66009521484375,
      "logps/rejected": -393.7281799316406,
      "loss": 0.482,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7475841045379639,
      "rewards/margins": 1.0372825860977173,
      "rewards/rejected": -2.7848668098449707,
      "step": 10420
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3852933272465068e-06,
      "logits/chosen": -1.0169692039489746,
      "logits/rejected": -0.9037554860115051,
      "logps/chosen": -393.3205871582031,
      "logps/rejected": -454.0645446777344,
      "loss": 0.4999,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.5546047687530518,
      "rewards/margins": 0.9596779942512512,
      "rewards/rejected": -2.514282703399658,
      "step": 10430
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3801853474040873e-06,
      "logits/chosen": -0.9115139245986938,
      "logits/rejected": -0.8385306596755981,
      "logps/chosen": -459.4090270996094,
      "logps/rejected": -559.7108154296875,
      "loss": 0.4073,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.2088780403137207,
      "rewards/margins": 1.1676206588745117,
      "rewards/rejected": -3.3764984607696533,
      "step": 10440
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3750832090634417e-06,
      "logits/chosen": -1.0661998987197876,
      "logits/rejected": -0.8567445874214172,
      "logps/chosen": -379.2359313964844,
      "logps/rejected": -462.881103515625,
      "loss": 0.5572,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.9306328296661377,
      "rewards/margins": 0.9829649925231934,
      "rewards/rejected": -2.913597822189331,
      "step": 10450
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3699869388398245e-06,
      "logits/chosen": -0.892473042011261,
      "logits/rejected": -0.7880361676216125,
      "logps/chosen": -453.9684143066406,
      "logps/rejected": -541.6419677734375,
      "loss": 0.5119,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.4094717502593994,
      "rewards/margins": 1.0612695217132568,
      "rewards/rejected": -3.470740795135498,
      "step": 10460
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3648965633178772e-06,
      "logits/chosen": -1.0346626043319702,
      "logits/rejected": -0.9228593707084656,
      "logps/chosen": -419.2098693847656,
      "logps/rejected": -561.0824584960938,
      "loss": 0.4033,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.184777021408081,
      "rewards/margins": 1.304609775543213,
      "rewards/rejected": -3.489386796951294,
      "step": 10470
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3598121090514938e-06,
      "logits/chosen": -0.790840744972229,
      "logits/rejected": -0.8971914052963257,
      "logps/chosen": -388.94677734375,
      "logps/rejected": -464.78369140625,
      "loss": 0.499,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.059389591217041,
      "rewards/margins": 0.9996012449264526,
      "rewards/rejected": -3.058990955352783,
      "step": 10480
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3547336025636753e-06,
      "logits/chosen": -0.8925707936286926,
      "logits/rejected": -0.6376835703849792,
      "logps/chosen": -498.9082946777344,
      "logps/rejected": -568.8594970703125,
      "loss": 0.4828,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2361178398132324,
      "rewards/margins": 1.1148022413253784,
      "rewards/rejected": -3.350919723510742,
      "step": 10490
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3496610703464022e-06,
      "logits/chosen": -1.1685224771499634,
      "logits/rejected": -0.8341997265815735,
      "logps/chosen": -453.0849609375,
      "logps/rejected": -490.42840576171875,
      "loss": 0.5752,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.238703727722168,
      "rewards/margins": 0.8286484479904175,
      "rewards/rejected": -3.067352771759033,
      "step": 10500
    },
    {
      "epoch": 0.69,
      "eval_logits/chosen": -1.0515855550765991,
      "eval_logits/rejected": -0.9209389090538025,
      "eval_logps/chosen": -436.2606506347656,
      "eval_logps/rejected": -528.1688842773438,
      "eval_loss": 0.4914741814136505,
      "eval_rewards/accuracies": 0.6679999828338623,
      "eval_rewards/chosen": -2.04255747795105,
      "eval_rewards/margins": 1.1230124235153198,
      "eval_rewards/rejected": -3.165569543838501,
      "eval_runtime": 733.7382,
      "eval_samples_per_second": 2.726,
      "eval_steps_per_second": 1.363,
      "step": 10500
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3445945388604848e-06,
      "logits/chosen": -1.14806067943573,
      "logits/rejected": -0.693806529045105,
      "logps/chosen": -479.4612731933594,
      "logps/rejected": -563.4382934570312,
      "loss": 0.5061,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.4123778343200684,
      "rewards/margins": 1.2713767290115356,
      "rewards/rejected": -3.6837546825408936,
      "step": 10510
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3395340345354358e-06,
      "logits/chosen": -1.1252193450927734,
      "logits/rejected": -1.1935184001922607,
      "logps/chosen": -446.6983337402344,
      "logps/rejected": -558.856689453125,
      "loss": 0.5534,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2300052642822266,
      "rewards/margins": 0.9491079449653625,
      "rewards/rejected": -3.1791129112243652,
      "step": 10520
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.334479583769322e-06,
      "logits/chosen": -1.2889360189437866,
      "logits/rejected": -1.2451645135879517,
      "logps/chosen": -458.30194091796875,
      "logps/rejected": -509.0130920410156,
      "loss": 0.4664,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.0346317291259766,
      "rewards/margins": 1.0068074464797974,
      "rewards/rejected": -3.0414390563964844,
      "step": 10530
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3294312129286366e-06,
      "logits/chosen": -1.0461938381195068,
      "logits/rejected": -0.9540836215019226,
      "logps/chosen": -434.68817138671875,
      "logps/rejected": -522.4949951171875,
      "loss": 0.4361,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.6795955896377563,
      "rewards/margins": 1.014776587486267,
      "rewards/rejected": -2.6943724155426025,
      "step": 10540
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.324388948348153e-06,
      "logits/chosen": -1.3969929218292236,
      "logits/rejected": -1.0430009365081787,
      "logps/chosen": -457.0474548339844,
      "logps/rejected": -478.5557556152344,
      "loss": 0.4524,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7180945873260498,
      "rewards/margins": 1.028747797012329,
      "rewards/rejected": -2.7468421459198,
      "step": 10550
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.319352816330796e-06,
      "logits/chosen": -1.3804547786712646,
      "logits/rejected": -0.9653580784797668,
      "logps/chosen": -473.27630615234375,
      "logps/rejected": -479.4024353027344,
      "loss": 0.4872,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9007914066314697,
      "rewards/margins": 1.1258151531219482,
      "rewards/rejected": -3.026606559753418,
      "step": 10560
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.314322843147494e-06,
      "logits/chosen": -0.979723334312439,
      "logits/rejected": -1.067060112953186,
      "logps/chosen": -402.91583251953125,
      "logps/rejected": -531.9745483398438,
      "loss": 0.5855,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.325900077819824,
      "rewards/margins": 0.7419096231460571,
      "rewards/rejected": -3.067809581756592,
      "step": 10570
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3092990550370526e-06,
      "logits/chosen": -1.077411413192749,
      "logits/rejected": -0.9932012557983398,
      "logps/chosen": -569.3215942382812,
      "logps/rejected": -607.20263671875,
      "loss": 0.5044,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2356317043304443,
      "rewards/margins": 1.2323225736618042,
      "rewards/rejected": -3.467954158782959,
      "step": 10580
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3042814782060131e-06,
      "logits/chosen": -0.7737795114517212,
      "logits/rejected": -0.7166412472724915,
      "logps/chosen": -329.2445068359375,
      "logps/rejected": -439.805908203125,
      "loss": 0.4284,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4774162769317627,
      "rewards/margins": 1.3447765111923218,
      "rewards/rejected": -2.822192668914795,
      "step": 10590
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.2992701388285112e-06,
      "logits/chosen": -0.8949775695800781,
      "logits/rejected": -0.774708092212677,
      "logps/chosen": -445.0848693847656,
      "logps/rejected": -506.23480224609375,
      "loss": 0.463,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7573579549789429,
      "rewards/margins": 1.0562195777893066,
      "rewards/rejected": -2.81357741355896,
      "step": 10600
    },
    {
      "epoch": 0.69,
      "eval_logits/chosen": -1.006567358970642,
      "eval_logits/rejected": -0.8792454600334167,
      "eval_logps/chosen": -427.3688659667969,
      "eval_logps/rejected": -517.7098999023438,
      "eval_loss": 0.49114155769348145,
      "eval_rewards/accuracies": 0.6654999852180481,
      "eval_rewards/chosen": -1.9536391496658325,
      "eval_rewards/margins": 1.1073405742645264,
      "eval_rewards/rejected": -3.0609798431396484,
      "eval_runtime": 736.8596,
      "eval_samples_per_second": 2.714,
      "eval_steps_per_second": 1.357,
      "step": 10600
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.29426506304615e-06,
      "logits/chosen": -0.9439483880996704,
      "logits/rejected": -0.9378741979598999,
      "logps/chosen": -457.62164306640625,
      "logps/rejected": -493.03228759765625,
      "loss": 0.7532,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.348982095718384,
      "rewards/margins": 0.548886775970459,
      "rewards/rejected": -2.8978688716888428,
      "step": 10610
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.289266276967855e-06,
      "logits/chosen": -1.2154090404510498,
      "logits/rejected": -0.9938980340957642,
      "logps/chosen": -554.221435546875,
      "logps/rejected": -565.4821166992188,
      "loss": 0.4864,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.156395435333252,
      "rewards/margins": 1.0141894817352295,
      "rewards/rejected": -3.1705851554870605,
      "step": 10620
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.284273806669745e-06,
      "logits/chosen": -0.9447933435440063,
      "logits/rejected": -0.9030729532241821,
      "logps/chosen": -509.0013122558594,
      "logps/rejected": -633.0047607421875,
      "loss": 0.4851,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.696763753890991,
      "rewards/margins": 1.0646278858184814,
      "rewards/rejected": -3.7613918781280518,
      "step": 10630
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2792876781949884e-06,
      "logits/chosen": -0.6062985062599182,
      "logits/rejected": -0.521242618560791,
      "logps/chosen": -408.0526428222656,
      "logps/rejected": -471.83612060546875,
      "loss": 0.542,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.0197997093200684,
      "rewards/margins": 0.8310112953186035,
      "rewards/rejected": -2.850811243057251,
      "step": 10640
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.274307917553676e-06,
      "logits/chosen": -0.9177217483520508,
      "logits/rejected": -0.7457525730133057,
      "logps/chosen": -424.8282165527344,
      "logps/rejected": -578.0344848632812,
      "loss": 0.4418,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.291694402694702,
      "rewards/margins": 1.2860668897628784,
      "rewards/rejected": -3.57776141166687,
      "step": 10650
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2693345507226767e-06,
      "logits/chosen": -0.9302674531936646,
      "logits/rejected": -0.7708099484443665,
      "logps/chosen": -467.2933654785156,
      "logps/rejected": -583.9351806640625,
      "loss": 0.4791,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.3730309009552,
      "rewards/margins": 1.1452833414077759,
      "rewards/rejected": -3.5183138847351074,
      "step": 10660
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2643676036455099e-06,
      "logits/chosen": -1.2523701190948486,
      "logits/rejected": -1.0938878059387207,
      "logps/chosen": -498.78857421875,
      "logps/rejected": -543.588134765625,
      "loss": 0.5088,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.109360933303833,
      "rewards/margins": 0.9554061889648438,
      "rewards/rejected": -3.064767360687256,
      "step": 10670
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.259407102232203e-06,
      "logits/chosen": -1.209685206413269,
      "logits/rejected": -0.7783905267715454,
      "logps/chosen": -513.1278076171875,
      "logps/rejected": -560.0278930664062,
      "loss": 0.4174,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.3549351692199707,
      "rewards/margins": 1.1984398365020752,
      "rewards/rejected": -3.553374767303467,
      "step": 10680
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.254453072359163e-06,
      "logits/chosen": -0.7804247140884399,
      "logits/rejected": -0.7799268364906311,
      "logps/chosen": -481.41668701171875,
      "logps/rejected": -561.271484375,
      "loss": 0.4861,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.473966121673584,
      "rewards/margins": 1.0172109603881836,
      "rewards/rejected": -3.4911770820617676,
      "step": 10690
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2495055398690337e-06,
      "logits/chosen": -1.2695982456207275,
      "logits/rejected": -1.037883996963501,
      "logps/chosen": -449.74151611328125,
      "logps/rejected": -520.7718505859375,
      "loss": 0.5865,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.19887113571167,
      "rewards/margins": 0.8031356930732727,
      "rewards/rejected": -3.002007007598877,
      "step": 10700
    },
    {
      "epoch": 0.7,
      "eval_logits/chosen": -0.882703423500061,
      "eval_logits/rejected": -0.762727677822113,
      "eval_logps/chosen": -458.7840576171875,
      "eval_logps/rejected": -548.8290405273438,
      "eval_loss": 0.4880741238594055,
      "eval_rewards/accuracies": 0.6679999828338623,
      "eval_rewards/chosen": -2.2677907943725586,
      "eval_rewards/margins": 1.104380488395691,
      "eval_rewards/rejected": -3.372171401977539,
      "eval_runtime": 733.7904,
      "eval_samples_per_second": 2.726,
      "eval_steps_per_second": 1.363,
      "step": 10700
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2445645305705718e-06,
      "logits/chosen": -1.0357564687728882,
      "logits/rejected": -1.0038548707962036,
      "logps/chosen": -434.9898376464844,
      "logps/rejected": -511.59906005859375,
      "loss": 0.5469,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.2866885662078857,
      "rewards/margins": 0.9725219011306763,
      "rewards/rejected": -3.2592105865478516,
      "step": 10710
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2396300702384995e-06,
      "logits/chosen": -1.0409080982208252,
      "logits/rejected": -0.9370497465133667,
      "logps/chosen": -509.2559509277344,
      "logps/rejected": -534.8280029296875,
      "loss": 0.6689,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.514479160308838,
      "rewards/margins": 0.6323734521865845,
      "rewards/rejected": -3.146852731704712,
      "step": 10720
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.234702184613381e-06,
      "logits/chosen": -0.8630155324935913,
      "logits/rejected": -0.661169171333313,
      "logps/chosen": -444.38623046875,
      "logps/rejected": -533.1038818359375,
      "loss": 0.5089,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.28903865814209,
      "rewards/margins": 0.8905321955680847,
      "rewards/rejected": -3.1795706748962402,
      "step": 10730
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2297808994014793e-06,
      "logits/chosen": -1.1123378276824951,
      "logits/rejected": -0.8954107165336609,
      "logps/chosen": -511.7689514160156,
      "logps/rejected": -571.2996826171875,
      "loss": 0.5019,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.258340835571289,
      "rewards/margins": 0.9442332983016968,
      "rewards/rejected": -3.2025742530822754,
      "step": 10740
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2248662402746314e-06,
      "logits/chosen": -0.8245559930801392,
      "logits/rejected": -0.9639986157417297,
      "logps/chosen": -443.68450927734375,
      "logps/rejected": -538.9236450195312,
      "loss": 0.4695,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.5376784801483154,
      "rewards/margins": 1.0135488510131836,
      "rewards/rejected": -3.551227569580078,
      "step": 10750
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2199582328701045e-06,
      "logits/chosen": -1.0178649425506592,
      "logits/rejected": -0.916810154914856,
      "logps/chosen": -500.6128845214844,
      "logps/rejected": -595.4609375,
      "loss": 0.4223,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.0997936725616455,
      "rewards/margins": 1.3513892889022827,
      "rewards/rejected": -3.4511828422546387,
      "step": 10760
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2150569027904712e-06,
      "logits/chosen": -0.9637805223464966,
      "logits/rejected": -0.9385682940483093,
      "logps/chosen": -465.7509765625,
      "logps/rejected": -568.2579345703125,
      "loss": 0.4797,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1976747512817383,
      "rewards/margins": 1.0880988836288452,
      "rewards/rejected": -3.285773754119873,
      "step": 10770
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.2101622756034688e-06,
      "logits/chosen": -0.9674359560012817,
      "logits/rejected": -0.918204665184021,
      "logps/chosen": -409.08953857421875,
      "logps/rejected": -481.28851318359375,
      "loss": 0.493,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.8633549213409424,
      "rewards/margins": 1.0256426334381104,
      "rewards/rejected": -2.8889975547790527,
      "step": 10780
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.2052743768418715e-06,
      "logits/chosen": -0.9215058088302612,
      "logits/rejected": -0.7507846355438232,
      "logps/chosen": -466.75250244140625,
      "logps/rejected": -534.5064086914062,
      "loss": 0.5046,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.189434289932251,
      "rewards/margins": 1.0176080465316772,
      "rewards/rejected": -3.2070422172546387,
      "step": 10790
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.2003932320033523e-06,
      "logits/chosen": -0.9605894088745117,
      "logits/rejected": -0.9722213745117188,
      "logps/chosen": -460.3763122558594,
      "logps/rejected": -618.0411987304688,
      "loss": 0.3972,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.3387362957000732,
      "rewards/margins": 1.570330023765564,
      "rewards/rejected": -3.9090659618377686,
      "step": 10800
    },
    {
      "epoch": 0.71,
      "eval_logits/chosen": -0.9055193662643433,
      "eval_logits/rejected": -0.7830135226249695,
      "eval_logps/chosen": -468.3777770996094,
      "eval_logps/rejected": -560.4705810546875,
      "eval_loss": 0.4904264211654663,
      "eval_rewards/accuracies": 0.6690000295639038,
      "eval_rewards/chosen": -2.3637280464172363,
      "eval_rewards/margins": 1.1248583793640137,
      "eval_rewards/rejected": -3.488586187362671,
      "eval_runtime": 733.0719,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 10800
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1955188665503553e-06,
      "logits/chosen": -0.8003166913986206,
      "logits/rejected": -0.7389184236526489,
      "logps/chosen": -425.4422302246094,
      "logps/rejected": -509.4952697753906,
      "loss": 0.4912,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.2031853199005127,
      "rewards/margins": 1.048431396484375,
      "rewards/rejected": -3.2516167163848877,
      "step": 10810
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1906513059099566e-06,
      "logits/chosen": -1.0434842109680176,
      "logits/rejected": -0.7776933908462524,
      "logps/chosen": -487.75811767578125,
      "logps/rejected": -602.5443725585938,
      "loss": 0.4286,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.562689781188965,
      "rewards/margins": 1.2352688312530518,
      "rewards/rejected": -3.7979588508605957,
      "step": 10820
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.185790575473738e-06,
      "logits/chosen": -0.9918256998062134,
      "logits/rejected": -0.6145702004432678,
      "logps/chosen": -465.2567443847656,
      "logps/rejected": -522.7682495117188,
      "loss": 0.5531,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.361233711242676,
      "rewards/margins": 1.0478578805923462,
      "rewards/rejected": -3.4090912342071533,
      "step": 10830
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1809367005976516e-06,
      "logits/chosen": -0.9616907238960266,
      "logits/rejected": -0.8075451850891113,
      "logps/chosen": -500.2339782714844,
      "logps/rejected": -505.9252014160156,
      "loss": 0.5588,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.177391767501831,
      "rewards/margins": 0.7673858404159546,
      "rewards/rejected": -2.944777488708496,
      "step": 10840
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1760897066018842e-06,
      "logits/chosen": -0.9130662679672241,
      "logits/rejected": -0.8023947477340698,
      "logps/chosen": -424.35919189453125,
      "logps/rejected": -545.6055297851562,
      "loss": 0.4358,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.01318097114563,
      "rewards/margins": 1.2724324464797974,
      "rewards/rejected": -3.2856132984161377,
      "step": 10850
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1712496187707327e-06,
      "logits/chosen": -0.8917104005813599,
      "logits/rejected": -1.089603066444397,
      "logps/chosen": -464.90960693359375,
      "logps/rejected": -616.8685913085938,
      "loss": 0.3678,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.246053695678711,
      "rewards/margins": 1.6602468490600586,
      "rewards/rejected": -3.9063007831573486,
      "step": 10860
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1664164623524646e-06,
      "logits/chosen": -0.9778538942337036,
      "logits/rejected": -0.7901339530944824,
      "logps/chosen": -417.0609436035156,
      "logps/rejected": -517.4918823242188,
      "loss": 0.4386,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9868589639663696,
      "rewards/margins": 1.2868831157684326,
      "rewards/rejected": -3.2737419605255127,
      "step": 10870
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1615902625591926e-06,
      "logits/chosen": -1.0647779703140259,
      "logits/rejected": -0.7082611322402954,
      "logps/chosen": -475.00164794921875,
      "logps/rejected": -568.5804443359375,
      "loss": 0.4011,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.497804641723633,
      "rewards/margins": 1.0299241542816162,
      "rewards/rejected": -3.527729034423828,
      "step": 10880
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.156771044566738e-06,
      "logits/chosen": -0.9863220453262329,
      "logits/rejected": -0.8786922693252563,
      "logps/chosen": -502.59735107421875,
      "logps/rejected": -572.4608154296875,
      "loss": 0.4652,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.3798270225524902,
      "rewards/margins": 1.1308090686798096,
      "rewards/rejected": -3.5106358528137207,
      "step": 10890
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1519588335145037e-06,
      "logits/chosen": -1.0075815916061401,
      "logits/rejected": -1.190239667892456,
      "logps/chosen": -456.916259765625,
      "logps/rejected": -555.6007080078125,
      "loss": 0.5572,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.4933745861053467,
      "rewards/margins": 0.7998501062393188,
      "rewards/rejected": -3.293224811553955,
      "step": 10900
    },
    {
      "epoch": 0.71,
      "eval_logits/chosen": -0.8925363421440125,
      "eval_logits/rejected": -0.7710062265396118,
      "eval_logps/chosen": -468.0953674316406,
      "eval_logps/rejected": -562.2437744140625,
      "eval_loss": 0.489229679107666,
      "eval_rewards/accuracies": 0.6679999828338623,
      "eval_rewards/chosen": -2.360903739929199,
      "eval_rewards/margins": 1.1454142332077026,
      "eval_rewards/rejected": -3.5063183307647705,
      "eval_runtime": 733.7383,
      "eval_samples_per_second": 2.726,
      "eval_steps_per_second": 1.363,
      "step": 10900
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1471536545053382e-06,
      "logits/chosen": -0.9088330268859863,
      "logits/rejected": -0.9032711982727051,
      "logps/chosen": -407.5400390625,
      "logps/rejected": -520.8419189453125,
      "loss": 0.5155,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.985050916671753,
      "rewards/margins": 1.057497262954712,
      "rewards/rejected": -3.042548656463623,
      "step": 10910
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1423555326054112e-06,
      "logits/chosen": -0.8974703550338745,
      "logits/rejected": -0.632853627204895,
      "logps/chosen": -522.8529052734375,
      "logps/rejected": -632.1292724609375,
      "loss": 0.3658,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.400184154510498,
      "rewards/margins": 1.8271236419677734,
      "rewards/rejected": -4.2273077964782715,
      "step": 10920
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1375644928440743e-06,
      "logits/chosen": -1.0343987941741943,
      "logits/rejected": -0.755792498588562,
      "logps/chosen": -445.74774169921875,
      "logps/rejected": -501.7667541503906,
      "loss": 0.4438,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.0906448364257812,
      "rewards/margins": 1.2352287769317627,
      "rewards/rejected": -3.325873613357544,
      "step": 10930
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1327805602137396e-06,
      "logits/chosen": -1.0418864488601685,
      "logits/rejected": -0.8314645886421204,
      "logps/chosen": -501.97607421875,
      "logps/rejected": -583.506103515625,
      "loss": 0.4581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.4323198795318604,
      "rewards/margins": 1.3450803756713867,
      "rewards/rejected": -3.777400255203247,
      "step": 10940
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1280037596697426e-06,
      "logits/chosen": -0.8969013094902039,
      "logits/rejected": -0.643790602684021,
      "logps/chosen": -492.24432373046875,
      "logps/rejected": -691.2158203125,
      "loss": 0.3989,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.713618516921997,
      "rewards/margins": 1.6204471588134766,
      "rewards/rejected": -4.3340654373168945,
      "step": 10950
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.123234116130216e-06,
      "logits/chosen": -0.7682039737701416,
      "logits/rejected": -0.6643818020820618,
      "logps/chosen": -445.10321044921875,
      "logps/rejected": -577.23291015625,
      "loss": 0.5684,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.592148780822754,
      "rewards/margins": 1.260753870010376,
      "rewards/rejected": -3.85290265083313,
      "step": 10960
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1184716544759553e-06,
      "logits/chosen": -0.5035034418106079,
      "logits/rejected": -0.47273555397987366,
      "logps/chosen": -387.4788818359375,
      "logps/rejected": -469.57037353515625,
      "loss": 0.5691,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.221147060394287,
      "rewards/margins": 0.6967689990997314,
      "rewards/rejected": -2.9179160594940186,
      "step": 10970
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1137163995502948e-06,
      "logits/chosen": -1.3651089668273926,
      "logits/rejected": -1.1751941442489624,
      "logps/chosen": -429.91729736328125,
      "logps/rejected": -511.0647888183594,
      "loss": 0.482,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.0838229656219482,
      "rewards/margins": 1.1384937763214111,
      "rewards/rejected": -3.2223167419433594,
      "step": 10980
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1089683761589717e-06,
      "logits/chosen": -0.779492974281311,
      "logits/rejected": -0.7341394424438477,
      "logps/chosen": -440.05682373046875,
      "logps/rejected": -590.7908935546875,
      "loss": 0.4098,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.998504877090454,
      "rewards/margins": 1.6964826583862305,
      "rewards/rejected": -3.6949875354766846,
      "step": 10990
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1042276090700044e-06,
      "logits/chosen": -0.8709524869918823,
      "logits/rejected": -0.927435576915741,
      "logps/chosen": -479.05706787109375,
      "logps/rejected": -579.4146728515625,
      "loss": 0.6689,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.687549352645874,
      "rewards/margins": 0.7018827199935913,
      "rewards/rejected": -3.3894317150115967,
      "step": 11000
    },
    {
      "epoch": 0.72,
      "eval_logits/chosen": -0.957138180732727,
      "eval_logits/rejected": -0.8341361880302429,
      "eval_logps/chosen": -453.06585693359375,
      "eval_logps/rejected": -539.7461547851562,
      "eval_loss": 0.488401859998703,
      "eval_rewards/accuracies": 0.6685000061988831,
      "eval_rewards/chosen": -2.210608720779419,
      "eval_rewards/margins": 1.0707330703735352,
      "eval_rewards/rejected": -3.281341552734375,
      "eval_runtime": 733.4399,
      "eval_samples_per_second": 2.727,
      "eval_steps_per_second": 1.363,
      "step": 11000
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0994941230135536e-06,
      "logits/chosen": -1.0178345441818237,
      "logits/rejected": -0.920695424079895,
      "logps/chosen": -431.5406188964844,
      "logps/rejected": -554.9949951171875,
      "loss": 0.3399,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9869062900543213,
      "rewards/margins": 1.5951406955718994,
      "rewards/rejected": -3.5820469856262207,
      "step": 11010
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.094767942681804e-06,
      "logits/chosen": -1.4254045486450195,
      "logits/rejected": -1.0217857360839844,
      "logps/chosen": -526.99169921875,
      "logps/rejected": -603.30224609375,
      "loss": 0.4858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.791226863861084,
      "rewards/margins": 1.1551108360290527,
      "rewards/rejected": -3.9463374614715576,
      "step": 11020
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0900490927288248e-06,
      "logits/chosen": -0.7323638200759888,
      "logits/rejected": -0.7848427295684814,
      "logps/chosen": -496.42279052734375,
      "logps/rejected": -534.5650024414062,
      "loss": 0.5168,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.3595452308654785,
      "rewards/margins": 0.8761852383613586,
      "rewards/rejected": -3.2357306480407715,
      "step": 11030
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0853375977704511e-06,
      "logits/chosen": -1.0628750324249268,
      "logits/rejected": -0.9022421836853027,
      "logps/chosen": -463.55535888671875,
      "logps/rejected": -495.8919372558594,
      "loss": 0.4713,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.2313079833984375,
      "rewards/margins": 0.9634731411933899,
      "rewards/rejected": -3.1947810649871826,
      "step": 11040
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0806334823841466e-06,
      "logits/chosen": -1.0171680450439453,
      "logits/rejected": -1.157289981842041,
      "logps/chosen": -465.65380859375,
      "logps/rejected": -539.416748046875,
      "loss": 0.7313,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.26037859916687,
      "rewards/margins": 0.4435979723930359,
      "rewards/rejected": -2.70397686958313,
      "step": 11050
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0759367711088825e-06,
      "logits/chosen": -0.8065532445907593,
      "logits/rejected": -1.0416743755340576,
      "logps/chosen": -377.97216796875,
      "logps/rejected": -480.1463317871094,
      "loss": 0.5569,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.941396713256836,
      "rewards/margins": 0.6869693398475647,
      "rewards/rejected": -2.628365993499756,
      "step": 11060
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0712474884450056e-06,
      "logits/chosen": -1.0178492069244385,
      "logits/rejected": -0.8606100082397461,
      "logps/chosen": -385.5984191894531,
      "logps/rejected": -468.76947021484375,
      "loss": 0.4454,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8276513814926147,
      "rewards/margins": 1.1195788383483887,
      "rewards/rejected": -2.9472298622131348,
      "step": 11070
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.066565658854112e-06,
      "logits/chosen": -0.8222743272781372,
      "logits/rejected": -0.9117767214775085,
      "logps/chosen": -333.2332458496094,
      "logps/rejected": -429.37579345703125,
      "loss": 0.5184,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.054013252258301,
      "rewards/margins": 0.9937500953674316,
      "rewards/rejected": -3.0477633476257324,
      "step": 11080
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0618913067589165e-06,
      "logits/chosen": -1.0981779098510742,
      "logits/rejected": -0.7434073686599731,
      "logps/chosen": -420.64434814453125,
      "logps/rejected": -498.775146484375,
      "loss": 0.4303,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.0482959747314453,
      "rewards/margins": 1.1559075117111206,
      "rewards/rejected": -3.2042033672332764,
      "step": 11090
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0572244565431313e-06,
      "logits/chosen": -0.938554584980011,
      "logits/rejected": -0.9342789649963379,
      "logps/chosen": -354.7508544921875,
      "logps/rejected": -450.92523193359375,
      "loss": 0.4435,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0933403968811035,
      "rewards/margins": 0.9101314544677734,
      "rewards/rejected": -3.003472089767456,
      "step": 11100
    },
    {
      "epoch": 0.73,
      "eval_logits/chosen": -1.0134085416793823,
      "eval_logits/rejected": -0.8863934874534607,
      "eval_logps/chosen": -443.8869323730469,
      "eval_logps/rejected": -533.0964965820312,
      "eval_loss": 0.4877125322818756,
      "eval_rewards/accuracies": 0.6704999804496765,
      "eval_rewards/chosen": -2.11881947517395,
      "eval_rewards/margins": 1.0960263013839722,
      "eval_rewards/rejected": -3.214846134185791,
      "eval_runtime": 733.4843,
      "eval_samples_per_second": 2.727,
      "eval_steps_per_second": 1.363,
      "step": 11100
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0525651325513317e-06,
      "logits/chosen": -1.0130351781845093,
      "logits/rejected": -1.0235871076583862,
      "logps/chosen": -542.2921142578125,
      "logps/rejected": -603.1007080078125,
      "loss": 0.6379,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.1102242469787598,
      "rewards/margins": 0.8196399807929993,
      "rewards/rejected": -2.9298644065856934,
      "step": 11110
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0479133590888351e-06,
      "logits/chosen": -1.0382142066955566,
      "logits/rejected": -0.9209200739860535,
      "logps/chosen": -472.59283447265625,
      "logps/rejected": -546.9326171875,
      "loss": 0.4894,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1713783740997314,
      "rewards/margins": 1.0131199359893799,
      "rewards/rejected": -3.1844985485076904,
      "step": 11120
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0432691604215695e-06,
      "logits/chosen": -1.076169729232788,
      "logits/rejected": -0.8656023144721985,
      "logps/chosen": -429.9717712402344,
      "logps/rejected": -481.2691345214844,
      "loss": 0.5108,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9492483139038086,
      "rewards/margins": 0.8301043510437012,
      "rewards/rejected": -2.7793526649475098,
      "step": 11130
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0386325607759515e-06,
      "logits/chosen": -1.0083991289138794,
      "logits/rejected": -0.793779730796814,
      "logps/chosen": -356.7492370605469,
      "logps/rejected": -465.8247985839844,
      "loss": 0.4141,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6535911560058594,
      "rewards/margins": 1.291459560394287,
      "rewards/rejected": -2.9450504779815674,
      "step": 11140
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0340035843387544e-06,
      "logits/chosen": -0.9116052389144897,
      "logits/rejected": -0.7750101089477539,
      "logps/chosen": -363.89166259765625,
      "logps/rejected": -446.844482421875,
      "loss": 0.524,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8415607213974,
      "rewards/margins": 1.1053569316864014,
      "rewards/rejected": -2.94691801071167,
      "step": 11150
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0293822552569887e-06,
      "logits/chosen": -1.178986668586731,
      "logits/rejected": -1.0037364959716797,
      "logps/chosen": -461.3528747558594,
      "logps/rejected": -518.0894775390625,
      "loss": 0.573,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.0946905612945557,
      "rewards/margins": 1.1256706714630127,
      "rewards/rejected": -3.220360517501831,
      "step": 11160
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0247685976377688e-06,
      "logits/chosen": -1.084922432899475,
      "logits/rejected": -0.8008192181587219,
      "logps/chosen": -388.384765625,
      "logps/rejected": -457.7376403808594,
      "loss": 0.5008,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.066997528076172,
      "rewards/margins": 1.0818424224853516,
      "rewards/rejected": -3.1488397121429443,
      "step": 11170
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0201626355481939e-06,
      "logits/chosen": -1.206965684890747,
      "logits/rejected": -1.0226151943206787,
      "logps/chosen": -425.4285583496094,
      "logps/rejected": -488.95074462890625,
      "loss": 0.3881,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0930800437927246,
      "rewards/margins": 1.1708210706710815,
      "rewards/rejected": -3.2639012336730957,
      "step": 11180
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0155643930152192e-06,
      "logits/chosen": -1.2319376468658447,
      "logits/rejected": -1.0966084003448486,
      "logps/chosen": -492.85614013671875,
      "logps/rejected": -529.0089111328125,
      "loss": 0.5333,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.2024879455566406,
      "rewards/margins": 0.9079867601394653,
      "rewards/rejected": -3.1104745864868164,
      "step": 11190
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0109738940255286e-06,
      "logits/chosen": -0.9945646524429321,
      "logits/rejected": -0.8503836393356323,
      "logps/chosen": -413.6936950683594,
      "logps/rejected": -578.9022216796875,
      "loss": 0.5282,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9671112298965454,
      "rewards/margins": 2.0248970985412598,
      "rewards/rejected": -3.9920082092285156,
      "step": 11200
    },
    {
      "epoch": 0.73,
      "eval_logits/chosen": -1.0309162139892578,
      "eval_logits/rejected": -0.902747392654419,
      "eval_logps/chosen": -437.6730651855469,
      "eval_logps/rejected": -526.853515625,
      "eval_loss": 0.487095445394516,
      "eval_rewards/accuracies": 0.671500027179718,
      "eval_rewards/chosen": -2.0566811561584473,
      "eval_rewards/margins": 1.0957354307174683,
      "eval_rewards/rejected": -3.152416229248047,
      "eval_runtime": 736.8631,
      "eval_samples_per_second": 2.714,
      "eval_steps_per_second": 1.357,
      "step": 11200
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0063911625254155e-06,
      "logits/chosen": -1.1311531066894531,
      "logits/rejected": -1.1501240730285645,
      "logps/chosen": -399.01885986328125,
      "logps/rejected": -517.2718505859375,
      "loss": 0.4506,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6309621334075928,
      "rewards/margins": 1.1762351989746094,
      "rewards/rejected": -2.8071975708007812,
      "step": 11210
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0018162224206502e-06,
      "logits/chosen": -0.9788483381271362,
      "logits/rejected": -0.9975950121879578,
      "logps/chosen": -379.59552001953125,
      "logps/rejected": -536.1707763671875,
      "loss": 0.4089,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0512068271636963,
      "rewards/margins": 1.5286515951156616,
      "rewards/rejected": -3.5798587799072266,
      "step": 11220
    },
    {
      "epoch": 0.73,
      "learning_rate": 9.97249097576363e-07,
      "logits/chosen": -1.4967749118804932,
      "logits/rejected": -1.068502426147461,
      "logps/chosen": -436.02069091796875,
      "logps/rejected": -541.9471435546875,
      "loss": 0.4054,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.0296413898468018,
      "rewards/margins": 1.494182825088501,
      "rewards/rejected": -3.5238239765167236,
      "step": 11230
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.92689811816913e-07,
      "logits/chosen": -1.0827951431274414,
      "logits/rejected": -0.7837275862693787,
      "logps/chosen": -419.0982360839844,
      "logps/rejected": -461.9170837402344,
      "loss": 0.5667,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.0657706260681152,
      "rewards/margins": 0.9022183418273926,
      "rewards/rejected": -2.967989206314087,
      "step": 11240
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.881383889257691e-07,
      "logits/chosen": -1.017508864402771,
      "logits/rejected": -1.1153932809829712,
      "logps/chosen": -382.99005126953125,
      "logps/rejected": -534.8026123046875,
      "loss": 0.5895,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1369316577911377,
      "rewards/margins": 0.8773406744003296,
      "rewards/rejected": -3.014272451400757,
      "step": 11250
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.835948526453817e-07,
      "logits/chosen": -0.7544322609901428,
      "logits/rejected": -1.1118907928466797,
      "logps/chosen": -390.35015869140625,
      "logps/rejected": -519.1009521484375,
      "loss": 0.5099,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.099207878112793,
      "rewards/margins": 0.8507896661758423,
      "rewards/rejected": -2.9499974250793457,
      "step": 11260
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.790592266770633e-07,
      "logits/chosen": -1.2587848901748657,
      "logits/rejected": -1.1021195650100708,
      "logps/chosen": -467.99273681640625,
      "logps/rejected": -533.5831298828125,
      "loss": 0.4609,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9864799976348877,
      "rewards/margins": 0.938291072845459,
      "rewards/rejected": -2.9247708320617676,
      "step": 11270
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.745315346808584e-07,
      "logits/chosen": -0.8920543789863586,
      "logits/rejected": -0.9167378544807434,
      "logps/chosen": -388.6293640136719,
      "logps/rejected": -438.75848388671875,
      "loss": 0.6022,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.695539116859436,
      "rewards/margins": 0.7251222729682922,
      "rewards/rejected": -2.420661449432373,
      "step": 11280
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.70011800275428e-07,
      "logits/chosen": -0.9127265214920044,
      "logits/rejected": -0.8961038589477539,
      "logps/chosen": -454.28143310546875,
      "logps/rejected": -575.9407348632812,
      "loss": 0.4861,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.16756534576416,
      "rewards/margins": 1.1021320819854736,
      "rewards/rejected": -3.269697666168213,
      "step": 11290
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.655000470379206e-07,
      "logits/chosen": -1.0204126834869385,
      "logits/rejected": -0.8335134387016296,
      "logps/chosen": -408.29669189453125,
      "logps/rejected": -516.9795532226562,
      "loss": 0.4652,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.979491949081421,
      "rewards/margins": 1.193654179573059,
      "rewards/rejected": -3.1731460094451904,
      "step": 11300
    },
    {
      "epoch": 0.74,
      "eval_logits/chosen": -1.0541530847549438,
      "eval_logits/rejected": -0.925934374332428,
      "eval_logps/chosen": -418.2159118652344,
      "eval_logps/rejected": -505.072998046875,
      "eval_loss": 0.487036794424057,
      "eval_rewards/accuracies": 0.6690000295639038,
      "eval_rewards/chosen": -1.8621094226837158,
      "eval_rewards/margins": 1.0725016593933105,
      "eval_rewards/rejected": -2.9346108436584473,
      "eval_runtime": 735.7356,
      "eval_samples_per_second": 2.718,
      "eval_steps_per_second": 1.359,
      "step": 11300
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.609962985038517e-07,
      "logits/chosen": -1.0937474966049194,
      "logits/rejected": -0.9918316006660461,
      "logps/chosen": -385.2691650390625,
      "logps/rejected": -524.9221801757812,
      "loss": 0.2925,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.6990811824798584,
      "rewards/margins": 1.6477792263031006,
      "rewards/rejected": -3.346860408782959,
      "step": 11310
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.565005781669786e-07,
      "logits/chosen": -1.3409435749053955,
      "logits/rejected": -0.9802799224853516,
      "logps/chosen": -449.4964294433594,
      "logps/rejected": -519.0867919921875,
      "loss": 0.4207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8890583515167236,
      "rewards/margins": 1.2290316820144653,
      "rewards/rejected": -3.1180901527404785,
      "step": 11320
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.520129094791822e-07,
      "logits/chosen": -0.9601613283157349,
      "logits/rejected": -0.7478787302970886,
      "logps/chosen": -385.11859130859375,
      "logps/rejected": -475.3666076660156,
      "loss": 0.5723,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.0562288761138916,
      "rewards/margins": 1.0045124292373657,
      "rewards/rejected": -3.0607409477233887,
      "step": 11330
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.475333158503389e-07,
      "logits/chosen": -0.997717559337616,
      "logits/rejected": -0.8612111806869507,
      "logps/chosen": -375.82659912109375,
      "logps/rejected": -420.026611328125,
      "loss": 0.4935,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5800435543060303,
      "rewards/margins": 0.8935853838920593,
      "rewards/rejected": -2.4736289978027344,
      "step": 11340
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.430618206482053e-07,
      "logits/chosen": -0.9635284543037415,
      "logits/rejected": -0.8489049077033997,
      "logps/chosen": -289.9509582519531,
      "logps/rejected": -372.4026794433594,
      "loss": 0.5246,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4952805042266846,
      "rewards/margins": 0.8185623288154602,
      "rewards/rejected": -2.3138427734375,
      "step": 11350
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.385984471982892e-07,
      "logits/chosen": -0.907095730304718,
      "logits/rejected": -0.7227785587310791,
      "logps/chosen": -398.25634765625,
      "logps/rejected": -517.0142822265625,
      "loss": 0.4243,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8343048095703125,
      "rewards/margins": 1.6949255466461182,
      "rewards/rejected": -3.5292305946350098,
      "step": 11360
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.341432187837343e-07,
      "logits/chosen": -1.2137072086334229,
      "logits/rejected": -1.0345278978347778,
      "logps/chosen": -358.2519836425781,
      "logps/rejected": -509.11590576171875,
      "loss": 0.4798,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.679481863975525,
      "rewards/margins": 1.35995614528656,
      "rewards/rejected": -3.039438247680664,
      "step": 11370
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.29696158645193e-07,
      "logits/chosen": -1.015655755996704,
      "logits/rejected": -1.135019063949585,
      "logps/chosen": -419.72821044921875,
      "logps/rejected": -604.85595703125,
      "loss": 0.3531,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.98625910282135,
      "rewards/margins": 1.5613784790039062,
      "rewards/rejected": -3.547637462615967,
      "step": 11380
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.252572899807111e-07,
      "logits/chosen": -1.0255063772201538,
      "logits/rejected": -0.8526045083999634,
      "logps/chosen": -461.0198669433594,
      "logps/rejected": -576.572998046875,
      "loss": 0.38,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9042030572891235,
      "rewards/margins": 1.4174002408981323,
      "rewards/rejected": -3.3216030597686768,
      "step": 11390
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.208266359456003e-07,
      "logits/chosen": -1.1188398599624634,
      "logits/rejected": -1.008388876914978,
      "logps/chosen": -364.1383056640625,
      "logps/rejected": -474.17626953125,
      "loss": 0.4956,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5833252668380737,
      "rewards/margins": 1.0046722888946533,
      "rewards/rejected": -2.5879976749420166,
      "step": 11400
    },
    {
      "epoch": 0.75,
      "eval_logits/chosen": -0.9939939379692078,
      "eval_logits/rejected": -0.8659761548042297,
      "eval_logps/chosen": -433.4949951171875,
      "eval_logps/rejected": -530.9140014648438,
      "eval_loss": 0.48666563630104065,
      "eval_rewards/accuracies": 0.6725000143051147,
      "eval_rewards/chosen": -2.0149004459381104,
      "eval_rewards/margins": 1.178120732307434,
      "eval_rewards/rejected": -3.193021774291992,
      "eval_runtime": 732.7351,
      "eval_samples_per_second": 2.729,
      "eval_steps_per_second": 1.365,
      "step": 11400
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.164042196523229e-07,
      "logits/chosen": -1.3033071756362915,
      "logits/rejected": -0.9964720010757446,
      "logps/chosen": -393.0906982421875,
      "logps/rejected": -522.20703125,
      "loss": 0.3878,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.0368967056274414,
      "rewards/margins": 1.4263392686843872,
      "rewards/rejected": -3.4632363319396973,
      "step": 11410
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.119900641703696e-07,
      "logits/chosen": -1.2340941429138184,
      "logits/rejected": -0.9759272336959839,
      "logps/chosen": -451.19805908203125,
      "logps/rejected": -494.56048583984375,
      "loss": 0.5809,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.377516984939575,
      "rewards/margins": 0.896973729133606,
      "rewards/rejected": -3.2744908332824707,
      "step": 11420
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.075841925261364e-07,
      "logits/chosen": -1.3645763397216797,
      "logits/rejected": -1.2368052005767822,
      "logps/chosen": -419.3846740722656,
      "logps/rejected": -540.921142578125,
      "loss": 0.4186,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.814655065536499,
      "rewards/margins": 1.3596525192260742,
      "rewards/rejected": -3.1743075847625732,
      "step": 11430
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.031866277028093e-07,
      "logits/chosen": -0.9969218969345093,
      "logits/rejected": -0.8433194160461426,
      "logps/chosen": -391.59197998046875,
      "logps/rejected": -549.1114501953125,
      "loss": 0.4337,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.005171775817871,
      "rewards/margins": 1.3070980310440063,
      "rewards/rejected": -3.312269687652588,
      "step": 11440
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.987973926402391e-07,
      "logits/chosen": -0.8501356244087219,
      "logits/rejected": -0.9896366000175476,
      "logps/chosen": -406.74560546875,
      "logps/rejected": -493.015625,
      "loss": 0.6122,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.9440596103668213,
      "rewards/margins": 0.8685744404792786,
      "rewards/rejected": -2.812633991241455,
      "step": 11450
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.944165102348273e-07,
      "logits/chosen": -1.1681933403015137,
      "logits/rejected": -1.0174529552459717,
      "logps/chosen": -300.1144104003906,
      "logps/rejected": -429.978271484375,
      "loss": 0.4504,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.431673288345337,
      "rewards/margins": 1.2260645627975464,
      "rewards/rejected": -2.657738208770752,
      "step": 11460
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.900440033394018e-07,
      "logits/chosen": -0.9283140301704407,
      "logits/rejected": -0.9719370603561401,
      "logps/chosen": -374.1133117675781,
      "logps/rejected": -455.5450134277344,
      "loss": 0.5721,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9159705638885498,
      "rewards/margins": 0.9071747064590454,
      "rewards/rejected": -2.8231453895568848,
      "step": 11470
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.856798947631009e-07,
      "logits/chosen": -1.1020092964172363,
      "logits/rejected": -1.047979712486267,
      "logps/chosen": -384.72314453125,
      "logps/rejected": -563.4594116210938,
      "loss": 0.422,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9384742975234985,
      "rewards/margins": 1.6178430318832397,
      "rewards/rejected": -3.5563175678253174,
      "step": 11480
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.813242072712519e-07,
      "logits/chosen": -0.578697681427002,
      "logits/rejected": -0.46960777044296265,
      "logps/chosen": -389.91998291015625,
      "logps/rejected": -459.9017639160156,
      "loss": 0.642,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.2805769443511963,
      "rewards/margins": 0.6884570121765137,
      "rewards/rejected": -2.969034194946289,
      "step": 11490
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.769769635852557e-07,
      "logits/chosen": -0.9524284601211548,
      "logits/rejected": -0.9921194911003113,
      "logps/chosen": -412.4534606933594,
      "logps/rejected": -475.46697998046875,
      "loss": 0.5636,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.9714571237564087,
      "rewards/margins": 0.9977054595947266,
      "rewards/rejected": -2.969162702560425,
      "step": 11500
    },
    {
      "epoch": 0.75,
      "eval_logits/chosen": -0.988334596157074,
      "eval_logits/rejected": -0.8627732396125793,
      "eval_logps/chosen": -444.1772766113281,
      "eval_logps/rejected": -533.0626220703125,
      "eval_loss": 0.4872725009918213,
      "eval_rewards/accuracies": 0.6704999804496765,
      "eval_rewards/chosen": -2.1217236518859863,
      "eval_rewards/margins": 1.0927841663360596,
      "eval_rewards/rejected": -3.214507818222046,
      "eval_runtime": 732.6713,
      "eval_samples_per_second": 2.73,
      "eval_steps_per_second": 1.365,
      "step": 11500
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.726381863824635e-07,
      "logits/chosen": -1.3175032138824463,
      "logits/rejected": -1.1112231016159058,
      "logps/chosen": -530.478759765625,
      "logps/rejected": -558.8153076171875,
      "loss": 0.5539,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.4540176391601562,
      "rewards/margins": 0.957426905632019,
      "rewards/rejected": -3.4114444255828857,
      "step": 11510
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.683078982960638e-07,
      "logits/chosen": -0.8189307451248169,
      "logits/rejected": -0.6247791647911072,
      "logps/chosen": -447.772705078125,
      "logps/rejected": -547.8245849609375,
      "loss": 0.3892,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.23087739944458,
      "rewards/margins": 1.5014287233352661,
      "rewards/rejected": -3.7323060035705566,
      "step": 11520
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.639861219149584e-07,
      "logits/chosen": -1.0422413349151611,
      "logits/rejected": -0.782747209072113,
      "logps/chosen": -491.7372131347656,
      "logps/rejected": -552.7254638671875,
      "loss": 0.5402,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.2775521278381348,
      "rewards/margins": 1.0391275882720947,
      "rewards/rejected": -3.3166794776916504,
      "step": 11530
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.596728797836532e-07,
      "logits/chosen": -0.9940250515937805,
      "logits/rejected": -0.9373093843460083,
      "logps/chosen": -416.41790771484375,
      "logps/rejected": -564.4520263671875,
      "loss": 0.4311,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.011092185974121,
      "rewards/margins": 1.184410810470581,
      "rewards/rejected": -3.1955032348632812,
      "step": 11540
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.553681944021294e-07,
      "logits/chosen": -1.1565699577331543,
      "logits/rejected": -1.3328361511230469,
      "logps/chosen": -427.739990234375,
      "logps/rejected": -529.1275634765625,
      "loss": 0.5004,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8865597248077393,
      "rewards/margins": 1.2073477506637573,
      "rewards/rejected": -3.0939078330993652,
      "step": 11550
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.510720882257365e-07,
      "logits/chosen": -0.7032291293144226,
      "logits/rejected": -0.7788906097412109,
      "logps/chosen": -366.40814208984375,
      "logps/rejected": -514.55224609375,
      "loss": 0.437,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.0510754585266113,
      "rewards/margins": 1.0691219568252563,
      "rewards/rejected": -3.120197057723999,
      "step": 11560
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.467845836650667e-07,
      "logits/chosen": -0.613045334815979,
      "logits/rejected": -0.6705749034881592,
      "logps/chosen": -421.2106018066406,
      "logps/rejected": -554.2785034179688,
      "loss": 0.4347,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.1294405460357666,
      "rewards/margins": 1.2651278972625732,
      "rewards/rejected": -3.3945682048797607,
      "step": 11570
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.425057030858461e-07,
      "logits/chosen": -0.7619770765304565,
      "logits/rejected": -0.7964997291564941,
      "logps/chosen": -354.86334228515625,
      "logps/rejected": -490.247802734375,
      "loss": 0.4928,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.9880750179290771,
      "rewards/margins": 1.0636155605316162,
      "rewards/rejected": -3.051690101623535,
      "step": 11580
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.382354688088098e-07,
      "logits/chosen": -0.8965905904769897,
      "logits/rejected": -0.9813228845596313,
      "logps/chosen": -345.85467529296875,
      "logps/rejected": -456.6095275878906,
      "loss": 0.4907,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8769928216934204,
      "rewards/margins": 0.9533191919326782,
      "rewards/rejected": -2.8303122520446777,
      "step": 11590
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.33973903109594e-07,
      "logits/chosen": -0.9670153856277466,
      "logits/rejected": -1.0012229681015015,
      "logps/chosen": -440.3134765625,
      "logps/rejected": -537.9239501953125,
      "loss": 0.4554,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.23716402053833,
      "rewards/margins": 1.291721224784851,
      "rewards/rejected": -3.5288853645324707,
      "step": 11600
    },
    {
      "epoch": 0.76,
      "eval_logits/chosen": -0.965965211391449,
      "eval_logits/rejected": -0.8415982723236084,
      "eval_logps/chosen": -461.8896179199219,
      "eval_logps/rejected": -550.7822265625,
      "eval_loss": 0.4887566864490509,
      "eval_rewards/accuracies": 0.6704999804496765,
      "eval_rewards/chosen": -2.298846483230591,
      "eval_rewards/margins": 1.0928568840026855,
      "eval_rewards/rejected": -3.3917033672332764,
      "eval_runtime": 733.1262,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 11600
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.297210282186102e-07,
      "logits/chosen": -1.029415249824524,
      "logits/rejected": -0.9706588983535767,
      "logps/chosen": -532.1493530273438,
      "logps/rejected": -625.8573608398438,
      "loss": 0.6493,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.021317958831787,
      "rewards/margins": 0.723414421081543,
      "rewards/rejected": -3.744732618331909,
      "step": 11610
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.254768663209397e-07,
      "logits/chosen": -0.818048357963562,
      "logits/rejected": -0.8541353940963745,
      "logps/chosen": -516.0035400390625,
      "logps/rejected": -537.320068359375,
      "loss": 0.4895,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.374237298965454,
      "rewards/margins": 0.8378809690475464,
      "rewards/rejected": -3.212118148803711,
      "step": 11620
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.212414395562079e-07,
      "logits/chosen": -0.9309345483779907,
      "logits/rejected": -0.8357607126235962,
      "logps/chosen": -479.80535888671875,
      "logps/rejected": -599.5394287109375,
      "loss": 0.5408,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.4959120750427246,
      "rewards/margins": 0.9581314921379089,
      "rewards/rejected": -3.45404314994812,
      "step": 11630
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.170147700184775e-07,
      "logits/chosen": -0.9578307867050171,
      "logits/rejected": -0.887701153755188,
      "logps/chosen": -495.539794921875,
      "logps/rejected": -609.1138916015625,
      "loss": 0.4221,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.3908846378326416,
      "rewards/margins": 1.225068211555481,
      "rewards/rejected": -3.615952968597412,
      "step": 11640
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.127968797561242e-07,
      "logits/chosen": -1.1409074068069458,
      "logits/rejected": -0.9202806353569031,
      "logps/chosen": -499.4234313964844,
      "logps/rejected": -620.0963134765625,
      "loss": 0.4226,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.7545275688171387,
      "rewards/margins": 1.430957555770874,
      "rewards/rejected": -4.185485363006592,
      "step": 11650
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.085877907717338e-07,
      "logits/chosen": -0.9937346577644348,
      "logits/rejected": -1.119011402130127,
      "logps/chosen": -436.72772216796875,
      "logps/rejected": -540.747314453125,
      "loss": 0.5345,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1581618785858154,
      "rewards/margins": 1.1569883823394775,
      "rewards/rejected": -3.315150022506714,
      "step": 11660
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.043875250219732e-07,
      "logits/chosen": -0.9474736452102661,
      "logits/rejected": -0.6932603120803833,
      "logps/chosen": -491.7503356933594,
      "logps/rejected": -531.57568359375,
      "loss": 0.5792,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.6234564781188965,
      "rewards/margins": 0.6089419722557068,
      "rewards/rejected": -3.232398271560669,
      "step": 11670
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.001961044174881e-07,
      "logits/chosen": -1.0843124389648438,
      "logits/rejected": -0.6968761086463928,
      "logps/chosen": -484.9103088378906,
      "logps/rejected": -499.39453125,
      "loss": 0.5963,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.5644583702087402,
      "rewards/margins": 0.7167236804962158,
      "rewards/rejected": -3.281182050704956,
      "step": 11680
    },
    {
      "epoch": 0.76,
      "learning_rate": 7.960135508227795e-07,
      "logits/chosen": -1.0498230457305908,
      "logits/rejected": -0.9083242416381836,
      "logps/chosen": -526.6635131835938,
      "logps/rejected": -561.640625,
      "loss": 0.4749,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.32794451713562,
      "rewards/margins": 0.9929312467575073,
      "rewards/rejected": -3.320875883102417,
      "step": 11690
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.91839886056098e-07,
      "logits/chosen": -1.1805912256240845,
      "logits/rejected": -0.7656027674674988,
      "logps/chosen": -533.066650390625,
      "logps/rejected": -621.0750732421875,
      "loss": 0.4871,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.4930436611175537,
      "rewards/margins": 1.185792326927185,
      "rewards/rejected": -3.6788361072540283,
      "step": 11700
    },
    {
      "epoch": 0.77,
      "eval_logits/chosen": -0.9552643299102783,
      "eval_logits/rejected": -0.8321646451950073,
      "eval_logps/chosen": -463.67156982421875,
      "eval_logps/rejected": -548.34375,
      "eval_loss": 0.49002426862716675,
      "eval_rewards/accuracies": 0.6654999852180481,
      "eval_rewards/chosen": -2.3166658878326416,
      "eval_rewards/margins": 1.050653100013733,
      "eval_rewards/rejected": -3.367318868637085,
      "eval_runtime": 734.3534,
      "eval_samples_per_second": 2.723,
      "eval_steps_per_second": 1.362,
      "step": 11700
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.876751318893217e-07,
      "logits/chosen": -0.935076892375946,
      "logits/rejected": -0.7096657752990723,
      "logps/chosen": -459.2938537597656,
      "logps/rejected": -537.5563354492188,
      "loss": 0.4755,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2142813205718994,
      "rewards/margins": 0.9985069036483765,
      "rewards/rejected": -3.212787628173828,
      "step": 11710
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.8351931004785e-07,
      "logits/chosen": -0.6283329129219055,
      "logits/rejected": -0.7397192120552063,
      "logps/chosen": -445.36517333984375,
      "logps/rejected": -535.31591796875,
      "loss": 0.469,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.3673691749572754,
      "rewards/margins": 1.1483402252197266,
      "rewards/rejected": -3.515709638595581,
      "step": 11720
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.793724422104834e-07,
      "logits/chosen": -1.0047307014465332,
      "logits/rejected": -0.9424341917037964,
      "logps/chosen": -439.0755310058594,
      "logps/rejected": -636.6625366210938,
      "loss": 0.4635,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.3025119304656982,
      "rewards/margins": 1.2810983657836914,
      "rewards/rejected": -3.5836100578308105,
      "step": 11730
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.752345500093184e-07,
      "logits/chosen": -1.1682775020599365,
      "logits/rejected": -1.0444492101669312,
      "logps/chosen": -455.862548828125,
      "logps/rejected": -490.4752502441406,
      "loss": 0.6232,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.4570751190185547,
      "rewards/margins": 0.6437710523605347,
      "rewards/rejected": -3.100846290588379,
      "step": 11740
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.711056550296253e-07,
      "logits/chosen": -1.2327778339385986,
      "logits/rejected": -0.9802483320236206,
      "logps/chosen": -422.8338928222656,
      "logps/rejected": -508.6863708496094,
      "loss": 0.4144,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7604007720947266,
      "rewards/margins": 1.228775143623352,
      "rewards/rejected": -2.989175796508789,
      "step": 11750
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.669857788097445e-07,
      "logits/chosen": -0.6518399119377136,
      "logits/rejected": -0.4853084981441498,
      "logps/chosen": -377.8327331542969,
      "logps/rejected": -519.7390747070312,
      "loss": 0.4804,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1626675128936768,
      "rewards/margins": 1.1528141498565674,
      "rewards/rejected": -3.3154819011688232,
      "step": 11760
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.628749428409676e-07,
      "logits/chosen": -1.0058742761611938,
      "logits/rejected": -0.8142207860946655,
      "logps/chosen": -459.57562255859375,
      "logps/rejected": -495.9466247558594,
      "loss": 0.6283,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.2268784046173096,
      "rewards/margins": 0.9512258768081665,
      "rewards/rejected": -3.1781039237976074,
      "step": 11770
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.587731685674288e-07,
      "logits/chosen": -1.2172415256500244,
      "logits/rejected": -1.179513692855835,
      "logps/chosen": -500.82598876953125,
      "logps/rejected": -601.9574584960938,
      "loss": 0.5644,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.272376775741577,
      "rewards/margins": 0.9329449534416199,
      "rewards/rejected": -3.205321788787842,
      "step": 11780
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.546804773859931e-07,
      "logits/chosen": -1.156795859336853,
      "logits/rejected": -1.0326814651489258,
      "logps/chosen": -423.5052795410156,
      "logps/rejected": -535.3846435546875,
      "loss": 0.4482,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9646520614624023,
      "rewards/margins": 1.3181536197662354,
      "rewards/rejected": -3.2828056812286377,
      "step": 11790
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.505968906461409e-07,
      "logits/chosen": -0.8699377179145813,
      "logits/rejected": -1.1452893018722534,
      "logps/chosen": -439.53387451171875,
      "logps/rejected": -521.7918701171875,
      "loss": 0.527,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9859380722045898,
      "rewards/margins": 1.0640430450439453,
      "rewards/rejected": -3.049980878829956,
      "step": 11800
    },
    {
      "epoch": 0.77,
      "eval_logits/chosen": -1.0908076763153076,
      "eval_logits/rejected": -0.9602593779563904,
      "eval_logps/chosen": -422.1820373535156,
      "eval_logps/rejected": -508.17919921875,
      "eval_loss": 0.48903682827949524,
      "eval_rewards/accuracies": 0.6690000295639038,
      "eval_rewards/chosen": -1.901770830154419,
      "eval_rewards/margins": 1.0639021396636963,
      "eval_rewards/rejected": -2.9656729698181152,
      "eval_runtime": 731.3982,
      "eval_samples_per_second": 2.734,
      "eval_steps_per_second": 1.367,
      "step": 11800
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.465224296498627e-07,
      "logits/chosen": -1.1006393432617188,
      "logits/rejected": -0.8764568567276001,
      "logps/chosen": -412.1211853027344,
      "logps/rejected": -477.8194885253906,
      "loss": 0.4437,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7916208505630493,
      "rewards/margins": 1.0682731866836548,
      "rewards/rejected": -2.859893798828125,
      "step": 11810
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.424571156515412e-07,
      "logits/chosen": -1.1791812181472778,
      "logits/rejected": -1.0099537372589111,
      "logps/chosen": -355.47320556640625,
      "logps/rejected": -466.53497314453125,
      "loss": 0.5231,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7776803970336914,
      "rewards/margins": 0.8838518261909485,
      "rewards/rejected": -2.661531925201416,
      "step": 11820
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.38400969857847e-07,
      "logits/chosen": -0.9811850786209106,
      "logits/rejected": -0.9850279092788696,
      "logps/chosen": -406.4681701660156,
      "logps/rejected": -532.5477905273438,
      "loss": 0.3842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.233257532119751,
      "rewards/margins": 1.229274868965149,
      "rewards/rejected": -3.4625325202941895,
      "step": 11830
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.343540134276225e-07,
      "logits/chosen": -1.1172263622283936,
      "logits/rejected": -1.1144663095474243,
      "logps/chosen": -332.30999755859375,
      "logps/rejected": -423.35772705078125,
      "loss": 0.578,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6159842014312744,
      "rewards/margins": 0.8915143013000488,
      "rewards/rejected": -2.5074985027313232,
      "step": 11840
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.303162674717762e-07,
      "logits/chosen": -0.7593609094619751,
      "logits/rejected": -0.6226508617401123,
      "logps/chosen": -429.37359619140625,
      "logps/rejected": -464.8836975097656,
      "loss": 0.53,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.1914584636688232,
      "rewards/margins": 0.913454532623291,
      "rewards/rejected": -3.1049129962921143,
      "step": 11850
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.26287753053167e-07,
      "logits/chosen": -1.109512209892273,
      "logits/rejected": -0.941886305809021,
      "logps/chosen": -501.8273010253906,
      "logps/rejected": -605.0291137695312,
      "loss": 0.5333,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.368511438369751,
      "rewards/margins": 1.0447001457214355,
      "rewards/rejected": -3.4132113456726074,
      "step": 11860
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.222684911865013e-07,
      "logits/chosen": -1.0845589637756348,
      "logits/rejected": -1.0413366556167603,
      "logps/chosen": -369.25897216796875,
      "logps/rejected": -516.8199462890625,
      "loss": 0.4041,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6209027767181396,
      "rewards/margins": 1.258754849433899,
      "rewards/rejected": -2.879657745361328,
      "step": 11870
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.182585028382166e-07,
      "logits/chosen": -1.2062350511550903,
      "logits/rejected": -1.0037099123001099,
      "logps/chosen": -459.37249755859375,
      "logps/rejected": -541.9036865234375,
      "loss": 0.3956,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8322092294692993,
      "rewards/margins": 1.1498112678527832,
      "rewards/rejected": -2.982020616531372,
      "step": 11880
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.142578089263769e-07,
      "logits/chosen": -1.2526991367340088,
      "logits/rejected": -1.2502632141113281,
      "logps/chosen": -507.63128662109375,
      "logps/rejected": -593.0853881835938,
      "loss": 0.4494,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8920265436172485,
      "rewards/margins": 1.4425415992736816,
      "rewards/rejected": -3.3345680236816406,
      "step": 11890
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.102664303205611e-07,
      "logits/chosen": -0.9701188802719116,
      "logits/rejected": -1.1144049167633057,
      "logps/chosen": -441.10723876953125,
      "logps/rejected": -530.4204711914062,
      "loss": 0.569,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.20544695854187,
      "rewards/margins": 1.0718988180160522,
      "rewards/rejected": -3.277346134185791,
      "step": 11900
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -1.0340651273727417,
      "eval_logits/rejected": -0.9051576852798462,
      "eval_logps/chosen": -439.36798095703125,
      "eval_logps/rejected": -531.2297973632812,
      "eval_loss": 0.48876819014549255,
      "eval_rewards/accuracies": 0.6669999957084656,
      "eval_rewards/chosen": -2.0736303329467773,
      "eval_rewards/margins": 1.1225486993789673,
      "eval_rewards/rejected": -3.196179151535034,
      "eval_runtime": 732.9071,
      "eval_samples_per_second": 2.729,
      "eval_steps_per_second": 1.364,
      "step": 11900
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.062843878417566e-07,
      "logits/chosen": -1.3267271518707275,
      "logits/rejected": -1.1309109926223755,
      "logps/chosen": -415.145751953125,
      "logps/rejected": -485.17266845703125,
      "loss": 0.5037,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.9171311855316162,
      "rewards/margins": 1.0307111740112305,
      "rewards/rejected": -2.947842836380005,
      "step": 11910
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.023117022622458e-07,
      "logits/chosen": -1.1064445972442627,
      "logits/rejected": -0.793212890625,
      "logps/chosen": -508.7882385253906,
      "logps/rejected": -580.7551879882812,
      "loss": 0.5068,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.7027292251586914,
      "rewards/margins": 0.9523531794548035,
      "rewards/rejected": -3.6550827026367188,
      "step": 11920
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.983483943055042e-07,
      "logits/chosen": -1.0946762561798096,
      "logits/rejected": -0.9343063235282898,
      "logps/chosen": -491.595947265625,
      "logps/rejected": -513.9833984375,
      "loss": 0.5994,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.1112940311431885,
      "rewards/margins": 0.791604220867157,
      "rewards/rejected": -2.9028983116149902,
      "step": 11930
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.943944846460859e-07,
      "logits/chosen": -0.8383496403694153,
      "logits/rejected": -0.8814895749092102,
      "logps/chosen": -407.0166931152344,
      "logps/rejected": -453.33123779296875,
      "loss": 0.5948,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.8826236724853516,
      "rewards/margins": 0.9305664896965027,
      "rewards/rejected": -2.81318998336792,
      "step": 11940
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.904499939095225e-07,
      "logits/chosen": -1.0897403955459595,
      "logits/rejected": -1.018691062927246,
      "logps/chosen": -407.6357421875,
      "logps/rejected": -522.668212890625,
      "loss": 0.4682,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8528856039047241,
      "rewards/margins": 1.3036080598831177,
      "rewards/rejected": -3.156493663787842,
      "step": 11950
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.865149426722079e-07,
      "logits/chosen": -0.8987745046615601,
      "logits/rejected": -0.8762083053588867,
      "logps/chosen": -502.9815368652344,
      "logps/rejected": -570.9379272460938,
      "loss": 0.5286,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.3179638385772705,
      "rewards/margins": 0.9865046739578247,
      "rewards/rejected": -3.3044686317443848,
      "step": 11960
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.825893514612985e-07,
      "logits/chosen": -0.8229151964187622,
      "logits/rejected": -0.7518579363822937,
      "logps/chosen": -412.1634216308594,
      "logps/rejected": -548.7064208984375,
      "loss": 0.4418,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7559655904769897,
      "rewards/margins": 1.3872650861740112,
      "rewards/rejected": -3.14323091506958,
      "step": 11970
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.786732407546001e-07,
      "logits/chosen": -0.8534737825393677,
      "logits/rejected": -0.7331782579421997,
      "logps/chosen": -396.0401611328125,
      "logps/rejected": -448.16082763671875,
      "loss": 0.5383,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.951589584350586,
      "rewards/margins": 0.9183629751205444,
      "rewards/rejected": -2.869952440261841,
      "step": 11980
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.747666309804654e-07,
      "logits/chosen": -1.3171064853668213,
      "logits/rejected": -1.0033562183380127,
      "logps/chosen": -522.2742919921875,
      "logps/rejected": -562.9515991210938,
      "loss": 0.3579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.345970392227173,
      "rewards/margins": 1.202128291130066,
      "rewards/rejected": -3.5480988025665283,
      "step": 11990
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.708695425176831e-07,
      "logits/chosen": -0.9916369318962097,
      "logits/rejected": -0.7585171461105347,
      "logps/chosen": -394.0176696777344,
      "logps/rejected": -539.1189575195312,
      "loss": 0.4233,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.249788999557495,
      "rewards/margins": 1.2434598207473755,
      "rewards/rejected": -3.493248701095581,
      "step": 12000
    },
    {
      "epoch": 0.79,
      "eval_logits/chosen": -1.0466409921646118,
      "eval_logits/rejected": -0.9172680377960205,
      "eval_logps/chosen": -441.6598815917969,
      "eval_logps/rejected": -530.766357421875,
      "eval_loss": 0.48879528045654297,
      "eval_rewards/accuracies": 0.6704999804496765,
      "eval_rewards/chosen": -2.0965492725372314,
      "eval_rewards/margins": 1.0949954986572266,
      "eval_rewards/rejected": -3.191545009613037,
      "eval_runtime": 730.9157,
      "eval_samples_per_second": 2.736,
      "eval_steps_per_second": 1.368,
      "step": 12000
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.669819956953768e-07,
      "logits/chosen": -0.8059449195861816,
      "logits/rejected": -0.8757872581481934,
      "logps/chosen": -352.72723388671875,
      "logps/rejected": -473.0796813964844,
      "loss": 0.4025,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.823960542678833,
      "rewards/margins": 1.118124008178711,
      "rewards/rejected": -2.942084550857544,
      "step": 12010
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.631040107928957e-07,
      "logits/chosen": -1.3854398727416992,
      "logits/rejected": -0.8418337106704712,
      "logps/chosen": -502.0740661621094,
      "logps/rejected": -510.20123291015625,
      "loss": 0.4322,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.338897228240967,
      "rewards/margins": 1.0442569255828857,
      "rewards/rejected": -3.3831546306610107,
      "step": 12020
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.592356080397072e-07,
      "logits/chosen": -1.0946362018585205,
      "logits/rejected": -0.8409433364868164,
      "logps/chosen": -424.310546875,
      "logps/rejected": -497.91796875,
      "loss": 0.4945,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9602136611938477,
      "rewards/margins": 1.300708293914795,
      "rewards/rejected": -3.2609219551086426,
      "step": 12030
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.553768076152963e-07,
      "logits/chosen": -0.8674459457397461,
      "logits/rejected": -1.0205134153366089,
      "logps/chosen": -333.92681884765625,
      "logps/rejected": -509.59979248046875,
      "loss": 0.4385,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7582921981811523,
      "rewards/margins": 1.4856898784637451,
      "rewards/rejected": -3.2439818382263184,
      "step": 12040
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.51527629649055e-07,
      "logits/chosen": -1.3031247854232788,
      "logits/rejected": -1.123964786529541,
      "logps/chosen": -489.2030334472656,
      "logps/rejected": -507.7479553222656,
      "loss": 0.6789,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.356199264526367,
      "rewards/margins": 0.47692030668258667,
      "rewards/rejected": -2.8331196308135986,
      "step": 12050
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.476880942201824e-07,
      "logits/chosen": -1.491958498954773,
      "logits/rejected": -1.0982462167739868,
      "logps/chosen": -372.901611328125,
      "logps/rejected": -446.5074768066406,
      "loss": 0.4808,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3156394958496094,
      "rewards/margins": 1.3223001956939697,
      "rewards/rejected": -2.637939929962158,
      "step": 12060
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.438582213575748e-07,
      "logits/chosen": -1.1286519765853882,
      "logits/rejected": -1.1970762014389038,
      "logps/chosen": -405.6150207519531,
      "logps/rejected": -532.4569091796875,
      "loss": 0.4548,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7569844722747803,
      "rewards/margins": 1.1569640636444092,
      "rewards/rejected": -2.9139487743377686,
      "step": 12070
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.400380310397267e-07,
      "logits/chosen": -1.1258952617645264,
      "logits/rejected": -0.8831712007522583,
      "logps/chosen": -435.85418701171875,
      "logps/rejected": -529.7272338867188,
      "loss": 0.7571,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.037438154220581,
      "rewards/margins": 0.6246122121810913,
      "rewards/rejected": -2.662050724029541,
      "step": 12080
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.362275431946202e-07,
      "logits/chosen": -0.9884549975395203,
      "logits/rejected": -1.0082800388336182,
      "logps/chosen": -429.74102783203125,
      "logps/rejected": -526.2401123046875,
      "loss": 0.5738,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9338738918304443,
      "rewards/margins": 0.904509425163269,
      "rewards/rejected": -2.838383436203003,
      "step": 12090
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.324267776996285e-07,
      "logits/chosen": -1.4050889015197754,
      "logits/rejected": -0.9676424264907837,
      "logps/chosen": -574.62158203125,
      "logps/rejected": -605.3466796875,
      "loss": 0.3903,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.00042462348938,
      "rewards/margins": 1.5270559787750244,
      "rewards/rejected": -3.527480363845825,
      "step": 12100
    },
    {
      "epoch": 0.79,
      "eval_logits/chosen": -1.1908217668533325,
      "eval_logits/rejected": -1.056341528892517,
      "eval_logps/chosen": -398.17730712890625,
      "eval_logps/rejected": -481.9284973144531,
      "eval_loss": 0.49027806520462036,
      "eval_rewards/accuracies": 0.6664999723434448,
      "eval_rewards/chosen": -1.6617236137390137,
      "eval_rewards/margins": 1.0414423942565918,
      "eval_rewards/rejected": -2.7031657695770264,
      "eval_runtime": 732.5323,
      "eval_samples_per_second": 2.73,
      "eval_steps_per_second": 1.365,
      "step": 12100
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.286357543814045e-07,
      "logits/chosen": -1.0828051567077637,
      "logits/rejected": -1.1512118577957153,
      "logps/chosen": -353.3865051269531,
      "logps/rejected": -540.902099609375,
      "loss": 0.4649,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5985664129257202,
      "rewards/margins": 1.1950440406799316,
      "rewards/rejected": -2.7936103343963623,
      "step": 12110
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.248544930157838e-07,
      "logits/chosen": -1.2649319171905518,
      "logits/rejected": -1.0828841924667358,
      "logps/chosen": -356.8739929199219,
      "logps/rejected": -485.9205627441406,
      "loss": 0.4563,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.7278845310211182,
      "rewards/margins": 1.4682705402374268,
      "rewards/rejected": -3.196155071258545,
      "step": 12120
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.21083013327678e-07,
      "logits/chosen": -1.1251111030578613,
      "logits/rejected": -1.065676212310791,
      "logps/chosen": -455.14129638671875,
      "logps/rejected": -474.03887939453125,
      "loss": 0.5947,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.5090054273605347,
      "rewards/margins": 0.748233437538147,
      "rewards/rejected": -2.2572388648986816,
      "step": 12130
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.17321334990973e-07,
      "logits/chosen": -1.2648893594741821,
      "logits/rejected": -1.0302037000656128,
      "logps/chosen": -353.55010986328125,
      "logps/rejected": -410.7579650878906,
      "loss": 0.5201,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5274560451507568,
      "rewards/margins": 0.9244512319564819,
      "rewards/rejected": -2.4519073963165283,
      "step": 12140
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.135694776284243e-07,
      "logits/chosen": -1.4515762329101562,
      "logits/rejected": -1.0432875156402588,
      "logps/chosen": -432.5904235839844,
      "logps/rejected": -496.59246826171875,
      "loss": 0.4491,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.617993950843811,
      "rewards/margins": 1.2877423763275146,
      "rewards/rejected": -2.9057364463806152,
      "step": 12150
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.098274608115595e-07,
      "logits/chosen": -1.2649531364440918,
      "logits/rejected": -0.9151164889335632,
      "logps/chosen": -360.52301025390625,
      "logps/rejected": -400.2120056152344,
      "loss": 0.5682,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.5487239360809326,
      "rewards/margins": 0.6832138895988464,
      "rewards/rejected": -2.231937885284424,
      "step": 12160
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.060953040605697e-07,
      "logits/chosen": -1.3290317058563232,
      "logits/rejected": -0.9182616472244263,
      "logps/chosen": -492.67193603515625,
      "logps/rejected": -576.432373046875,
      "loss": 0.4024,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5354218482971191,
      "rewards/margins": 1.536554217338562,
      "rewards/rejected": -3.0719761848449707,
      "step": 12170
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.023730268442144e-07,
      "logits/chosen": -1.117250680923462,
      "logits/rejected": -0.9653674364089966,
      "logps/chosen": -363.29437255859375,
      "logps/rejected": -477.4454040527344,
      "loss": 0.3805,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.584735631942749,
      "rewards/margins": 1.4259586334228516,
      "rewards/rejected": -3.0106940269470215,
      "step": 12180
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.986606485797131e-07,
      "logits/chosen": -1.083444356918335,
      "logits/rejected": -1.1372604370117188,
      "logps/chosen": -383.82757568359375,
      "logps/rejected": -481.3438415527344,
      "loss": 0.5378,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.8393703699111938,
      "rewards/margins": 0.9245020747184753,
      "rewards/rejected": -2.7638726234436035,
      "step": 12190
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.949581886326511e-07,
      "logits/chosen": -1.0700901746749878,
      "logits/rejected": -1.1865692138671875,
      "logps/chosen": -461.2632751464844,
      "logps/rejected": -524.6785888671875,
      "loss": 0.4775,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6347122192382812,
      "rewards/margins": 0.9232962727546692,
      "rewards/rejected": -2.5580084323883057,
      "step": 12200
    },
    {
      "epoch": 0.8,
      "eval_logits/chosen": -1.195405125617981,
      "eval_logits/rejected": -1.060084581375122,
      "eval_logps/chosen": -398.98553466796875,
      "eval_logps/rejected": -484.2724609375,
      "eval_loss": 0.48996442556381226,
      "eval_rewards/accuracies": 0.6679999828338623,
      "eval_rewards/chosen": -1.669805645942688,
      "eval_rewards/margins": 1.0568000078201294,
      "eval_rewards/rejected": -2.7266058921813965,
      "eval_runtime": 732.7693,
      "eval_samples_per_second": 2.729,
      "eval_steps_per_second": 1.365,
      "step": 12200
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.912656663168717e-07,
      "logits/chosen": -1.3327730894088745,
      "logits/rejected": -1.2746068239212036,
      "logps/chosen": -395.2479248046875,
      "logps/rejected": -469.552734375,
      "loss": 0.5479,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7123855352401733,
      "rewards/margins": 0.8067728281021118,
      "rewards/rejected": -2.5191586017608643,
      "step": 12210
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.875831008943817e-07,
      "logits/chosen": -1.0541003942489624,
      "logits/rejected": -1.028871774673462,
      "logps/chosen": -334.6326904296875,
      "logps/rejected": -380.36761474609375,
      "loss": 0.6011,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.569886565208435,
      "rewards/margins": 0.6833506226539612,
      "rewards/rejected": -2.253237247467041,
      "step": 12220
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.839105115752442e-07,
      "logits/chosen": -1.0335330963134766,
      "logits/rejected": -0.9258529543876648,
      "logps/chosen": -411.098876953125,
      "logps/rejected": -468.7681579589844,
      "loss": 0.5753,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8812135457992554,
      "rewards/margins": 0.9767706990242004,
      "rewards/rejected": -2.8579843044281006,
      "step": 12230
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.802479175174855e-07,
      "logits/chosen": -1.0795843601226807,
      "logits/rejected": -1.0609333515167236,
      "logps/chosen": -330.22186279296875,
      "logps/rejected": -442.25689697265625,
      "loss": 0.5117,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6422760486602783,
      "rewards/margins": 1.0610793828964233,
      "rewards/rejected": -2.703355312347412,
      "step": 12240
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.765953378269901e-07,
      "logits/chosen": -1.23228120803833,
      "logits/rejected": -1.238350510597229,
      "logps/chosen": -364.57135009765625,
      "logps/rejected": -524.7144775390625,
      "loss": 0.4668,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.575779914855957,
      "rewards/margins": 1.2909927368164062,
      "rewards/rejected": -2.8667726516723633,
      "step": 12250
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.729527915574037e-07,
      "logits/chosen": -1.114385962486267,
      "logits/rejected": -1.1761515140533447,
      "logps/chosen": -394.2991943359375,
      "logps/rejected": -498.36712646484375,
      "loss": 0.428,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7388063669204712,
      "rewards/margins": 1.0372883081436157,
      "rewards/rejected": -2.776094436645508,
      "step": 12260
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.693202977100304e-07,
      "logits/chosen": -1.0736769437789917,
      "logits/rejected": -0.946702778339386,
      "logps/chosen": -348.699462890625,
      "logps/rejected": -429.6670837402344,
      "loss": 0.5823,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.78554368019104,
      "rewards/margins": 0.8430388569831848,
      "rewards/rejected": -2.62858247756958,
      "step": 12270
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.656978752337389e-07,
      "logits/chosen": -1.2007639408111572,
      "logits/rejected": -1.1803643703460693,
      "logps/chosen": -399.9928894042969,
      "logps/rejected": -526.2418212890625,
      "loss": 0.5012,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.0033774375915527,
      "rewards/margins": 1.2484910488128662,
      "rewards/rejected": -3.251868486404419,
      "step": 12280
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.620855430248581e-07,
      "logits/chosen": -1.0226736068725586,
      "logits/rejected": -1.0721031427383423,
      "logps/chosen": -274.5044250488281,
      "logps/rejected": -405.9659729003906,
      "loss": 0.4094,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.125093698501587,
      "rewards/margins": 1.336398959159851,
      "rewards/rejected": -2.4614925384521484,
      "step": 12290
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.584833199270837e-07,
      "logits/chosen": -1.3347752094268799,
      "logits/rejected": -1.03730046749115,
      "logps/chosen": -417.37310791015625,
      "logps/rejected": -525.0247802734375,
      "loss": 0.4513,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8874075412750244,
      "rewards/margins": 1.1534497737884521,
      "rewards/rejected": -3.0408573150634766,
      "step": 12300
    },
    {
      "epoch": 0.8,
      "eval_logits/chosen": -1.197284460067749,
      "eval_logits/rejected": -1.0617622137069702,
      "eval_logps/chosen": -395.2168273925781,
      "eval_logps/rejected": -481.4833068847656,
      "eval_loss": 0.4890175759792328,
      "eval_rewards/accuracies": 0.6704999804496765,
      "eval_rewards/chosen": -1.632118582725525,
      "eval_rewards/margins": 1.0665956735610962,
      "eval_rewards/rejected": -2.698714017868042,
      "eval_runtime": 734.6568,
      "eval_samples_per_second": 2.722,
      "eval_steps_per_second": 1.361,
      "step": 12300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.548912247313742e-07,
      "logits/chosen": -1.559452772140503,
      "logits/rejected": -1.2473337650299072,
      "logps/chosen": -469.1334533691406,
      "logps/rejected": -515.8812255859375,
      "loss": 0.4888,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8533347845077515,
      "rewards/margins": 0.9432352185249329,
      "rewards/rejected": -2.796569585800171,
      "step": 12310
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.513092761758596e-07,
      "logits/chosen": -1.4145115613937378,
      "logits/rejected": -1.1812801361083984,
      "logps/chosen": -486.68231201171875,
      "logps/rejected": -490.2225036621094,
      "loss": 0.653,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.154723882675171,
      "rewards/margins": 0.7052110433578491,
      "rewards/rejected": -2.8599350452423096,
      "step": 12320
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.477374929457363e-07,
      "logits/chosen": -1.4101641178131104,
      "logits/rejected": -1.215849757194519,
      "logps/chosen": -375.29510498046875,
      "logps/rejected": -438.86187744140625,
      "loss": 0.5445,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7323395013809204,
      "rewards/margins": 0.8273865580558777,
      "rewards/rejected": -2.5597259998321533,
      "step": 12330
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.441758936731772e-07,
      "logits/chosen": -1.2345798015594482,
      "logits/rejected": -0.9684113264083862,
      "logps/chosen": -402.4441223144531,
      "logps/rejected": -490.9378967285156,
      "loss": 0.5247,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6709884405136108,
      "rewards/margins": 1.062840223312378,
      "rewards/rejected": -2.7338287830352783,
      "step": 12340
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.406244969372273e-07,
      "logits/chosen": -1.2190395593643188,
      "logits/rejected": -1.0561177730560303,
      "logps/chosen": -358.269287109375,
      "logps/rejected": -521.3367309570312,
      "loss": 0.3672,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5854642391204834,
      "rewards/margins": 1.5656545162200928,
      "rewards/rejected": -3.1511189937591553,
      "step": 12350
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.370833212637122e-07,
      "logits/chosen": -1.0202383995056152,
      "logits/rejected": -0.8779742121696472,
      "logps/chosen": -389.3901672363281,
      "logps/rejected": -490.498291015625,
      "loss": 0.466,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.708791732788086,
      "rewards/margins": 1.1489331722259521,
      "rewards/rejected": -2.857724905014038,
      "step": 12360
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.335523851251392e-07,
      "logits/chosen": -1.2496132850646973,
      "logits/rejected": -1.1335455179214478,
      "logps/chosen": -362.69232177734375,
      "logps/rejected": -447.7772521972656,
      "loss": 0.6054,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.565584421157837,
      "rewards/margins": 1.0588310956954956,
      "rewards/rejected": -2.624415397644043,
      "step": 12370
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.300317069406003e-07,
      "logits/chosen": -1.2288280725479126,
      "logits/rejected": -1.0688912868499756,
      "logps/chosen": -311.70428466796875,
      "logps/rejected": -439.04925537109375,
      "loss": 0.466,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5095841884613037,
      "rewards/margins": 1.2357814311981201,
      "rewards/rejected": -2.745365619659424,
      "step": 12380
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.265213050756782e-07,
      "logits/chosen": -1.4298914670944214,
      "logits/rejected": -1.1860029697418213,
      "logps/chosen": -406.72088623046875,
      "logps/rejected": -503.35589599609375,
      "loss": 0.6494,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.7466964721679688,
      "rewards/margins": 0.9876617193222046,
      "rewards/rejected": -2.734358310699463,
      "step": 12390
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.230211978423477e-07,
      "logits/chosen": -1.2988332509994507,
      "logits/rejected": -1.1887037754058838,
      "logps/chosen": -382.57122802734375,
      "logps/rejected": -462.72625732421875,
      "loss": 0.5514,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.6377102136611938,
      "rewards/margins": 0.9532381296157837,
      "rewards/rejected": -2.5909483432769775,
      "step": 12400
    },
    {
      "epoch": 0.81,
      "eval_logits/chosen": -1.191559910774231,
      "eval_logits/rejected": -1.0565053224563599,
      "eval_logps/chosen": -392.5486145019531,
      "eval_logps/rejected": -475.8312072753906,
      "eval_loss": 0.4892647862434387,
      "eval_rewards/accuracies": 0.6664999723434448,
      "eval_rewards/chosen": -1.60543692111969,
      "eval_rewards/margins": 1.0367563962936401,
      "eval_rewards/rejected": -2.64219331741333,
      "eval_runtime": 733.9406,
      "eval_samples_per_second": 2.725,
      "eval_steps_per_second": 1.363,
      "step": 12400
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.195314034988835e-07,
      "logits/chosen": -1.4519113302230835,
      "logits/rejected": -1.2425501346588135,
      "logps/chosen": -361.20416259765625,
      "logps/rejected": -404.88507080078125,
      "loss": 0.495,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4191709756851196,
      "rewards/margins": 1.1072009801864624,
      "rewards/rejected": -2.5263724327087402,
      "step": 12410
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.160519402497616e-07,
      "logits/chosen": -1.230360746383667,
      "logits/rejected": -1.1314585208892822,
      "logps/chosen": -421.28509521484375,
      "logps/rejected": -527.0120239257812,
      "loss": 0.4999,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9123108386993408,
      "rewards/margins": 1.0978678464889526,
      "rewards/rejected": -3.010178804397583,
      "step": 12420
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.125828262455679e-07,
      "logits/chosen": -1.1922781467437744,
      "logits/rejected": -0.9991927146911621,
      "logps/chosen": -407.961669921875,
      "logps/rejected": -493.1849060058594,
      "loss": 0.46,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5224859714508057,
      "rewards/margins": 1.155627727508545,
      "rewards/rejected": -2.6781134605407715,
      "step": 12430
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.091240795828992e-07,
      "logits/chosen": -0.9438712000846863,
      "logits/rejected": -0.9053109288215637,
      "logps/chosen": -362.1900939941406,
      "logps/rejected": -483.2745056152344,
      "loss": 0.453,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.596996545791626,
      "rewards/margins": 1.0125372409820557,
      "rewards/rejected": -2.6095337867736816,
      "step": 12440
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.056757183042732e-07,
      "logits/chosen": -1.1937294006347656,
      "logits/rejected": -1.0304909944534302,
      "logps/chosen": -407.8619384765625,
      "logps/rejected": -496.8045959472656,
      "loss": 0.4515,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7430578470230103,
      "rewards/margins": 1.0882017612457275,
      "rewards/rejected": -2.8312594890594482,
      "step": 12450
    },
    {
      "epoch": 0.82,
      "learning_rate": 5.022377603980308e-07,
      "logits/chosen": -1.404693841934204,
      "logits/rejected": -0.9854405522346497,
      "logps/chosen": -406.7601013183594,
      "logps/rejected": -469.7742614746094,
      "loss": 0.407,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.574753999710083,
      "rewards/margins": 1.2948553562164307,
      "rewards/rejected": -2.8696091175079346,
      "step": 12460
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.988102237982454e-07,
      "logits/chosen": -1.2445813417434692,
      "logits/rejected": -1.039184331893921,
      "logps/chosen": -412.42230224609375,
      "logps/rejected": -452.43231201171875,
      "loss": 0.5084,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9024187326431274,
      "rewards/margins": 0.8058527112007141,
      "rewards/rejected": -2.7082715034484863,
      "step": 12470
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.953931263846251e-07,
      "logits/chosen": -1.2200262546539307,
      "logits/rejected": -1.0197694301605225,
      "logps/chosen": -442.448974609375,
      "logps/rejected": -515.5526123046875,
      "loss": 0.5027,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8074004650115967,
      "rewards/margins": 1.1706429719924927,
      "rewards/rejected": -2.978043556213379,
      "step": 12480
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.919864859824266e-07,
      "logits/chosen": -1.1267067193984985,
      "logits/rejected": -1.1097023487091064,
      "logps/chosen": -420.2432556152344,
      "logps/rejected": -481.21209716796875,
      "loss": 0.4969,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.8280909061431885,
      "rewards/margins": 1.0600405931472778,
      "rewards/rejected": -2.888131618499756,
      "step": 12490
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.885903203623532e-07,
      "logits/chosen": -1.5061956644058228,
      "logits/rejected": -0.9625085592269897,
      "logps/chosen": -440.2173767089844,
      "logps/rejected": -514.4992065429688,
      "loss": 0.4187,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.552034854888916,
      "rewards/margins": 1.3805524110794067,
      "rewards/rejected": -2.932587146759033,
      "step": 12500
    },
    {
      "epoch": 0.82,
      "eval_logits/chosen": -1.143744945526123,
      "eval_logits/rejected": -1.0092847347259521,
      "eval_logps/chosen": -400.1340026855469,
      "eval_logps/rejected": -489.6676330566406,
      "eval_loss": 0.487697571516037,
      "eval_rewards/accuracies": 0.6685000061988831,
      "eval_rewards/chosen": -1.6812905073165894,
      "eval_rewards/margins": 1.0992670059204102,
      "eval_rewards/rejected": -2.78055739402771,
      "eval_runtime": 731.5547,
      "eval_samples_per_second": 2.734,
      "eval_steps_per_second": 1.367,
      "step": 12500
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.852046472404695e-07,
      "logits/chosen": -1.213766098022461,
      "logits/rejected": -0.7593834400177002,
      "logps/chosen": -455.54986572265625,
      "logps/rejected": -451.78857421875,
      "loss": 0.4655,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5766116380691528,
      "rewards/margins": 1.219120740890503,
      "rewards/rejected": -2.7957324981689453,
      "step": 12510
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.818294842781035e-07,
      "logits/chosen": -1.3219027519226074,
      "logits/rejected": -1.0103774070739746,
      "logps/chosen": -366.6271057128906,
      "logps/rejected": -465.59600830078125,
      "loss": 0.4485,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4041863679885864,
      "rewards/margins": 1.5132675170898438,
      "rewards/rejected": -2.9174537658691406,
      "step": 12520
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.784648490817601e-07,
      "logits/chosen": -1.2231624126434326,
      "logits/rejected": -1.0757906436920166,
      "logps/chosen": -376.15869140625,
      "logps/rejected": -442.31689453125,
      "loss": 0.552,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.5605192184448242,
      "rewards/margins": 1.0941671133041382,
      "rewards/rejected": -2.654686689376831,
      "step": 12530
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.751107592030235e-07,
      "logits/chosen": -1.202649474143982,
      "logits/rejected": -0.9246581196784973,
      "logps/chosen": -297.8594665527344,
      "logps/rejected": -425.0567321777344,
      "loss": 0.3797,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2800767421722412,
      "rewards/margins": 1.4429724216461182,
      "rewards/rejected": -2.7230491638183594,
      "step": 12540
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.717672321384703e-07,
      "logits/chosen": -1.1258949041366577,
      "logits/rejected": -0.6755282282829285,
      "logps/chosen": -390.7144470214844,
      "logps/rejected": -458.26007080078125,
      "loss": 0.4647,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.674687147140503,
      "rewards/margins": 1.0974136590957642,
      "rewards/rejected": -2.7721009254455566,
      "step": 12550
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.684342853295748e-07,
      "logits/chosen": -0.9751138687133789,
      "logits/rejected": -0.9520295262336731,
      "logps/chosen": -335.04559326171875,
      "logps/rejected": -444.92578125,
      "loss": 0.5004,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.473488211631775,
      "rewards/margins": 1.1380419731140137,
      "rewards/rejected": -2.611530065536499,
      "step": 12560
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.651119361626213e-07,
      "logits/chosen": -1.5337803363800049,
      "logits/rejected": -1.0685784816741943,
      "logps/chosen": -383.00390625,
      "logps/rejected": -446.67694091796875,
      "loss": 0.4472,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4188528060913086,
      "rewards/margins": 1.0990186929702759,
      "rewards/rejected": -2.517871379852295,
      "step": 12570
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.618002019686091e-07,
      "logits/chosen": -1.1882233619689941,
      "logits/rejected": -1.1861683130264282,
      "logps/chosen": -456.61456298828125,
      "logps/rejected": -500.18994140625,
      "loss": 0.6327,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.774022102355957,
      "rewards/margins": 1.0021147727966309,
      "rewards/rejected": -2.776136875152588,
      "step": 12580
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.5849910002316757e-07,
      "logits/chosen": -1.2046785354614258,
      "logits/rejected": -1.0692670345306396,
      "logps/chosen": -360.4275817871094,
      "logps/rejected": -454.0997009277344,
      "loss": 0.4372,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.799154281616211,
      "rewards/margins": 1.2693634033203125,
      "rewards/rejected": -3.0685176849365234,
      "step": 12590
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.5520864754645984e-07,
      "logits/chosen": -1.4719334840774536,
      "logits/rejected": -1.2618463039398193,
      "logps/chosen": -452.3905334472656,
      "logps/rejected": -497.9466857910156,
      "loss": 0.549,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7104072570800781,
      "rewards/margins": 0.8938120007514954,
      "rewards/rejected": -2.6042189598083496,
      "step": 12600
    },
    {
      "epoch": 0.82,
      "eval_logits/chosen": -1.1529422998428345,
      "eval_logits/rejected": -1.0171226263046265,
      "eval_logps/chosen": -399.7242736816406,
      "eval_logps/rejected": -491.4219665527344,
      "eval_loss": 0.4873626232147217,
      "eval_rewards/accuracies": 0.6694999933242798,
      "eval_rewards/chosen": -1.677193522453308,
      "eval_rewards/margins": 1.1209073066711426,
      "eval_rewards/rejected": -2.7981014251708984,
      "eval_runtime": 733.7259,
      "eval_samples_per_second": 2.726,
      "eval_steps_per_second": 1.363,
      "step": 12600
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.5192886170309896e-07,
      "logits/chosen": -1.0846290588378906,
      "logits/rejected": -1.0304473638534546,
      "logps/chosen": -357.4346618652344,
      "logps/rejected": -440.633544921875,
      "loss": 0.6216,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6666618585586548,
      "rewards/margins": 0.8301478624343872,
      "rewards/rejected": -2.496809482574463,
      "step": 12610
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.486597596020548e-07,
      "logits/chosen": -1.0042126178741455,
      "logits/rejected": -1.0394890308380127,
      "logps/chosen": -413.8369140625,
      "logps/rejected": -502.90618896484375,
      "loss": 0.4393,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9425557851791382,
      "rewards/margins": 1.2739102840423584,
      "rewards/rejected": -3.2164664268493652,
      "step": 12620
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.454013582965644e-07,
      "logits/chosen": -0.9420401453971863,
      "logits/rejected": -0.7326041460037231,
      "logps/chosen": -453.96148681640625,
      "logps/rejected": -489.3179626464844,
      "loss": 0.5523,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.9050474166870117,
      "rewards/margins": 0.8579981923103333,
      "rewards/rejected": -2.7630457878112793,
      "step": 12630
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.4215367478404605e-07,
      "logits/chosen": -1.0251177549362183,
      "logits/rejected": -0.948135495185852,
      "logps/chosen": -467.2874450683594,
      "logps/rejected": -587.9501953125,
      "loss": 0.5126,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9652748107910156,
      "rewards/margins": 0.9629101753234863,
      "rewards/rejected": -2.928185224533081,
      "step": 12640
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.389167260060068e-07,
      "logits/chosen": -1.1111929416656494,
      "logits/rejected": -1.0775830745697021,
      "logps/chosen": -350.09063720703125,
      "logps/rejected": -454.56695556640625,
      "loss": 0.3838,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4140043258666992,
      "rewards/margins": 1.4265081882476807,
      "rewards/rejected": -2.84051251411438,
      "step": 12650
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.356905288479579e-07,
      "logits/chosen": -1.1360782384872437,
      "logits/rejected": -0.882355809211731,
      "logps/chosen": -378.951416015625,
      "logps/rejected": -504.17218017578125,
      "loss": 0.3417,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5243055820465088,
      "rewards/margins": 1.5398471355438232,
      "rewards/rejected": -3.064152956008911,
      "step": 12660
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.3247510013932377e-07,
      "logits/chosen": -0.8389037251472473,
      "logits/rejected": -0.9085738062858582,
      "logps/chosen": -431.64239501953125,
      "logps/rejected": -547.0601806640625,
      "loss": 0.5185,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7945387363433838,
      "rewards/margins": 1.1559791564941406,
      "rewards/rejected": -2.9505176544189453,
      "step": 12670
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.2927045665335594e-07,
      "logits/chosen": -0.5805936455726624,
      "logits/rejected": -0.7223195433616638,
      "logps/chosen": -361.5916442871094,
      "logps/rejected": -439.7505798339844,
      "loss": 0.549,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.9671390056610107,
      "rewards/margins": 0.8171416521072388,
      "rewards/rejected": -2.784280300140381,
      "step": 12680
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.260766151070439e-07,
      "logits/chosen": -0.8927919268608093,
      "logits/rejected": -0.881097137928009,
      "logps/chosen": -398.68975830078125,
      "logps/rejected": -498.0592346191406,
      "loss": 0.5202,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7425625324249268,
      "rewards/margins": 1.106300711631775,
      "rewards/rejected": -2.848863124847412,
      "step": 12690
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.228935921610308e-07,
      "logits/chosen": -1.243019461631775,
      "logits/rejected": -0.9916293025016785,
      "logps/chosen": -446.41015625,
      "logps/rejected": -461.8265686035156,
      "loss": 0.5839,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8269145488739014,
      "rewards/margins": 0.7946796417236328,
      "rewards/rejected": -2.621594190597534,
      "step": 12700
    },
    {
      "epoch": 0.83,
      "eval_logits/chosen": -1.1701284646987915,
      "eval_logits/rejected": -1.0344866514205933,
      "eval_logps/chosen": -393.4916687011719,
      "eval_logps/rejected": -482.1238098144531,
      "eval_loss": 0.4880444407463074,
      "eval_rewards/accuracies": 0.6690000295639038,
      "eval_rewards/chosen": -1.6148673295974731,
      "eval_rewards/margins": 1.0902516841888428,
      "eval_rewards/rejected": -2.7051191329956055,
      "eval_runtime": 732.9848,
      "eval_samples_per_second": 2.729,
      "eval_steps_per_second": 1.364,
      "step": 12700
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1972140441952246e-07,
      "logits/chosen": -1.0796960592269897,
      "logits/rejected": -1.1231991052627563,
      "logps/chosen": -391.99188232421875,
      "logps/rejected": -498.2669372558594,
      "loss": 0.5049,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5240862369537354,
      "rewards/margins": 1.0426585674285889,
      "rewards/rejected": -2.5667450428009033,
      "step": 12710
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.165600684302046e-07,
      "logits/chosen": -1.2587755918502808,
      "logits/rejected": -1.1907663345336914,
      "logps/chosen": -297.47430419921875,
      "logps/rejected": -418.65264892578125,
      "loss": 0.3999,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2169029712677002,
      "rewards/margins": 1.1122041940689087,
      "rewards/rejected": -2.3291072845458984,
      "step": 12720
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.13409600684154e-07,
      "logits/chosen": -1.240088701248169,
      "logits/rejected": -1.0318121910095215,
      "logps/chosen": -382.499755859375,
      "logps/rejected": -454.86083984375,
      "loss": 0.4893,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.6748014688491821,
      "rewards/margins": 1.0154438018798828,
      "rewards/rejected": -2.6902451515197754,
      "step": 12730
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.102700176157548e-07,
      "logits/chosen": -1.3106701374053955,
      "logits/rejected": -1.010158896446228,
      "logps/chosen": -507.3036193847656,
      "logps/rejected": -505.44390869140625,
      "loss": 0.5486,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8254835605621338,
      "rewards/margins": 0.9592690467834473,
      "rewards/rejected": -2.784752368927002,
      "step": 12740
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.0714133560260884e-07,
      "logits/chosen": -1.215022087097168,
      "logits/rejected": -1.1796523332595825,
      "logps/chosen": -440.1405334472656,
      "logps/rejected": -468.4134216308594,
      "loss": 0.5162,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8060743808746338,
      "rewards/margins": 0.8650380969047546,
      "rewards/rejected": -2.671112537384033,
      "step": 12750
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.0402357096545527e-07,
      "logits/chosen": -0.9729182124137878,
      "logits/rejected": -1.1191080808639526,
      "logps/chosen": -415.92535400390625,
      "logps/rejected": -502.9111328125,
      "loss": 0.4751,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6416374444961548,
      "rewards/margins": 0.9715788960456848,
      "rewards/rejected": -2.6132164001464844,
      "step": 12760
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.0091673996808025e-07,
      "logits/chosen": -1.3912891149520874,
      "logits/rejected": -1.1987309455871582,
      "logps/chosen": -401.42315673828125,
      "logps/rejected": -461.45721435546875,
      "loss": 0.5461,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.043630361557007,
      "rewards/margins": 0.8466132283210754,
      "rewards/rejected": -2.8902435302734375,
      "step": 12770
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9782085881723776e-07,
      "logits/chosen": -1.142588496208191,
      "logits/rejected": -1.0690782070159912,
      "logps/chosen": -325.50653076171875,
      "logps/rejected": -455.8252868652344,
      "loss": 0.5445,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.5990889072418213,
      "rewards/margins": 1.1835216283798218,
      "rewards/rejected": -2.7826106548309326,
      "step": 12780
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.947359436625592e-07,
      "logits/chosen": -1.1046050786972046,
      "logits/rejected": -0.9014351963996887,
      "logps/chosen": -420.23046875,
      "logps/rejected": -497.6240234375,
      "loss": 0.4034,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8556751012802124,
      "rewards/margins": 1.1495764255523682,
      "rewards/rejected": -3.005251407623291,
      "step": 12790
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9166201059647386e-07,
      "logits/chosen": -1.1169941425323486,
      "logits/rejected": -1.1712825298309326,
      "logps/chosen": -450.0757751464844,
      "logps/rejected": -472.9488220214844,
      "loss": 0.6596,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.8822559118270874,
      "rewards/margins": 0.5980395078659058,
      "rewards/rejected": -2.4802956581115723,
      "step": 12800
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": -1.1303144693374634,
      "eval_logits/rejected": -0.9964880347251892,
      "eval_logps/chosen": -411.1650390625,
      "eval_logps/rejected": -499.86004638671875,
      "eval_loss": 0.4863698184490204,
      "eval_rewards/accuracies": 0.6704999804496765,
      "eval_rewards/chosen": -1.7916008234024048,
      "eval_rewards/margins": 1.0908806324005127,
      "eval_rewards/rejected": -2.882481336593628,
      "eval_runtime": 730.2818,
      "eval_samples_per_second": 2.739,
      "eval_steps_per_second": 1.369,
      "step": 12800
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.8859907565412194e-07,
      "logits/chosen": -1.0224831104278564,
      "logits/rejected": -1.3116912841796875,
      "logps/chosen": -375.24005126953125,
      "logps/rejected": -483.36016845703125,
      "loss": 0.4366,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.955908179283142,
      "rewards/margins": 1.0869848728179932,
      "rewards/rejected": -3.042893171310425,
      "step": 12810
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.8554715481327303e-07,
      "logits/chosen": -1.0355466604232788,
      "logits/rejected": -1.0103200674057007,
      "logps/chosen": -429.4497985839844,
      "logps/rejected": -506.0899353027344,
      "loss": 0.5426,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.9619081020355225,
      "rewards/margins": 1.1203787326812744,
      "rewards/rejected": -3.082286834716797,
      "step": 12820
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.8250626399424007e-07,
      "logits/chosen": -1.2228209972381592,
      "logits/rejected": -1.1389085054397583,
      "logps/chosen": -451.1490173339844,
      "logps/rejected": -517.5762939453125,
      "loss": 0.5481,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.996708631515503,
      "rewards/margins": 0.8527741432189941,
      "rewards/rejected": -2.849483013153076,
      "step": 12830
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.7947641905980104e-07,
      "logits/chosen": -1.0452697277069092,
      "logits/rejected": -1.1376216411590576,
      "logps/chosen": -362.114501953125,
      "logps/rejected": -431.8590393066406,
      "loss": 0.4584,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5479605197906494,
      "rewards/margins": 1.044252634048462,
      "rewards/rejected": -2.5922131538391113,
      "step": 12840
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.764576358151098e-07,
      "logits/chosen": -1.148471474647522,
      "logits/rejected": -1.061302900314331,
      "logps/chosen": -334.8290710449219,
      "logps/rejected": -426.7361755371094,
      "loss": 0.5224,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.6001205444335938,
      "rewards/margins": 1.037514090538025,
      "rewards/rejected": -2.637634754180908,
      "step": 12850
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.7344993000761944e-07,
      "logits/chosen": -1.1538784503936768,
      "logits/rejected": -1.0829343795776367,
      "logps/chosen": -366.9634094238281,
      "logps/rejected": -522.6871337890625,
      "loss": 0.467,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.896482229232788,
      "rewards/margins": 1.1275200843811035,
      "rewards/rejected": -3.0240025520324707,
      "step": 12860
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.7045331732699585e-07,
      "logits/chosen": -1.1157572269439697,
      "logits/rejected": -0.9886964559555054,
      "logps/chosen": -388.4722595214844,
      "logps/rejected": -488.2107849121094,
      "loss": 0.4774,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8327420949935913,
      "rewards/margins": 1.3543800115585327,
      "rewards/rejected": -3.187121868133545,
      "step": 12870
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.6746781340503993e-07,
      "logits/chosen": -1.1017290353775024,
      "logits/rejected": -0.8482405543327332,
      "logps/chosen": -395.7943420410156,
      "logps/rejected": -491.15899658203125,
      "loss": 0.5106,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.613785743713379,
      "rewards/margins": 1.0339164733886719,
      "rewards/rejected": -2.647702217102051,
      "step": 12880
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.6449343381560116e-07,
      "logits/chosen": -0.9443025588989258,
      "logits/rejected": -0.782778799533844,
      "logps/chosen": -458.7662048339844,
      "logps/rejected": -560.9354248046875,
      "loss": 0.4543,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.2734053134918213,
      "rewards/margins": 1.155586838722229,
      "rewards/rejected": -3.428992509841919,
      "step": 12890
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.615301940745017e-07,
      "logits/chosen": -1.4761455059051514,
      "logits/rejected": -0.9457404017448425,
      "logps/chosen": -489.544189453125,
      "logps/rejected": -482.9058532714844,
      "loss": 0.5277,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7234907150268555,
      "rewards/margins": 0.9853858947753906,
      "rewards/rejected": -2.708876848220825,
      "step": 12900
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": -1.1100314855575562,
      "eval_logits/rejected": -0.9771387577056885,
      "eval_logps/chosen": -417.5810241699219,
      "eval_logps/rejected": -506.6070251464844,
      "eval_loss": 0.4858885109424591,
      "eval_rewards/accuracies": 0.6694999933242798,
      "eval_rewards/chosen": -1.8557612895965576,
      "eval_rewards/margins": 1.0941897630691528,
      "eval_rewards/rejected": -2.949951171875,
      "eval_runtime": 735.1341,
      "eval_samples_per_second": 2.721,
      "eval_steps_per_second": 1.36,
      "step": 12900
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.5857810963945084e-07,
      "logits/chosen": -0.7660588622093201,
      "logits/rejected": -0.8179569244384766,
      "logps/chosen": -419.36907958984375,
      "logps/rejected": -510.74688720703125,
      "loss": 0.4735,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0485999584198,
      "rewards/margins": 1.0620059967041016,
      "rewards/rejected": -3.1106061935424805,
      "step": 12910
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.556371959099678e-07,
      "logits/chosen": -1.3508169651031494,
      "logits/rejected": -1.0648918151855469,
      "logps/chosen": -485.3834533691406,
      "logps/rejected": -553.64501953125,
      "loss": 0.4721,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8820909261703491,
      "rewards/margins": 1.0823203325271606,
      "rewards/rejected": -2.9644112586975098,
      "step": 12920
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.5270746822729797e-07,
      "logits/chosen": -1.164005160331726,
      "logits/rejected": -0.9906147122383118,
      "logps/chosen": -466.7239685058594,
      "logps/rejected": -584.4600830078125,
      "loss": 0.5559,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1924190521240234,
      "rewards/margins": 1.0516093969345093,
      "rewards/rejected": -3.244028091430664,
      "step": 12930
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4978894187433746e-07,
      "logits/chosen": -1.0629559755325317,
      "logits/rejected": -1.0066983699798584,
      "logps/chosen": -319.3057556152344,
      "logps/rejected": -384.53765869140625,
      "loss": 0.5708,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6823055744171143,
      "rewards/margins": 0.7614030241966248,
      "rewards/rejected": -2.443708896636963,
      "step": 12940
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.468816320755486e-07,
      "logits/chosen": -0.8171752691268921,
      "logits/rejected": -0.8537039756774902,
      "logps/chosen": -386.3449401855469,
      "logps/rejected": -427.5325622558594,
      "loss": 0.5521,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.6241718530654907,
      "rewards/margins": 0.8079736828804016,
      "rewards/rejected": -2.432145357131958,
      "step": 12950
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4398555399688336e-07,
      "logits/chosen": -1.1284414529800415,
      "logits/rejected": -0.9537376165390015,
      "logps/chosen": -435.8028259277344,
      "logps/rejected": -462.26953125,
      "loss": 0.7083,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.244523286819458,
      "rewards/margins": 0.471309095621109,
      "rewards/rejected": -2.715832233428955,
      "step": 12960
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.411007227457047e-07,
      "logits/chosen": -1.224440097808838,
      "logits/rejected": -0.9128150939941406,
      "logps/chosen": -418.45770263671875,
      "logps/rejected": -505.29766845703125,
      "loss": 0.459,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7069644927978516,
      "rewards/margins": 1.191146969795227,
      "rewards/rejected": -2.898111343383789,
      "step": 12970
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.382271533707043e-07,
      "logits/chosen": -0.9220924377441406,
      "logits/rejected": -0.8552336692810059,
      "logps/chosen": -370.10992431640625,
      "logps/rejected": -422.84381103515625,
      "loss": 0.5728,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.785047173500061,
      "rewards/margins": 0.7561759948730469,
      "rewards/rejected": -2.5412230491638184,
      "step": 12980
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.353648608618287e-07,
      "logits/chosen": -1.0896251201629639,
      "logits/rejected": -0.9746156930923462,
      "logps/chosen": -324.8300476074219,
      "logps/rejected": -415.791015625,
      "loss": 0.4819,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5695884227752686,
      "rewards/margins": 1.0093162059783936,
      "rewards/rejected": -2.578904628753662,
      "step": 12990
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.3251386015019676e-07,
      "logits/chosen": -1.3970059156417847,
      "logits/rejected": -1.1359691619873047,
      "logps/chosen": -388.5389709472656,
      "logps/rejected": -452.7955627441406,
      "loss": 0.4608,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8577959537506104,
      "rewards/margins": 0.980010986328125,
      "rewards/rejected": -2.8378071784973145,
      "step": 13000
    },
    {
      "epoch": 0.85,
      "eval_logits/chosen": -1.0929280519485474,
      "eval_logits/rejected": -0.9613592028617859,
      "eval_logps/chosen": -421.5428161621094,
      "eval_logps/rejected": -508.9826965332031,
      "eval_loss": 0.4859364330768585,
      "eval_rewards/accuracies": 0.6735000014305115,
      "eval_rewards/chosen": -1.8953787088394165,
      "eval_rewards/margins": 1.0783287286758423,
      "eval_rewards/rejected": -2.973707437515259,
      "eval_runtime": 733.5256,
      "eval_samples_per_second": 2.727,
      "eval_steps_per_second": 1.363,
      "step": 13000
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.296741661080255e-07,
      "logits/chosen": -1.226764440536499,
      "logits/rejected": -1.0761001110076904,
      "logps/chosen": -456.44622802734375,
      "logps/rejected": -541.66796875,
      "loss": 0.7879,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.233079195022583,
      "rewards/margins": 0.917628288269043,
      "rewards/rejected": -3.150707244873047,
      "step": 13010
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.2684579354854974e-07,
      "logits/chosen": -1.285154104232788,
      "logits/rejected": -1.1769609451293945,
      "logps/chosen": -517.5054931640625,
      "logps/rejected": -646.6309204101562,
      "loss": 0.4975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2504684925079346,
      "rewards/margins": 1.1886699199676514,
      "rewards/rejected": -3.439138412475586,
      "step": 13020
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.2402875722594653e-07,
      "logits/chosen": -1.0246667861938477,
      "logits/rejected": -1.128968596458435,
      "logps/chosen": -337.71929931640625,
      "logps/rejected": -448.1748962402344,
      "loss": 0.526,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.68673837184906,
      "rewards/margins": 1.0430407524108887,
      "rewards/rejected": -2.729779005050659,
      "step": 13030
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.212230718352566e-07,
      "logits/chosen": -1.0177228450775146,
      "logits/rejected": -0.9467980265617371,
      "logps/chosen": -415.2057189941406,
      "logps/rejected": -393.44915771484375,
      "loss": 0.661,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.90155827999115,
      "rewards/margins": 0.41940444707870483,
      "rewards/rejected": -2.32096266746521,
      "step": 13040
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.1842875201231025e-07,
      "logits/chosen": -1.015235185623169,
      "logits/rejected": -0.871992290019989,
      "logps/chosen": -386.43499755859375,
      "logps/rejected": -459.2357482910156,
      "loss": 0.4994,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.6469495296478271,
      "rewards/margins": 0.985759437084198,
      "rewards/rejected": -2.63270902633667,
      "step": 13050
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.156458123336478e-07,
      "logits/chosen": -0.8194257020950317,
      "logits/rejected": -0.7359941601753235,
      "logps/chosen": -321.8291015625,
      "logps/rejected": -450.9248046875,
      "loss": 0.424,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6331720352172852,
      "rewards/margins": 1.3924449682235718,
      "rewards/rejected": -3.0256171226501465,
      "step": 13060
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.128742673164459e-07,
      "logits/chosen": -1.3152307271957397,
      "logits/rejected": -0.8516858816146851,
      "logps/chosen": -475.12158203125,
      "logps/rejected": -545.9874267578125,
      "loss": 0.4057,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.9219434261322021,
      "rewards/margins": 1.1538183689117432,
      "rewards/rejected": -3.0757617950439453,
      "step": 13070
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.101141314184414e-07,
      "logits/chosen": -1.448878288269043,
      "logits/rejected": -1.2280422449111938,
      "logps/chosen": -382.58502197265625,
      "logps/rejected": -466.08428955078125,
      "loss": 0.4464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.77200448513031,
      "rewards/margins": 0.9271053075790405,
      "rewards/rejected": -2.6991097927093506,
      "step": 13080
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.0736541903785526e-07,
      "logits/chosen": -0.8780330419540405,
      "logits/rejected": -0.9904329180717468,
      "logps/chosen": -415.8080139160156,
      "logps/rejected": -569.5153198242188,
      "loss": 0.523,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.0679306983947754,
      "rewards/margins": 1.0424773693084717,
      "rewards/rejected": -3.110407590866089,
      "step": 13090
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.0462814451331704e-07,
      "logits/chosen": -1.0929917097091675,
      "logits/rejected": -0.8251727223396301,
      "logps/chosen": -455.87774658203125,
      "logps/rejected": -526.8124389648438,
      "loss": 0.5661,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2520129680633545,
      "rewards/margins": 0.710297167301178,
      "rewards/rejected": -2.962310314178467,
      "step": 13100
    },
    {
      "epoch": 0.86,
      "eval_logits/chosen": -1.0824329853057861,
      "eval_logits/rejected": -0.9514362215995789,
      "eval_logps/chosen": -421.4238586425781,
      "eval_logps/rejected": -507.9122009277344,
      "eval_loss": 0.48598480224609375,
      "eval_rewards/accuracies": 0.6725000143051147,
      "eval_rewards/chosen": -1.894188642501831,
      "eval_rewards/margins": 1.0688139200210571,
      "eval_rewards/rejected": -2.9630024433135986,
      "eval_runtime": 733.4557,
      "eval_samples_per_second": 2.727,
      "eval_steps_per_second": 1.363,
      "step": 13100
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.019023221237927e-07,
      "logits/chosen": -1.0396716594696045,
      "logits/rejected": -0.8928040266036987,
      "logps/chosen": -417.76763916015625,
      "logps/rejected": -469.63482666015625,
      "loss": 0.4552,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8103529214859009,
      "rewards/margins": 1.0943458080291748,
      "rewards/rejected": -2.9046988487243652,
      "step": 13110
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.991879660885058e-07,
      "logits/chosen": -1.2439924478530884,
      "logits/rejected": -1.165687084197998,
      "logps/chosen": -440.7635803222656,
      "logps/rejected": -545.8884887695312,
      "loss": 0.475,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7721668481826782,
      "rewards/margins": 1.2148511409759521,
      "rewards/rejected": -2.987017869949341,
      "step": 13120
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.9648509056686786e-07,
      "logits/chosen": -1.1436467170715332,
      "logits/rejected": -0.9742773175239563,
      "logps/chosen": -370.8565368652344,
      "logps/rejected": -660.5780029296875,
      "loss": 0.5094,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.939147710800171,
      "rewards/margins": 3.1529717445373535,
      "rewards/rejected": -5.0921196937561035,
      "step": 13130
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.937937096584012e-07,
      "logits/chosen": -1.150695562362671,
      "logits/rejected": -0.8165030479431152,
      "logps/chosen": -500.3816833496094,
      "logps/rejected": -528.1851806640625,
      "loss": 0.5713,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0998072624206543,
      "rewards/margins": 0.8828747868537903,
      "rewards/rejected": -2.9826817512512207,
      "step": 13140
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.9111383740266756e-07,
      "logits/chosen": -0.9092572927474976,
      "logits/rejected": -0.9110123515129089,
      "logps/chosen": -444.96563720703125,
      "logps/rejected": -513.8148193359375,
      "loss": 0.5191,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.0918147563934326,
      "rewards/margins": 0.7463721036911011,
      "rewards/rejected": -2.8381869792938232,
      "step": 13150
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.8844548777919255e-07,
      "logits/chosen": -1.1265146732330322,
      "logits/rejected": -1.0310777425765991,
      "logps/chosen": -376.72259521484375,
      "logps/rejected": -427.83428955078125,
      "loss": 0.6115,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.728387475013733,
      "rewards/margins": 0.7339189648628235,
      "rewards/rejected": -2.462306261062622,
      "step": 13160
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.8578867470739594e-07,
      "logits/chosen": -0.7835243940353394,
      "logits/rejected": -0.7032335996627808,
      "logps/chosen": -369.31378173828125,
      "logps/rejected": -463.86767578125,
      "loss": 0.4816,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8482542037963867,
      "rewards/margins": 1.259765863418579,
      "rewards/rejected": -3.1080198287963867,
      "step": 13170
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.8314341204651484e-07,
      "logits/chosen": -1.4030210971832275,
      "logits/rejected": -1.210394263267517,
      "logps/chosen": -436.91845703125,
      "logps/rejected": -492.1163635253906,
      "loss": 0.3838,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.6087934970855713,
      "rewards/margins": 1.2798042297363281,
      "rewards/rejected": -2.8885979652404785,
      "step": 13180
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.805097135955362e-07,
      "logits/chosen": -1.0102050304412842,
      "logits/rejected": -0.8716405630111694,
      "logps/chosen": -389.81170654296875,
      "logps/rejected": -457.72113037109375,
      "loss": 0.4974,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.7728769779205322,
      "rewards/margins": 1.0283530950546265,
      "rewards/rejected": -2.801229953765869,
      "step": 13190
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.778875930931213e-07,
      "logits/chosen": -1.0638980865478516,
      "logits/rejected": -0.816801905632019,
      "logps/chosen": -417.91619873046875,
      "logps/rejected": -522.2945556640625,
      "loss": 0.4732,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8279731273651123,
      "rewards/margins": 1.217405080795288,
      "rewards/rejected": -3.0453782081604004,
      "step": 13200
    },
    {
      "epoch": 0.86,
      "eval_logits/chosen": -1.0934062004089355,
      "eval_logits/rejected": -0.9613891243934631,
      "eval_logps/chosen": -416.2484130859375,
      "eval_logps/rejected": -504.401611328125,
      "eval_loss": 0.4857296645641327,
      "eval_rewards/accuracies": 0.6704999804496765,
      "eval_rewards/chosen": -1.8424345254898071,
      "eval_rewards/margins": 1.0854624509811401,
      "eval_rewards/rejected": -2.9278969764709473,
      "eval_runtime": 731.1175,
      "eval_samples_per_second": 2.736,
      "eval_steps_per_second": 1.368,
      "step": 13200
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.7527706421753426e-07,
      "logits/chosen": -1.2587956190109253,
      "logits/rejected": -1.0719627141952515,
      "logps/chosen": -374.5654602050781,
      "logps/rejected": -472.62548828125,
      "loss": 0.5174,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7616428136825562,
      "rewards/margins": 0.9357126355171204,
      "rewards/rejected": -2.6973557472229004,
      "step": 13210
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.726781405865736e-07,
      "logits/chosen": -1.1134307384490967,
      "logits/rejected": -0.9242690205574036,
      "logps/chosen": -492.21295166015625,
      "logps/rejected": -477.84063720703125,
      "loss": 0.5215,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8954721689224243,
      "rewards/margins": 1.0139615535736084,
      "rewards/rejected": -2.9094338417053223,
      "step": 13220
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.7009083575749687e-07,
      "logits/chosen": -0.9584072828292847,
      "logits/rejected": -0.9138525724411011,
      "logps/chosen": -425.8014221191406,
      "logps/rejected": -512.4229125976562,
      "loss": 0.5108,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.819411039352417,
      "rewards/margins": 0.8825702667236328,
      "rewards/rejected": -2.7019810676574707,
      "step": 13230
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.6751516322695457e-07,
      "logits/chosen": -1.102455496788025,
      "logits/rejected": -1.0879199504852295,
      "logps/chosen": -396.62811279296875,
      "logps/rejected": -453.2647399902344,
      "loss": 0.5622,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.063845634460449,
      "rewards/margins": 0.6267648935317993,
      "rewards/rejected": -2.690610885620117,
      "step": 13240
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.649511364309154e-07,
      "logits/chosen": -1.3207348585128784,
      "logits/rejected": -1.175112009048462,
      "logps/chosen": -387.6021423339844,
      "logps/rejected": -464.5248107910156,
      "loss": 0.4696,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8531246185302734,
      "rewards/margins": 0.9780380129814148,
      "rewards/rejected": -2.831162214279175,
      "step": 13250
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.6239876874460003e-07,
      "logits/chosen": -1.285677194595337,
      "logits/rejected": -1.268032431602478,
      "logps/chosen": -476.5404357910156,
      "logps/rejected": -601.5635986328125,
      "loss": 0.4047,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.921252965927124,
      "rewards/margins": 1.5557920932769775,
      "rewards/rejected": -3.4770455360412598,
      "step": 13260
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5985807348240744e-07,
      "logits/chosen": -1.1611740589141846,
      "logits/rejected": -0.7273094058036804,
      "logps/chosen": -423.51837158203125,
      "logps/rejected": -547.768798828125,
      "loss": 0.3697,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.911747694015503,
      "rewards/margins": 1.7219756841659546,
      "rewards/rejected": -3.633723020553589,
      "step": 13270
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5732906389785014e-07,
      "logits/chosen": -1.2326469421386719,
      "logits/rejected": -1.1700255870819092,
      "logps/chosen": -460.8599548339844,
      "logps/rejected": -551.9319458007812,
      "loss": 0.4286,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.831538438796997,
      "rewards/margins": 1.2921754121780396,
      "rewards/rejected": -3.123713970184326,
      "step": 13280
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5481175318347956e-07,
      "logits/chosen": -0.9984075427055359,
      "logits/rejected": -1.1866605281829834,
      "logps/chosen": -407.84014892578125,
      "logps/rejected": -542.4286499023438,
      "loss": 0.458,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8034251928329468,
      "rewards/margins": 1.1669247150421143,
      "rewards/rejected": -2.9703497886657715,
      "step": 13290
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5230615447082246e-07,
      "logits/chosen": -1.0014846324920654,
      "logits/rejected": -0.8525576591491699,
      "logps/chosen": -443.6893005371094,
      "logps/rejected": -539.8489990234375,
      "loss": 0.5427,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9237266778945923,
      "rewards/margins": 1.1094491481781006,
      "rewards/rejected": -3.0331757068634033,
      "step": 13300
    },
    {
      "epoch": 0.87,
      "eval_logits/chosen": -1.0766102075576782,
      "eval_logits/rejected": -0.9451166391372681,
      "eval_logps/chosen": -422.79327392578125,
      "eval_logps/rejected": -511.8057861328125,
      "eval_loss": 0.4857572019100189,
      "eval_rewards/accuracies": 0.6710000038146973,
      "eval_rewards/chosen": -1.9078832864761353,
      "eval_rewards/margins": 1.09405517578125,
      "eval_rewards/rejected": -3.0019383430480957,
      "eval_runtime": 732.2689,
      "eval_samples_per_second": 2.731,
      "eval_steps_per_second": 1.366,
      "step": 13300
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.49812280830308e-07,
      "logits/chosen": -1.0418148040771484,
      "logits/rejected": -0.788641095161438,
      "logps/chosen": -402.3210144042969,
      "logps/rejected": -550.3643798828125,
      "loss": 0.3707,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.7728382349014282,
      "rewards/margins": 1.828993797302246,
      "rewards/rejected": -3.6018319129943848,
      "step": 13310
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.4733014527120457e-07,
      "logits/chosen": -0.8111721277236938,
      "logits/rejected": -0.937555193901062,
      "logps/chosen": -460.4014587402344,
      "logps/rejected": -551.224853515625,
      "loss": 0.5392,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.5798659324645996,
      "rewards/margins": 1.1371663808822632,
      "rewards/rejected": -3.7170321941375732,
      "step": 13320
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.4485976074154565e-07,
      "logits/chosen": -1.1172932386398315,
      "logits/rejected": -1.1888206005096436,
      "logps/chosen": -412.2958068847656,
      "logps/rejected": -470.85528564453125,
      "loss": 0.6788,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.032978057861328,
      "rewards/margins": 0.285163015127182,
      "rewards/rejected": -2.318140983581543,
      "step": 13330
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.4240114012806763e-07,
      "logits/chosen": -1.0329598188400269,
      "logits/rejected": -1.0528901815414429,
      "logps/chosen": -394.06512451171875,
      "logps/rejected": -464.07159423828125,
      "loss": 0.5461,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8626697063446045,
      "rewards/margins": 0.8707631826400757,
      "rewards/rejected": -2.7334327697753906,
      "step": 13340
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.399542962561399e-07,
      "logits/chosen": -1.0653612613677979,
      "logits/rejected": -0.863553524017334,
      "logps/chosen": -392.567138671875,
      "logps/rejected": -456.32061767578125,
      "loss": 0.4809,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6711117029190063,
      "rewards/margins": 1.1116281747817993,
      "rewards/rejected": -2.7827396392822266,
      "step": 13350
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3751924188969876e-07,
      "logits/chosen": -1.0322988033294678,
      "logits/rejected": -0.889003574848175,
      "logps/chosen": -443.11468505859375,
      "logps/rejected": -542.2542114257812,
      "loss": 0.484,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.913761854171753,
      "rewards/margins": 1.1897408962249756,
      "rewards/rejected": -3.1035027503967285,
      "step": 13360
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3509598973118024e-07,
      "logits/chosen": -1.3392784595489502,
      "logits/rejected": -1.1734321117401123,
      "logps/chosen": -397.5035095214844,
      "logps/rejected": -399.9672546386719,
      "loss": 0.6019,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7649815082550049,
      "rewards/margins": 0.652515709400177,
      "rewards/rejected": -2.417497158050537,
      "step": 13370
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.326845524214555e-07,
      "logits/chosen": -0.9857600927352905,
      "logits/rejected": -1.0095943212509155,
      "logps/chosen": -435.65728759765625,
      "logps/rejected": -429.5391540527344,
      "loss": 0.73,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.0060293674468994,
      "rewards/margins": 0.20006613433361053,
      "rewards/rejected": -2.2060954570770264,
      "step": 13380
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.3028494253976158e-07,
      "logits/chosen": -1.102888822555542,
      "logits/rejected": -0.930350124835968,
      "logps/chosen": -558.3133544921875,
      "logps/rejected": -571.853515625,
      "loss": 0.5706,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1704790592193604,
      "rewards/margins": 0.7812148928642273,
      "rewards/rejected": -2.9516937732696533,
      "step": 13390
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2789717260364026e-07,
      "logits/chosen": -1.1375192403793335,
      "logits/rejected": -0.9567796587944031,
      "logps/chosen": -323.5530700683594,
      "logps/rejected": -386.8634338378906,
      "loss": 0.5223,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.5844700336456299,
      "rewards/margins": 0.8503804206848145,
      "rewards/rejected": -2.4348502159118652,
      "step": 13400
    },
    {
      "epoch": 0.88,
      "eval_logits/chosen": -1.087154746055603,
      "eval_logits/rejected": -0.9559369087219238,
      "eval_logps/chosen": -422.0847473144531,
      "eval_logps/rejected": -508.4212951660156,
      "eval_loss": 0.4862857758998871,
      "eval_rewards/accuracies": 0.671999990940094,
      "eval_rewards/chosen": -1.9007982015609741,
      "eval_rewards/margins": 1.0672953128814697,
      "eval_rewards/rejected": -2.968093156814575,
      "eval_runtime": 733.4672,
      "eval_samples_per_second": 2.727,
      "eval_steps_per_second": 1.363,
      "step": 13400
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.255212550688682e-07,
      "logits/chosen": -1.1616137027740479,
      "logits/rejected": -1.485405683517456,
      "logps/chosen": -426.1238708496094,
      "logps/rejected": -621.4586791992188,
      "loss": 0.4524,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.081688165664673,
      "rewards/margins": 1.3114808797836304,
      "rewards/rejected": -3.393169403076172,
      "step": 13410
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2315720232939598e-07,
      "logits/chosen": -1.5970458984375,
      "logits/rejected": -1.1075289249420166,
      "logps/chosen": -440.81884765625,
      "logps/rejected": -485.1519470214844,
      "loss": 0.3706,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7870508432388306,
      "rewards/margins": 1.339354395866394,
      "rewards/rejected": -3.1264052391052246,
      "step": 13420
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2080502671727956e-07,
      "logits/chosen": -1.2229093313217163,
      "logits/rejected": -1.0283030271530151,
      "logps/chosen": -372.5098876953125,
      "logps/rejected": -454.7687072753906,
      "loss": 0.4706,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.524280309677124,
      "rewards/margins": 1.04086434841156,
      "rewards/rejected": -2.5651450157165527,
      "step": 13430
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1846474050262078e-07,
      "logits/chosen": -1.0370843410491943,
      "logits/rejected": -0.8305977582931519,
      "logps/chosen": -425.2176208496094,
      "logps/rejected": -456.01605224609375,
      "loss": 0.4756,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7865688800811768,
      "rewards/margins": 0.9659556150436401,
      "rewards/rejected": -2.7525246143341064,
      "step": 13440
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1613635589349756e-07,
      "logits/chosen": -0.8326600193977356,
      "logits/rejected": -0.7741021513938904,
      "logps/chosen": -381.1373291015625,
      "logps/rejected": -534.6483154296875,
      "loss": 0.4565,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8397401571273804,
      "rewards/margins": 1.259902834892273,
      "rewards/rejected": -3.0996429920196533,
      "step": 13450
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1381988503590578e-07,
      "logits/chosen": -0.7525622248649597,
      "logits/rejected": -0.8662757873535156,
      "logps/chosen": -411.94189453125,
      "logps/rejected": -541.89892578125,
      "loss": 0.3546,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9607007503509521,
      "rewards/margins": 1.361067771911621,
      "rewards/rejected": -3.3217685222625732,
      "step": 13460
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.11515340013691e-07,
      "logits/chosen": -1.2946085929870605,
      "logits/rejected": -1.1581755876541138,
      "logps/chosen": -438.44049072265625,
      "logps/rejected": -542.212158203125,
      "loss": 0.4053,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.072679042816162,
      "rewards/margins": 1.253519058227539,
      "rewards/rejected": -3.326197862625122,
      "step": 13470
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.092227328484897e-07,
      "logits/chosen": -0.8586756587028503,
      "logits/rejected": -0.8986477851867676,
      "logps/chosen": -372.24542236328125,
      "logps/rejected": -510.7012634277344,
      "loss": 0.5042,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6823673248291016,
      "rewards/margins": 1.0779831409454346,
      "rewards/rejected": -2.760350465774536,
      "step": 13480
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0694207549966345e-07,
      "logits/chosen": -0.8356062173843384,
      "logits/rejected": -0.9943629503250122,
      "logps/chosen": -416.49627685546875,
      "logps/rejected": -452.17205810546875,
      "loss": 0.6632,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.0758779048919678,
      "rewards/margins": 0.525452733039856,
      "rewards/rejected": -2.601330280303955,
      "step": 13490
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0467337986423864e-07,
      "logits/chosen": -1.269989252090454,
      "logits/rejected": -1.0888997316360474,
      "logps/chosen": -505.70941162109375,
      "logps/rejected": -569.6568603515625,
      "loss": 0.4808,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9998592138290405,
      "rewards/margins": 0.971601665019989,
      "rewards/rejected": -2.9714608192443848,
      "step": 13500
    },
    {
      "epoch": 0.88,
      "eval_logits/chosen": -1.0681487321853638,
      "eval_logits/rejected": -0.9375593066215515,
      "eval_logps/chosen": -425.8811950683594,
      "eval_logps/rejected": -514.4193115234375,
      "eval_loss": 0.4859347641468048,
      "eval_rewards/accuracies": 0.6735000014305115,
      "eval_rewards/chosen": -1.9387621879577637,
      "eval_rewards/margins": 1.0893113613128662,
      "eval_rewards/rejected": -3.028073787689209,
      "eval_runtime": 730.6979,
      "eval_samples_per_second": 2.737,
      "eval_steps_per_second": 1.369,
      "step": 13500
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0241665777684272e-07,
      "logits/chosen": -1.312229871749878,
      "logits/rejected": -1.1031113862991333,
      "logps/chosen": -432.3946228027344,
      "logps/rejected": -561.6215209960938,
      "loss": 0.3229,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.6410160064697266,
      "rewards/margins": 1.7284218072891235,
      "rewards/rejected": -3.3694376945495605,
      "step": 13510
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0017192100964366e-07,
      "logits/chosen": -0.85126131772995,
      "logits/rejected": -0.8515372276306152,
      "logps/chosen": -403.20086669921875,
      "logps/rejected": -506.77337646484375,
      "loss": 0.4693,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0324487686157227,
      "rewards/margins": 1.0145536661148071,
      "rewards/rejected": -3.0470023155212402,
      "step": 13520
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.9793918127228777e-07,
      "logits/chosen": -1.3386536836624146,
      "logits/rejected": -0.9312573671340942,
      "logps/chosen": -534.556640625,
      "logps/rejected": -611.8395385742188,
      "loss": 0.4614,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.1825718879699707,
      "rewards/margins": 1.3136504888534546,
      "rewards/rejected": -3.4962220191955566,
      "step": 13530
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.9571845021184005e-07,
      "logits/chosen": -0.8405078053474426,
      "logits/rejected": -0.8910934329032898,
      "logps/chosen": -434.1742248535156,
      "logps/rejected": -528.9740600585938,
      "loss": 0.5757,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.0318832397460938,
      "rewards/margins": 0.9068182706832886,
      "rewards/rejected": -2.938701629638672,
      "step": 13540
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.9350973941272027e-07,
      "logits/chosen": -1.212944746017456,
      "logits/rejected": -0.9825285077095032,
      "logps/chosen": -435.71917724609375,
      "logps/rejected": -512.9835205078125,
      "loss": 0.5183,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.319441318511963,
      "rewards/margins": 0.9776519536972046,
      "rewards/rejected": -3.297093152999878,
      "step": 13550
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.9131306039664676e-07,
      "logits/chosen": -0.9572928547859192,
      "logits/rejected": -0.8474523425102234,
      "logps/chosen": -381.08544921875,
      "logps/rejected": -526.7711791992188,
      "loss": 0.4356,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.8118137121200562,
      "rewards/margins": 1.213293433189392,
      "rewards/rejected": -3.0251073837280273,
      "step": 13560
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8912842462257358e-07,
      "logits/chosen": -1.0587059259414673,
      "logits/rejected": -0.9345768094062805,
      "logps/chosen": -402.04144287109375,
      "logps/rejected": -514.9813232421875,
      "loss": 0.4445,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8187220096588135,
      "rewards/margins": 1.2992680072784424,
      "rewards/rejected": -3.117990016937256,
      "step": 13570
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.869558434866303e-07,
      "logits/chosen": -1.0689226388931274,
      "logits/rejected": -1.2264662981033325,
      "logps/chosen": -382.54583740234375,
      "logps/rejected": -523.0657348632812,
      "loss": 0.5252,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.0145599842071533,
      "rewards/margins": 1.2249000072479248,
      "rewards/rejected": -3.239459991455078,
      "step": 13580
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.847953283220652e-07,
      "logits/chosen": -1.1595171689987183,
      "logits/rejected": -0.9138990640640259,
      "logps/chosen": -453.76806640625,
      "logps/rejected": -552.489990234375,
      "loss": 0.3843,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.9765832424163818,
      "rewards/margins": 1.7735812664031982,
      "rewards/rejected": -3.750164747238159,
      "step": 13590
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8264689039918265e-07,
      "logits/chosen": -1.06974196434021,
      "logits/rejected": -0.9910194277763367,
      "logps/chosen": -487.11968994140625,
      "logps/rejected": -554.9024658203125,
      "loss": 0.5138,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.272892951965332,
      "rewards/margins": 0.9520902633666992,
      "rewards/rejected": -3.2249832153320312,
      "step": 13600
    },
    {
      "epoch": 0.89,
      "eval_logits/chosen": -1.066765546798706,
      "eval_logits/rejected": -0.9360716938972473,
      "eval_logps/chosen": -430.43524169921875,
      "eval_logps/rejected": -518.9196166992188,
      "eval_loss": 0.48558491468429565,
      "eval_rewards/accuracies": 0.671500027179718,
      "eval_rewards/chosen": -1.9843029975891113,
      "eval_rewards/margins": 1.0887740850448608,
      "eval_rewards/rejected": -3.0730769634246826,
      "eval_runtime": 730.1997,
      "eval_samples_per_second": 2.739,
      "eval_steps_per_second": 1.369,
      "step": 13600
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8051054092528857e-07,
      "logits/chosen": -1.1157863140106201,
      "logits/rejected": -0.9228916168212891,
      "logps/chosen": -462.395263671875,
      "logps/rejected": -571.1812744140625,
      "loss": 0.4968,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.012938976287842,
      "rewards/margins": 1.185462236404419,
      "rewards/rejected": -3.1984007358551025,
      "step": 13610
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.783862910446271e-07,
      "logits/chosen": -0.8563364148139954,
      "logits/rejected": -0.8741576075553894,
      "logps/chosen": -356.21136474609375,
      "logps/rejected": -471.511474609375,
      "loss": 0.4717,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8335803747177124,
      "rewards/margins": 1.2461497783660889,
      "rewards/rejected": -3.079730272293091,
      "step": 13620
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.762741518383271e-07,
      "logits/chosen": -1.1820201873779297,
      "logits/rejected": -0.908908486366272,
      "logps/chosen": -425.9429626464844,
      "logps/rejected": -497.426513671875,
      "loss": 0.5404,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0411064624786377,
      "rewards/margins": 0.9914758801460266,
      "rewards/rejected": -3.0325820446014404,
      "step": 13630
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7417413432434082e-07,
      "logits/chosen": -1.0338904857635498,
      "logits/rejected": -0.9325830340385437,
      "logps/chosen": -451.5857849121094,
      "logps/rejected": -513.577880859375,
      "loss": 0.4921,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9892606735229492,
      "rewards/margins": 1.1475944519042969,
      "rewards/rejected": -3.136855363845825,
      "step": 13640
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7208624945738855e-07,
      "logits/chosen": -1.2731138467788696,
      "logits/rejected": -1.2202355861663818,
      "logps/chosen": -426.15582275390625,
      "logps/rejected": -516.65283203125,
      "loss": 0.5701,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0807149410247803,
      "rewards/margins": 0.8305963277816772,
      "rewards/rejected": -2.911311626434326,
      "step": 13650
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7001050812889995e-07,
      "logits/chosen": -1.3216979503631592,
      "logits/rejected": -1.127280592918396,
      "logps/chosen": -480.98974609375,
      "logps/rejected": -547.7494506835938,
      "loss": 0.5034,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2545692920684814,
      "rewards/margins": 1.0701066255569458,
      "rewards/rejected": -3.3246757984161377,
      "step": 13660
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.679469211669596e-07,
      "logits/chosen": -1.0741422176361084,
      "logits/rejected": -0.8863421678543091,
      "logps/chosen": -439.17437744140625,
      "logps/rejected": -493.56982421875,
      "loss": 0.4553,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1422924995422363,
      "rewards/margins": 1.0599985122680664,
      "rewards/rejected": -3.2022907733917236,
      "step": 13670
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6589549933624715e-07,
      "logits/chosen": -1.0650371313095093,
      "logits/rejected": -0.8889235258102417,
      "logps/chosen": -416.11181640625,
      "logps/rejected": -547.8284301757812,
      "loss": 0.3695,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7590478658676147,
      "rewards/margins": 1.8320204019546509,
      "rewards/rejected": -3.5910682678222656,
      "step": 13680
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.638562533379845e-07,
      "logits/chosen": -0.9663646817207336,
      "logits/rejected": -0.861709475517273,
      "logps/chosen": -444.4697265625,
      "logps/rejected": -477.4546813964844,
      "loss": 0.5241,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8876707553863525,
      "rewards/margins": 0.9525297284126282,
      "rewards/rejected": -2.840200424194336,
      "step": 13690
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6182919380987676e-07,
      "logits/chosen": -1.0882418155670166,
      "logits/rejected": -1.0266286134719849,
      "logps/chosen": -420.1298828125,
      "logps/rejected": -474.15777587890625,
      "loss": 0.5878,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.9441192150115967,
      "rewards/margins": 0.6940501928329468,
      "rewards/rejected": -2.638169765472412,
      "step": 13700
    },
    {
      "epoch": 0.9,
      "eval_logits/chosen": -1.0581167936325073,
      "eval_logits/rejected": -0.9280232191085815,
      "eval_logps/chosen": -436.266357421875,
      "eval_logps/rejected": -523.874267578125,
      "eval_loss": 0.48549070954322815,
      "eval_rewards/accuracies": 0.6694999933242798,
      "eval_rewards/chosen": -2.042614221572876,
      "eval_rewards/margins": 1.0800092220306396,
      "eval_rewards/rejected": -3.1226232051849365,
      "eval_runtime": 732.6369,
      "eval_samples_per_second": 2.73,
      "eval_steps_per_second": 1.365,
      "step": 13700
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.598143313260603e-07,
      "logits/chosen": -0.7987823486328125,
      "logits/rejected": -0.8568712472915649,
      "logps/chosen": -401.6100158691406,
      "logps/rejected": -476.331298828125,
      "loss": 0.5398,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.1355624198913574,
      "rewards/margins": 0.8673583269119263,
      "rewards/rejected": -3.0029211044311523,
      "step": 13710
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.5781167639704415e-07,
      "logits/chosen": -1.1982240676879883,
      "logits/rejected": -0.8309241533279419,
      "logps/chosen": -548.0106811523438,
      "logps/rejected": -538.3633422851562,
      "loss": 0.4387,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1584393978118896,
      "rewards/margins": 1.145960807800293,
      "rewards/rejected": -3.3044002056121826,
      "step": 13720
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.5582123946965787e-07,
      "logits/chosen": -0.946783185005188,
      "logits/rejected": -0.7460772395133972,
      "logps/chosen": -436.0660095214844,
      "logps/rejected": -565.3934326171875,
      "loss": 0.4354,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.016709566116333,
      "rewards/margins": 1.191794991493225,
      "rewards/rejected": -3.2085044384002686,
      "step": 13730
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.5384303092699504e-07,
      "logits/chosen": -1.15846848487854,
      "logits/rejected": -0.7557247281074524,
      "logps/chosen": -466.0997009277344,
      "logps/rejected": -632.2220458984375,
      "loss": 0.3254,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.7911508083343506,
      "rewards/margins": 1.678033471107483,
      "rewards/rejected": -3.469184160232544,
      "step": 13740
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.518770610883613e-07,
      "logits/chosen": -0.9040036201477051,
      "logits/rejected": -0.802951991558075,
      "logps/chosen": -455.0104064941406,
      "logps/rejected": -563.3392333984375,
      "loss": 0.3819,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.4047179222106934,
      "rewards/margins": 1.3774263858795166,
      "rewards/rejected": -3.782144069671631,
      "step": 13750
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4992334020921735e-07,
      "logits/chosen": -1.0886790752410889,
      "logits/rejected": -1.0631605386734009,
      "logps/chosen": -353.92230224609375,
      "logps/rejected": -463.3160705566406,
      "loss": 0.3949,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8094984292984009,
      "rewards/margins": 1.3900763988494873,
      "rewards/rejected": -3.1995749473571777,
      "step": 13760
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4798187848112905e-07,
      "logits/chosen": -1.0941932201385498,
      "logits/rejected": -0.7571912407875061,
      "logps/chosen": -474.4635314941406,
      "logps/rejected": -545.6170654296875,
      "loss": 0.5698,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.514927864074707,
      "rewards/margins": 1.1201101541519165,
      "rewards/rejected": -3.635037899017334,
      "step": 13770
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.460526860317113e-07,
      "logits/chosen": -1.2194105386734009,
      "logits/rejected": -1.0897470712661743,
      "logps/chosen": -366.4600524902344,
      "logps/rejected": -543.8425903320312,
      "loss": 0.4088,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9317538738250732,
      "rewards/margins": 1.4563566446304321,
      "rewards/rejected": -3.388110637664795,
      "step": 13780
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.441357729245771e-07,
      "logits/chosen": -1.2731847763061523,
      "logits/rejected": -0.9451271891593933,
      "logps/chosen": -504.37548828125,
      "logps/rejected": -548.6998901367188,
      "loss": 0.5107,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.5638363361358643,
      "rewards/margins": 1.0031163692474365,
      "rewards/rejected": -3.56695294380188,
      "step": 13790
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4223114915928482e-07,
      "logits/chosen": -0.6414790153503418,
      "logits/rejected": -0.7103398442268372,
      "logps/chosen": -432.17767333984375,
      "logps/rejected": -554.3941650390625,
      "loss": 0.4051,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0625510215759277,
      "rewards/margins": 1.1544933319091797,
      "rewards/rejected": -3.2170441150665283,
      "step": 13800
    },
    {
      "epoch": 0.9,
      "eval_logits/chosen": -1.058732509613037,
      "eval_logits/rejected": -0.9284118413925171,
      "eval_logps/chosen": -435.3294982910156,
      "eval_logps/rejected": -524.1821899414062,
      "eval_loss": 0.4853297770023346,
      "eval_rewards/accuracies": 0.6725000143051147,
      "eval_rewards/chosen": -2.033245325088501,
      "eval_rewards/margins": 1.0924570560455322,
      "eval_rewards/rejected": -3.125702142715454,
      "eval_runtime": 731.8825,
      "eval_samples_per_second": 2.733,
      "eval_steps_per_second": 1.366,
      "step": 13800
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.403388246712842e-07,
      "logits/chosen": -1.068618893623352,
      "logits/rejected": -0.9264750480651855,
      "logps/chosen": -354.43719482421875,
      "logps/rejected": -426.0357971191406,
      "loss": 0.5102,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.9018609523773193,
      "rewards/margins": 0.7875036597251892,
      "rewards/rejected": -2.6893649101257324,
      "step": 13810
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.3845880933186757e-07,
      "logits/chosen": -1.1902129650115967,
      "logits/rejected": -1.0742738246917725,
      "logps/chosen": -435.77813720703125,
      "logps/rejected": -479.9771423339844,
      "loss": 0.556,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.9703929424285889,
      "rewards/margins": 0.7843495607376099,
      "rewards/rejected": -2.7547426223754883,
      "step": 13820
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.3659111294811457e-07,
      "logits/chosen": -1.0660393238067627,
      "logits/rejected": -1.0473939180374146,
      "logps/chosen": -409.37060546875,
      "logps/rejected": -486.2066955566406,
      "loss": 0.5132,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.175274133682251,
      "rewards/margins": 0.947323203086853,
      "rewards/rejected": -3.1225974559783936,
      "step": 13830
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.347357452628459e-07,
      "logits/chosen": -1.465331792831421,
      "logits/rejected": -1.3321690559387207,
      "logps/chosen": -442.4307556152344,
      "logps/rejected": -534.9903564453125,
      "loss": 0.5425,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9713321924209595,
      "rewards/margins": 1.0006132125854492,
      "rewards/rejected": -2.971945285797119,
      "step": 13840
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.3289271595456732e-07,
      "logits/chosen": -1.0778005123138428,
      "logits/rejected": -0.7507137656211853,
      "logps/chosen": -442.45904541015625,
      "logps/rejected": -531.1512451171875,
      "loss": 0.4908,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.3812978267669678,
      "rewards/margins": 1.144858717918396,
      "rewards/rejected": -3.5261569023132324,
      "step": 13850
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.310620346374228e-07,
      "logits/chosen": -1.010206937789917,
      "logits/rejected": -0.8545116186141968,
      "logps/chosen": -430.58880615234375,
      "logps/rejected": -510.44842529296875,
      "loss": 0.4736,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9836689233779907,
      "rewards/margins": 1.1666280031204224,
      "rewards/rejected": -3.150296688079834,
      "step": 13860
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2924371086114274e-07,
      "logits/chosen": -1.1741849184036255,
      "logits/rejected": -0.7831594347953796,
      "logps/chosen": -452.38055419921875,
      "logps/rejected": -548.46728515625,
      "loss": 0.5199,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.1631908416748047,
      "rewards/margins": 1.0211350917816162,
      "rewards/rejected": -3.184326171875,
      "step": 13870
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.274377541109953e-07,
      "logits/chosen": -0.9694555401802063,
      "logits/rejected": -1.0235732793807983,
      "logps/chosen": -373.6258239746094,
      "logps/rejected": -528.3242797851562,
      "loss": 0.5527,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.0927529335021973,
      "rewards/margins": 0.7677472233772278,
      "rewards/rejected": -2.8605000972747803,
      "step": 13880
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2564417380773435e-07,
      "logits/chosen": -0.822708785533905,
      "logits/rejected": -0.5482523441314697,
      "logps/chosen": -407.5481872558594,
      "logps/rejected": -551.6847534179688,
      "loss": 0.4509,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.2905116081237793,
      "rewards/margins": 1.2165449857711792,
      "rewards/rejected": -3.507056713104248,
      "step": 13890
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2386297930755436e-07,
      "logits/chosen": -1.2471745014190674,
      "logits/rejected": -1.1345198154449463,
      "logps/chosen": -508.18096923828125,
      "logps/rejected": -608.7763671875,
      "loss": 0.5562,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.602952003479004,
      "rewards/margins": 1.0542181730270386,
      "rewards/rejected": -3.657170057296753,
      "step": 13900
    },
    {
      "epoch": 0.91,
      "eval_logits/chosen": -1.0553574562072754,
      "eval_logits/rejected": -0.9256516098976135,
      "eval_logps/chosen": -433.4284362792969,
      "eval_logps/rejected": -521.5325927734375,
      "eval_loss": 0.48544591665267944,
      "eval_rewards/accuracies": 0.6725000143051147,
      "eval_rewards/chosen": -2.014234781265259,
      "eval_rewards/margins": 1.0849719047546387,
      "eval_rewards/rejected": -3.0992064476013184,
      "eval_runtime": 734.9627,
      "eval_samples_per_second": 2.721,
      "eval_steps_per_second": 1.361,
      "step": 13900
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.220941799020378e-07,
      "logits/chosen": -0.9485194087028503,
      "logits/rejected": -0.8109304308891296,
      "logps/chosen": -417.08258056640625,
      "logps/rejected": -526.6878662109375,
      "loss": 0.4801,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9800148010253906,
      "rewards/margins": 1.319581389427185,
      "rewards/rejected": -3.2995963096618652,
      "step": 13910
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2033778481810975e-07,
      "logits/chosen": -1.0876718759536743,
      "logits/rejected": -1.0527876615524292,
      "logps/chosen": -403.35272216796875,
      "logps/rejected": -484.29534912109375,
      "loss": 0.5012,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8335920572280884,
      "rewards/margins": 1.182887315750122,
      "rewards/rejected": -3.0164794921875,
      "step": 13920
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1859380321798591e-07,
      "logits/chosen": -1.12349271774292,
      "logits/rejected": -1.3553979396820068,
      "logps/chosen": -406.2502746582031,
      "logps/rejected": -502.95263671875,
      "loss": 0.526,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.0368897914886475,
      "rewards/margins": 0.8195746541023254,
      "rewards/rejected": -2.8564646244049072,
      "step": 13930
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1686224419912989e-07,
      "logits/chosen": -1.007314920425415,
      "logits/rejected": -0.8068926930427551,
      "logps/chosen": -491.4189453125,
      "logps/rejected": -588.8319091796875,
      "loss": 0.5216,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.385463237762451,
      "rewards/margins": 1.267038106918335,
      "rewards/rejected": -3.6525015830993652,
      "step": 13940
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1514311679420104e-07,
      "logits/chosen": -0.6763644218444824,
      "logits/rejected": -0.7290538549423218,
      "logps/chosen": -371.91876220703125,
      "logps/rejected": -528.335693359375,
      "loss": 0.5789,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1677021980285645,
      "rewards/margins": 1.0059720277786255,
      "rewards/rejected": -3.1736743450164795,
      "step": 13950
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1343642997101029e-07,
      "logits/chosen": -1.1263518333435059,
      "logits/rejected": -0.943850040435791,
      "logps/chosen": -407.45391845703125,
      "logps/rejected": -496.84991455078125,
      "loss": 0.497,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0510058403015137,
      "rewards/margins": 1.0261781215667725,
      "rewards/rejected": -3.077183723449707,
      "step": 13960
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1174219263247188e-07,
      "logits/chosen": -0.659032940864563,
      "logits/rejected": -0.6590057015419006,
      "logps/chosen": -381.11053466796875,
      "logps/rejected": -466.47900390625,
      "loss": 0.4919,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8229297399520874,
      "rewards/margins": 1.028996229171753,
      "rewards/rejected": -2.85192608833313,
      "step": 13970
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1006041361655839e-07,
      "logits/chosen": -1.2615095376968384,
      "logits/rejected": -0.8521437644958496,
      "logps/chosen": -404.3982238769531,
      "logps/rejected": -455.39410400390625,
      "loss": 0.526,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.9596093893051147,
      "rewards/margins": 0.9093338847160339,
      "rewards/rejected": -2.868943452835083,
      "step": 13980
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0839110169625189e-07,
      "logits/chosen": -0.8986303210258484,
      "logits/rejected": -0.9911459684371948,
      "logps/chosen": -447.0167541503906,
      "logps/rejected": -582.6688842773438,
      "loss": 0.4752,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.4017446041107178,
      "rewards/margins": 1.4300748109817505,
      "rewards/rejected": -3.831820011138916,
      "step": 13990
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.06734265579502e-07,
      "logits/chosen": -1.0780047178268433,
      "logits/rejected": -0.7636250257492065,
      "logps/chosen": -478.39306640625,
      "logps/rejected": -528.6465454101562,
      "loss": 0.4542,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1687636375427246,
      "rewards/margins": 1.1675869226455688,
      "rewards/rejected": -3.336350202560425,
      "step": 14000
    },
    {
      "epoch": 0.92,
      "eval_logits/chosen": -1.0565329790115356,
      "eval_logits/rejected": -0.9270216226577759,
      "eval_logps/chosen": -434.0428466796875,
      "eval_logps/rejected": -521.0421142578125,
      "eval_loss": 0.4857202172279358,
      "eval_rewards/accuracies": 0.671500027179718,
      "eval_rewards/chosen": -2.020378828048706,
      "eval_rewards/margins": 1.073923110961914,
      "eval_rewards/rejected": -3.094302177429199,
      "eval_runtime": 733.211,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 14000
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.050899139091771e-07,
      "logits/chosen": -1.2847198247909546,
      "logits/rejected": -0.883298397064209,
      "logps/chosen": -477.986083984375,
      "logps/rejected": -561.3118896484375,
      "loss": 0.3692,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9935863018035889,
      "rewards/margins": 1.298438549041748,
      "rewards/rejected": -3.292024612426758,
      "step": 14010
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0345805526302072e-07,
      "logits/chosen": -1.177329659461975,
      "logits/rejected": -1.022749662399292,
      "logps/chosen": -404.83477783203125,
      "logps/rejected": -488.59747314453125,
      "loss": 0.4919,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.036724328994751,
      "rewards/margins": 0.9100037813186646,
      "rewards/rejected": -2.946727991104126,
      "step": 14020
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0183869815360764e-07,
      "logits/chosen": -1.1223341226577759,
      "logits/rejected": -1.1892069578170776,
      "logps/chosen": -397.103271484375,
      "logps/rejected": -512.4364013671875,
      "loss": 0.5226,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.0518977642059326,
      "rewards/margins": 0.8269307017326355,
      "rewards/rejected": -2.878828525543213,
      "step": 14030
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0023185102829763e-07,
      "logits/chosen": -0.8264619708061218,
      "logits/rejected": -0.9047967791557312,
      "logps/chosen": -467.5877380371094,
      "logps/rejected": -589.7196044921875,
      "loss": 0.4809,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.367655038833618,
      "rewards/margins": 1.179471731185913,
      "rewards/rejected": -3.5471267700195312,
      "step": 14040
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.863752226919182e-08,
      "logits/chosen": -0.9276908040046692,
      "logits/rejected": -0.7621574401855469,
      "logps/chosen": -425.6780700683594,
      "logps/rejected": -497.79925537109375,
      "loss": 0.4506,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8794587850570679,
      "rewards/margins": 1.376997709274292,
      "rewards/rejected": -3.2564563751220703,
      "step": 14050
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.705572019309107e-08,
      "logits/chosen": -1.0182292461395264,
      "logits/rejected": -0.8643664121627808,
      "logps/chosen": -485.4905700683594,
      "logps/rejected": -562.658935546875,
      "loss": 0.5055,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.1746318340301514,
      "rewards/margins": 1.0865226984024048,
      "rewards/rejected": -3.2611546516418457,
      "step": 14060
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.548645305144849e-08,
      "logits/chosen": -1.2598340511322021,
      "logits/rejected": -1.004939079284668,
      "logps/chosen": -353.81964111328125,
      "logps/rejected": -457.3272399902344,
      "loss": 0.5245,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.8032443523406982,
      "rewards/margins": 0.9953397512435913,
      "rewards/rejected": -2.798584222793579,
      "step": 14070
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.392972903033149e-08,
      "logits/chosen": -0.8470398187637329,
      "logits/rejected": -1.0979256629943848,
      "logps/chosen": -397.806884765625,
      "logps/rejected": -458.0087890625,
      "loss": 0.5621,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.7467561960220337,
      "rewards/margins": 0.677453339099884,
      "rewards/rejected": -2.4242093563079834,
      "step": 14080
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.238555625037449e-08,
      "logits/chosen": -0.8955826759338379,
      "logits/rejected": -0.8132731318473816,
      "logps/chosen": -405.64996337890625,
      "logps/rejected": -455.35821533203125,
      "loss": 0.535,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.1494174003601074,
      "rewards/margins": 0.7939003109931946,
      "rewards/rejected": -2.9433178901672363,
      "step": 14090
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.085394276673903e-08,
      "logits/chosen": -1.0234901905059814,
      "logits/rejected": -1.0090752840042114,
      "logps/chosen": -471.3775329589844,
      "logps/rejected": -562.6046752929688,
      "loss": 0.4657,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.032114267349243,
      "rewards/margins": 1.0567667484283447,
      "rewards/rejected": -3.088881015777588,
      "step": 14100
    },
    {
      "epoch": 0.92,
      "eval_logits/chosen": -1.0566540956497192,
      "eval_logits/rejected": -0.92729651927948,
      "eval_logps/chosen": -432.3822021484375,
      "eval_logps/rejected": -519.443115234375,
      "eval_loss": 0.48553240299224854,
      "eval_rewards/accuracies": 0.6694999933242798,
      "eval_rewards/chosen": -2.003772020339966,
      "eval_rewards/margins": 1.074540376663208,
      "eval_rewards/rejected": -3.078312397003174,
      "eval_runtime": 733.883,
      "eval_samples_per_second": 2.725,
      "eval_steps_per_second": 1.363,
      "step": 14100
    },
    {
      "epoch": 0.92,
      "learning_rate": 8.933489656907157e-08,
      "logits/chosen": -0.9661086201667786,
      "logits/rejected": -1.0341233015060425,
      "logps/chosen": -431.1689453125,
      "logps/rejected": -524.0797119140625,
      "loss": 0.6029,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.1383349895477295,
      "rewards/margins": 0.6797035932540894,
      "rewards/rejected": -2.8180384635925293,
      "step": 14110
    },
    {
      "epoch": 0.92,
      "learning_rate": 8.782842558146127e-08,
      "logits/chosen": -0.9491564631462097,
      "logits/rejected": -0.8661885261535645,
      "logps/chosen": -321.4067687988281,
      "logps/rejected": -459.0927734375,
      "loss": 0.3766,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5952284336090088,
      "rewards/margins": 1.3939249515533447,
      "rewards/rejected": -2.9891533851623535,
      "step": 14120
    },
    {
      "epoch": 0.92,
      "learning_rate": 8.633453766239836e-08,
      "logits/chosen": -1.1976830959320068,
      "logits/rejected": -1.1062679290771484,
      "logps/chosen": -402.8575134277344,
      "logps/rejected": -448.3551330566406,
      "loss": 0.5217,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6773631572723389,
      "rewards/margins": 0.7735291123390198,
      "rewards/rejected": -2.450892448425293,
      "step": 14130
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.485324060473448e-08,
      "logits/chosen": -1.0934231281280518,
      "logits/rejected": -0.9173671007156372,
      "logps/chosen": -457.7579650878906,
      "logps/rejected": -525.3034057617188,
      "loss": 0.5423,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.1747584342956543,
      "rewards/margins": 0.7993627190589905,
      "rewards/rejected": -2.974121332168579,
      "step": 14140
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.338454213564052e-08,
      "logits/chosen": -1.0765687227249146,
      "logits/rejected": -0.832840085029602,
      "logps/chosen": -443.53997802734375,
      "logps/rejected": -558.3150634765625,
      "loss": 0.4001,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1264307498931885,
      "rewards/margins": 1.3196923732757568,
      "rewards/rejected": -3.4461231231689453,
      "step": 14150
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.192844991656679e-08,
      "logits/chosen": -0.9876272082328796,
      "logits/rejected": -0.7183239459991455,
      "logps/chosen": -453.659423828125,
      "logps/rejected": -496.2523498535156,
      "loss": 0.5436,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1494274139404297,
      "rewards/margins": 0.7900088429450989,
      "rewards/rejected": -2.939436197280884,
      "step": 14160
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.048497154320434e-08,
      "logits/chosen": -1.0411174297332764,
      "logits/rejected": -1.1419099569320679,
      "logps/chosen": -332.55426025390625,
      "logps/rejected": -408.7363586425781,
      "loss": 0.5648,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0300471782684326,
      "rewards/margins": 0.6708464622497559,
      "rewards/rejected": -2.7008938789367676,
      "step": 14170
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.905411454544265e-08,
      "logits/chosen": -1.0737169981002808,
      "logits/rejected": -1.0451024770736694,
      "logps/chosen": -464.43511962890625,
      "logps/rejected": -555.2205200195312,
      "loss": 0.5399,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.279308795928955,
      "rewards/margins": 0.8604310154914856,
      "rewards/rejected": -3.139739513397217,
      "step": 14180
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.763588638733332e-08,
      "logits/chosen": -1.0381790399551392,
      "logits/rejected": -1.0612866878509521,
      "logps/chosen": -481.591064453125,
      "logps/rejected": -569.1793823242188,
      "loss": 0.4937,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.191709041595459,
      "rewards/margins": 1.0741246938705444,
      "rewards/rejected": -3.265834093093872,
      "step": 14190
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.623029446704899e-08,
      "logits/chosen": -1.2095810174942017,
      "logits/rejected": -1.2709791660308838,
      "logps/chosen": -510.81903076171875,
      "logps/rejected": -631.96240234375,
      "loss": 0.3963,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.993775725364685,
      "rewards/margins": 1.5803800821304321,
      "rewards/rejected": -3.574155330657959,
      "step": 14200
    },
    {
      "epoch": 0.93,
      "eval_logits/chosen": -1.0539846420288086,
      "eval_logits/rejected": -0.9247413277626038,
      "eval_logps/chosen": -430.5838928222656,
      "eval_logps/rejected": -518.67236328125,
      "eval_loss": 0.4853227138519287,
      "eval_rewards/accuracies": 0.6710000038146973,
      "eval_rewards/chosen": -1.9857898950576782,
      "eval_rewards/margins": 1.0848145484924316,
      "eval_rewards/rejected": -3.0706045627593994,
      "eval_runtime": 733.9148,
      "eval_samples_per_second": 2.725,
      "eval_steps_per_second": 1.363,
      "step": 14200
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.483734611684557e-08,
      "logits/chosen": -0.8158127069473267,
      "logits/rejected": -0.6502638459205627,
      "logps/chosen": -426.8959045410156,
      "logps/rejected": -492.63525390625,
      "loss": 0.454,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.700100302696228,
      "rewards/margins": 1.1990031003952026,
      "rewards/rejected": -2.8991036415100098,
      "step": 14210
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.345704860302366e-08,
      "logits/chosen": -1.4092390537261963,
      "logits/rejected": -1.051461935043335,
      "logps/chosen": -456.6201171875,
      "logps/rejected": -592.5618896484375,
      "loss": 0.3809,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.083674430847168,
      "rewards/margins": 1.3662012815475464,
      "rewards/rejected": -3.449875593185425,
      "step": 14220
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.208940912589224e-08,
      "logits/chosen": -1.0438339710235596,
      "logits/rejected": -0.8418275117874146,
      "logps/chosen": -439.68048095703125,
      "logps/rejected": -531.1444091796875,
      "loss": 0.4848,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.300941228866577,
      "rewards/margins": 1.2884247303009033,
      "rewards/rejected": -3.5893661975860596,
      "step": 14230
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.073443481972753e-08,
      "logits/chosen": -0.9687435030937195,
      "logits/rejected": -0.8915712237358093,
      "logps/chosen": -404.16595458984375,
      "logps/rejected": -524.464599609375,
      "loss": 0.5968,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.2001020908355713,
      "rewards/margins": 1.007969856262207,
      "rewards/rejected": -3.2080719470977783,
      "step": 14240
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.939213275274027e-08,
      "logits/chosen": -1.1182448863983154,
      "logits/rejected": -1.1295720338821411,
      "logps/chosen": -431.3018493652344,
      "logps/rejected": -505.3377990722656,
      "loss": 0.4954,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9204801321029663,
      "rewards/margins": 0.9363868832588196,
      "rewards/rejected": -2.8568673133850098,
      "step": 14250
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.806250992703461e-08,
      "logits/chosen": -0.9891894459724426,
      "logits/rejected": -0.9061968922615051,
      "logps/chosen": -407.3405456542969,
      "logps/rejected": -472.3490295410156,
      "loss": 0.5011,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9064586162567139,
      "rewards/margins": 0.9121868014335632,
      "rewards/rejected": -2.81864595413208,
      "step": 14260
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.674557327857572e-08,
      "logits/chosen": -1.1973614692687988,
      "logits/rejected": -1.063331961631775,
      "logps/chosen": -454.7339782714844,
      "logps/rejected": -570.90673828125,
      "loss": 0.4754,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.0046329498291016,
      "rewards/margins": 1.252607822418213,
      "rewards/rejected": -3.2572407722473145,
      "step": 14270
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.544132967714917e-08,
      "logits/chosen": -0.838614821434021,
      "logits/rejected": -0.762345016002655,
      "logps/chosen": -484.97625732421875,
      "logps/rejected": -575.3137817382812,
      "loss": 0.6345,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.3811631202697754,
      "rewards/margins": 1.0404995679855347,
      "rewards/rejected": -3.4216628074645996,
      "step": 14280
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.414978592632932e-08,
      "logits/chosen": -0.9945710301399231,
      "logits/rejected": -0.9295830726623535,
      "logps/chosen": -472.0386657714844,
      "logps/rejected": -528.8554077148438,
      "loss": 0.4747,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.109393835067749,
      "rewards/margins": 1.032721757888794,
      "rewards/rejected": -3.142115592956543,
      "step": 14290
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.287094876344046e-08,
      "logits/chosen": -1.283722162246704,
      "logits/rejected": -1.1530678272247314,
      "logps/chosen": -319.38885498046875,
      "logps/rejected": -428.88323974609375,
      "loss": 0.4414,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.469685673713684,
      "rewards/margins": 1.0285536050796509,
      "rewards/rejected": -2.498239278793335,
      "step": 14300
    },
    {
      "epoch": 0.94,
      "eval_logits/chosen": -1.0557368993759155,
      "eval_logits/rejected": -0.9262323975563049,
      "eval_logps/chosen": -431.466552734375,
      "eval_logps/rejected": -519.5145263671875,
      "eval_loss": 0.4854941666126251,
      "eval_rewards/accuracies": 0.671500027179718,
      "eval_rewards/chosen": -1.994615912437439,
      "eval_rewards/margins": 1.0844100713729858,
      "eval_rewards/rejected": -3.079025983810425,
      "eval_runtime": 734.1054,
      "eval_samples_per_second": 2.724,
      "eval_steps_per_second": 1.362,
      "step": 14300
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.160482485952413e-08,
      "logits/chosen": -1.2454659938812256,
      "logits/rejected": -1.209099531173706,
      "logps/chosen": -450.26055908203125,
      "logps/rejected": -511.4938049316406,
      "loss": 0.5019,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.1381077766418457,
      "rewards/margins": 0.9302912950515747,
      "rewards/rejected": -3.068398952484131,
      "step": 14310
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.035142081930234e-08,
      "logits/chosen": -1.0615335702896118,
      "logits/rejected": -0.8751511573791504,
      "logps/chosen": -484.47412109375,
      "logps/rejected": -493.105712890625,
      "loss": 0.4779,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2167773246765137,
      "rewards/margins": 0.8564853668212891,
      "rewards/rejected": -3.0732626914978027,
      "step": 14320
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.911074318114496e-08,
      "logits/chosen": -0.9687420129776001,
      "logits/rejected": -0.8334226608276367,
      "logps/chosen": -415.8157653808594,
      "logps/rejected": -576.0595703125,
      "loss": 0.4533,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.122276544570923,
      "rewards/margins": 1.1811892986297607,
      "rewards/rejected": -3.3034660816192627,
      "step": 14330
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.788279841703381e-08,
      "logits/chosen": -1.2568345069885254,
      "logits/rejected": -0.9866706728935242,
      "logps/chosen": -372.8052062988281,
      "logps/rejected": -492.4381408691406,
      "loss": 0.4512,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.888956069946289,
      "rewards/margins": 1.2730000019073486,
      "rewards/rejected": -3.1619560718536377,
      "step": 14340
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.66675929325311e-08,
      "logits/chosen": -1.1829434633255005,
      "logits/rejected": -0.9463958740234375,
      "logps/chosen": -423.58984375,
      "logps/rejected": -519.6142578125,
      "loss": 0.5006,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.0292537212371826,
      "rewards/margins": 1.0737316608428955,
      "rewards/rejected": -3.1029858589172363,
      "step": 14350
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.546513306674301e-08,
      "logits/chosen": -0.8857857584953308,
      "logits/rejected": -0.7765743136405945,
      "logps/chosen": -510.887939453125,
      "logps/rejected": -541.4619140625,
      "loss": 0.555,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.3237123489379883,
      "rewards/margins": 1.0565690994262695,
      "rewards/rejected": -3.380281925201416,
      "step": 14360
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.4275425092290004e-08,
      "logits/chosen": -1.5319944620132446,
      "logits/rejected": -1.317714810371399,
      "logps/chosen": -457.83807373046875,
      "logps/rejected": -551.9368896484375,
      "loss": 0.4606,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.0304198265075684,
      "rewards/margins": 1.1254143714904785,
      "rewards/rejected": -3.155834197998047,
      "step": 14370
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.309847521527078e-08,
      "logits/chosen": -0.78449946641922,
      "logits/rejected": -0.6805815100669861,
      "logps/chosen": -494.6311950683594,
      "logps/rejected": -557.3088989257812,
      "loss": 0.4461,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.100578784942627,
      "rewards/margins": 1.0560588836669922,
      "rewards/rejected": -3.156637668609619,
      "step": 14380
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.1934289575233385e-08,
      "logits/chosen": -0.8264106512069702,
      "logits/rejected": -0.606268584728241,
      "logps/chosen": -445.46697998046875,
      "logps/rejected": -517.4832763671875,
      "loss": 0.4771,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0593760013580322,
      "rewards/margins": 1.0638360977172852,
      "rewards/rejected": -3.1232123374938965,
      "step": 14390
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.078287424513994e-08,
      "logits/chosen": -1.2722312211990356,
      "logits/rejected": -1.099533200263977,
      "logps/chosen": -480.6007385253906,
      "logps/rejected": -511.64013671875,
      "loss": 0.5011,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.10552978515625,
      "rewards/margins": 1.0586974620819092,
      "rewards/rejected": -3.164227247238159,
      "step": 14400
    },
    {
      "epoch": 0.94,
      "eval_logits/chosen": -1.0528078079223633,
      "eval_logits/rejected": -0.9236515164375305,
      "eval_logps/chosen": -431.91925048828125,
      "eval_logps/rejected": -520.1353759765625,
      "eval_loss": 0.48540934920310974,
      "eval_rewards/accuracies": 0.6725000143051147,
      "eval_rewards/chosen": -1.999143362045288,
      "eval_rewards/margins": 1.0860916376113892,
      "eval_rewards/rejected": -3.085235118865967,
      "eval_runtime": 732.8713,
      "eval_samples_per_second": 2.729,
      "eval_steps_per_second": 1.364,
      "step": 14400
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.964423523133671e-08,
      "logits/chosen": -1.3805128335952759,
      "logits/rejected": -0.9356088638305664,
      "logps/chosen": -385.4630126953125,
      "logps/rejected": -447.2681579589844,
      "loss": 0.4266,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.677315354347229,
      "rewards/margins": 0.9615432024002075,
      "rewards/rejected": -2.6388583183288574,
      "step": 14410
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.8518378473522976e-08,
      "logits/chosen": -1.160941481590271,
      "logits/rejected": -0.9066068530082703,
      "logps/chosen": -430.5773010253906,
      "logps/rejected": -544.8810424804688,
      "loss": 0.5317,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.8006746768951416,
      "rewards/margins": 1.210309386253357,
      "rewards/rejected": -3.010983943939209,
      "step": 14420
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.7405309844718584e-08,
      "logits/chosen": -1.129230260848999,
      "logits/rejected": -0.9649642109870911,
      "logps/chosen": -415.31048583984375,
      "logps/rejected": -553.1641845703125,
      "loss": 0.4613,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.228114366531372,
      "rewards/margins": 1.2865746021270752,
      "rewards/rejected": -3.5146892070770264,
      "step": 14430
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.630503515123508e-08,
      "logits/chosen": -1.2513015270233154,
      "logits/rejected": -0.9588994979858398,
      "logps/chosen": -391.487060546875,
      "logps/rejected": -449.8251953125,
      "loss": 0.4846,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.90621018409729,
      "rewards/margins": 1.0820457935333252,
      "rewards/rejected": -2.9882559776306152,
      "step": 14440
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.5217560132644056e-08,
      "logits/chosen": -0.9577115774154663,
      "logits/rejected": -0.7936094403266907,
      "logps/chosen": -338.69146728515625,
      "logps/rejected": -458.49383544921875,
      "loss": 0.5356,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8807188272476196,
      "rewards/margins": 1.0140142440795898,
      "rewards/rejected": -2.89473295211792,
      "step": 14450
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.41428904617483e-08,
      "logits/chosen": -1.065185308456421,
      "logits/rejected": -1.139467716217041,
      "logps/chosen": -379.53497314453125,
      "logps/rejected": -466.9539489746094,
      "loss": 0.5526,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.0242626667022705,
      "rewards/margins": 0.8275849223136902,
      "rewards/rejected": -2.8518476486206055,
      "step": 14460
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.3081031744550696e-08,
      "logits/chosen": -1.1921031475067139,
      "logits/rejected": -1.2130444049835205,
      "logps/chosen": -435.697265625,
      "logps/rejected": -533.8173828125,
      "loss": 0.417,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8255189657211304,
      "rewards/margins": 1.2044792175292969,
      "rewards/rejected": -3.029998302459717,
      "step": 14470
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.2031989520227025e-08,
      "logits/chosen": -1.0036526918411255,
      "logits/rejected": -0.9116290807723999,
      "logps/chosen": -423.17938232421875,
      "logps/rejected": -516.8751831054688,
      "loss": 0.3985,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9486496448516846,
      "rewards/margins": 1.1558328866958618,
      "rewards/rejected": -3.104482650756836,
      "step": 14480
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.099576926109461e-08,
      "logits/chosen": -1.3175681829452515,
      "logits/rejected": -0.9569937586784363,
      "logps/chosen": -435.8815002441406,
      "logps/rejected": -468.4258728027344,
      "loss": 0.5283,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.9609966278076172,
      "rewards/margins": 1.116987943649292,
      "rewards/rejected": -3.077984571456909,
      "step": 14490
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.997237637258705e-08,
      "logits/chosen": -1.1571390628814697,
      "logits/rejected": -0.9494622349739075,
      "logps/chosen": -518.707763671875,
      "logps/rejected": -591.4268798828125,
      "loss": 0.4677,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.0145442485809326,
      "rewards/margins": 1.1606231927871704,
      "rewards/rejected": -3.1751675605773926,
      "step": 14500
    },
    {
      "epoch": 0.95,
      "eval_logits/chosen": -1.0542787313461304,
      "eval_logits/rejected": -0.924854040145874,
      "eval_logps/chosen": -432.1260986328125,
      "eval_logps/rejected": -520.5852661132812,
      "eval_loss": 0.48530498147010803,
      "eval_rewards/accuracies": 0.671500027179718,
      "eval_rewards/chosen": -2.001211643218994,
      "eval_rewards/margins": 1.08852219581604,
      "eval_rewards/rejected": -3.089733839035034,
      "eval_runtime": 734.0536,
      "eval_samples_per_second": 2.725,
      "eval_steps_per_second": 1.362,
      "step": 14500
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.8961816193222035e-08,
      "logits/chosen": -1.1297836303710938,
      "logits/rejected": -0.8937932252883911,
      "logps/chosen": -470.29498291015625,
      "logps/rejected": -474.1014709472656,
      "loss": 0.6551,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.344447374343872,
      "rewards/margins": 0.6113969087600708,
      "rewards/rejected": -2.9558444023132324,
      "step": 14510
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.79640939945769e-08,
      "logits/chosen": -1.0833925008773804,
      "logits/rejected": -0.9126625061035156,
      "logps/chosen": -436.407958984375,
      "logps/rejected": -439.82403564453125,
      "loss": 0.4772,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.5018702745437622,
      "rewards/margins": 0.8968018293380737,
      "rewards/rejected": -2.398672103881836,
      "step": 14520
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.697921498125895e-08,
      "logits/chosen": -0.9268239140510559,
      "logits/rejected": -1.0825777053833008,
      "logps/chosen": -411.1507873535156,
      "logps/rejected": -523.050537109375,
      "loss": 0.5322,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.0119776725769043,
      "rewards/margins": 1.060199499130249,
      "rewards/rejected": -3.0721774101257324,
      "step": 14530
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.6007184290880456e-08,
      "logits/chosen": -1.1957523822784424,
      "logits/rejected": -1.0776623487472534,
      "logps/chosen": -448.6844177246094,
      "logps/rejected": -524.9656982421875,
      "loss": 0.6034,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.367114305496216,
      "rewards/margins": 0.9294492602348328,
      "rewards/rejected": -3.2965633869171143,
      "step": 14540
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.504800699402872e-08,
      "logits/chosen": -1.2815639972686768,
      "logits/rejected": -1.082935094833374,
      "logps/chosen": -589.32470703125,
      "logps/rejected": -599.1885986328125,
      "loss": 0.6407,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.4367051124572754,
      "rewards/margins": 0.8323081731796265,
      "rewards/rejected": -3.2690131664276123,
      "step": 14550
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.4101688094242967e-08,
      "logits/chosen": -1.0604369640350342,
      "logits/rejected": -0.8951923251152039,
      "logps/chosen": -520.3701782226562,
      "logps/rejected": -614.2811279296875,
      "loss": 0.5229,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.508091926574707,
      "rewards/margins": 1.1908717155456543,
      "rewards/rejected": -3.6989638805389404,
      "step": 14560
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.3168232527985564e-08,
      "logits/chosen": -0.7122337222099304,
      "logits/rejected": -0.692426860332489,
      "logps/chosen": -414.7545471191406,
      "logps/rejected": -489.59112548828125,
      "loss": 0.487,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.731359839439392,
      "rewards/margins": 1.2958645820617676,
      "rewards/rejected": -3.02722430229187,
      "step": 14570
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.224764516461892e-08,
      "logits/chosen": -1.1161458492279053,
      "logits/rejected": -0.8429574966430664,
      "logps/chosen": -435.79296875,
      "logps/rejected": -527.206298828125,
      "loss": 0.4533,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.807941198348999,
      "rewards/margins": 1.221619963645935,
      "rewards/rejected": -3.0295608043670654,
      "step": 14580
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.133993080637665e-08,
      "logits/chosen": -1.1565264463424683,
      "logits/rejected": -0.911404013633728,
      "logps/chosen": -393.412841796875,
      "logps/rejected": -508.4385681152344,
      "loss": 0.478,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9053192138671875,
      "rewards/margins": 1.2743666172027588,
      "rewards/rejected": -3.1796860694885254,
      "step": 14590
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.0445094188342186e-08,
      "logits/chosen": -0.5498150587081909,
      "logits/rejected": -0.4293433725833893,
      "logps/chosen": -443.096923828125,
      "logps/rejected": -475.79681396484375,
      "loss": 0.4234,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8962033987045288,
      "rewards/margins": 1.1296409368515015,
      "rewards/rejected": -3.025844097137451,
      "step": 14600
    },
    {
      "epoch": 0.96,
      "eval_logits/chosen": -1.05791175365448,
      "eval_logits/rejected": -0.9282616376876831,
      "eval_logps/chosen": -432.10369873046875,
      "eval_logps/rejected": -520.2671508789062,
      "eval_loss": 0.48540204763412476,
      "eval_rewards/accuracies": 0.6710000038146973,
      "eval_rewards/chosen": -2.0009877681732178,
      "eval_rewards/margins": 1.0855647325515747,
      "eval_rewards/rejected": -3.086552143096924,
      "eval_runtime": 734.9099,
      "eval_samples_per_second": 2.721,
      "eval_steps_per_second": 1.361,
      "step": 14600
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.9563139978421028e-08,
      "logits/chosen": -0.9228948354721069,
      "logits/rejected": -0.9851576089859009,
      "logps/chosen": -426.33856201171875,
      "logps/rejected": -487.5555725097656,
      "loss": 0.6203,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.004666805267334,
      "rewards/margins": 0.6900150179862976,
      "rewards/rejected": -2.6946821212768555,
      "step": 14610
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.869407277731939e-08,
      "logits/chosen": -0.7606264352798462,
      "logits/rejected": -0.7889705896377563,
      "logps/chosen": -361.2055969238281,
      "logps/rejected": -435.94757080078125,
      "loss": 0.4694,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.759484052658081,
      "rewards/margins": 0.9791158437728882,
      "rewards/rejected": -2.738600254058838,
      "step": 14620
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.783789711851642e-08,
      "logits/chosen": -1.157167911529541,
      "logits/rejected": -0.877331554889679,
      "logps/chosen": -355.41644287109375,
      "logps/rejected": -471.55120849609375,
      "loss": 0.4778,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9356542825698853,
      "rewards/margins": 1.385674238204956,
      "rewards/rejected": -3.321328639984131,
      "step": 14630
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.6994617468244778e-08,
      "logits/chosen": -1.0710012912750244,
      "logits/rejected": -0.9255205988883972,
      "logps/chosen": -413.6026916503906,
      "logps/rejected": -466.4881896972656,
      "loss": 0.4256,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.0058205127716064,
      "rewards/margins": 1.1680415868759155,
      "rewards/rejected": -3.1738619804382324,
      "step": 14640
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.6164238225463155e-08,
      "logits/chosen": -0.9301780462265015,
      "logits/rejected": -0.6651344299316406,
      "logps/chosen": -469.53314208984375,
      "logps/rejected": -512.8493041992188,
      "loss": 0.4997,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8710609674453735,
      "rewards/margins": 1.2182409763336182,
      "rewards/rejected": -3.089301824569702,
      "step": 14650
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.534676372183742e-08,
      "logits/chosen": -0.8242905735969543,
      "logits/rejected": -0.8369461894035339,
      "logps/chosen": -473.88983154296875,
      "logps/rejected": -563.2259521484375,
      "loss": 0.5052,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8993546962738037,
      "rewards/margins": 1.4166282415390015,
      "rewards/rejected": -3.315983295440674,
      "step": 14660
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4542198221714218e-08,
      "logits/chosen": -0.7648957967758179,
      "logits/rejected": -0.6464065313339233,
      "logps/chosen": -311.3927307128906,
      "logps/rejected": -418.7562561035156,
      "loss": 0.5343,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.7328574657440186,
      "rewards/margins": 1.0713664293289185,
      "rewards/rejected": -2.8042235374450684,
      "step": 14670
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3750545922101854e-08,
      "logits/chosen": -1.4671556949615479,
      "logits/rejected": -0.8755186200141907,
      "logps/chosen": -518.0368041992188,
      "logps/rejected": -562.1136474609375,
      "loss": 0.4419,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.086455821990967,
      "rewards/margins": 1.1247096061706543,
      "rewards/rejected": -3.2111656665802,
      "step": 14680
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.2971810952646112e-08,
      "logits/chosen": -1.1903879642486572,
      "logits/rejected": -0.998426616191864,
      "logps/chosen": -480.20343017578125,
      "logps/rejected": -508.8087463378906,
      "loss": 0.566,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.2184062004089355,
      "rewards/margins": 0.7631117701530457,
      "rewards/rejected": -2.981517791748047,
      "step": 14690
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.2205997375610576e-08,
      "logits/chosen": -0.8135994076728821,
      "logits/rejected": -0.7705693244934082,
      "logps/chosen": -336.8621826171875,
      "logps/rejected": -462.5765075683594,
      "loss": 0.4681,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.513562798500061,
      "rewards/margins": 1.2055425643920898,
      "rewards/rejected": -2.7191052436828613,
      "step": 14700
    },
    {
      "epoch": 0.96,
      "eval_logits/chosen": -1.055994987487793,
      "eval_logits/rejected": -0.9266590476036072,
      "eval_logps/chosen": -431.9801025390625,
      "eval_logps/rejected": -520.0926513671875,
      "eval_loss": 0.4855150580406189,
      "eval_rewards/accuracies": 0.6700000166893005,
      "eval_rewards/chosen": -1.9997512102127075,
      "eval_rewards/margins": 1.0850558280944824,
      "eval_rewards/rejected": -3.0848069190979004,
      "eval_runtime": 733.2428,
      "eval_samples_per_second": 2.728,
      "eval_steps_per_second": 1.364,
      "step": 14700
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.1453109185853304e-08,
      "logits/chosen": -1.1346324682235718,
      "logits/rejected": -1.1408488750457764,
      "logps/chosen": -374.3671569824219,
      "logps/rejected": -476.6963806152344,
      "loss": 0.4865,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7228870391845703,
      "rewards/margins": 1.005819320678711,
      "rewards/rejected": -2.7287065982818604,
      "step": 14710
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.0713150310808784e-08,
      "logits/chosen": -1.140235185623169,
      "logits/rejected": -1.0906654596328735,
      "logps/chosen": -420.65869140625,
      "logps/rejected": -480.44708251953125,
      "loss": 0.5937,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.9517085552215576,
      "rewards/margins": 0.5402504205703735,
      "rewards/rejected": -2.4919590950012207,
      "step": 14720
    },
    {
      "epoch": 0.96,
      "learning_rate": 1.9986124610464064e-08,
      "logits/chosen": -0.7904390096664429,
      "logits/rejected": -0.6426718235015869,
      "logps/chosen": -528.2882690429688,
      "logps/rejected": -612.8759155273438,
      "loss": 0.3436,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.410581111907959,
      "rewards/margins": 1.5439999103546143,
      "rewards/rejected": -3.954580783843994,
      "step": 14730
    },
    {
      "epoch": 0.96,
      "learning_rate": 1.927203587734211e-08,
      "logits/chosen": -0.7250292897224426,
      "logits/rejected": -0.6967679262161255,
      "logps/chosen": -438.3182067871094,
      "logps/rejected": -526.4837646484375,
      "loss": 0.5014,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8532650470733643,
      "rewards/margins": 1.366919994354248,
      "rewards/rejected": -3.2201850414276123,
      "step": 14740
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.8570887836479034e-08,
      "logits/chosen": -0.9606091380119324,
      "logits/rejected": -0.7177838087081909,
      "logps/chosen": -425.73175048828125,
      "logps/rejected": -569.6988525390625,
      "loss": 0.4986,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.3210859298706055,
      "rewards/margins": 0.9115177989006042,
      "rewards/rejected": -3.2326037883758545,
      "step": 14750
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.7882684145406616e-08,
      "logits/chosen": -1.1041526794433594,
      "logits/rejected": -1.0806796550750732,
      "logps/chosen": -485.91790771484375,
      "logps/rejected": -578.8372802734375,
      "loss": 0.5158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8949730396270752,
      "rewards/margins": 0.9480817914009094,
      "rewards/rejected": -2.84305477142334,
      "step": 14760
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.7207428394132865e-08,
      "logits/chosen": -1.3055307865142822,
      "logits/rejected": -0.9931842088699341,
      "logps/chosen": -477.50469970703125,
      "logps/rejected": -567.6167602539062,
      "loss": 0.4226,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.0772435665130615,
      "rewards/margins": 1.447993516921997,
      "rewards/rejected": -3.5252373218536377,
      "step": 14770
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.654512410512177e-08,
      "logits/chosen": -1.0979361534118652,
      "logits/rejected": -0.8360413312911987,
      "logps/chosen": -447.86029052734375,
      "logps/rejected": -473.5550842285156,
      "loss": 0.5438,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9428339004516602,
      "rewards/margins": 0.8808323740959167,
      "rewards/rejected": -2.8236660957336426,
      "step": 14780
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.5895774733277468e-08,
      "logits/chosen": -0.984605610370636,
      "logits/rejected": -0.8599230051040649,
      "logps/chosen": -512.8638916015625,
      "logps/rejected": -566.1119384765625,
      "loss": 0.5088,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.3304696083068848,
      "rewards/margins": 1.0348457098007202,
      "rewards/rejected": -3.3653149604797363,
      "step": 14790
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.5259383665924e-08,
      "logits/chosen": -1.5153400897979736,
      "logits/rejected": -1.1625539064407349,
      "logps/chosen": -537.54150390625,
      "logps/rejected": -583.3179931640625,
      "loss": 0.4417,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.954496145248413,
      "rewards/margins": 1.3978536128997803,
      "rewards/rejected": -3.3523495197296143,
      "step": 14800
    },
    {
      "epoch": 0.97,
      "eval_logits/chosen": -1.0548804998397827,
      "eval_logits/rejected": -0.9253528118133545,
      "eval_logps/chosen": -432.188232421875,
      "eval_logps/rejected": -520.3868408203125,
      "eval_loss": 0.48528552055358887,
      "eval_rewards/accuracies": 0.671500027179718,
      "eval_rewards/chosen": -2.0018327236175537,
      "eval_rewards/margins": 1.0859160423278809,
      "eval_rewards/rejected": -3.0877487659454346,
      "eval_runtime": 735.1276,
      "eval_samples_per_second": 2.721,
      "eval_steps_per_second": 1.36,
      "step": 14800
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.4635954222789461e-08,
      "logits/chosen": -1.218711256980896,
      "logits/rejected": -1.1651209592819214,
      "logps/chosen": -416.23004150390625,
      "logps/rejected": -518.4617919921875,
      "loss": 0.5465,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.9912240505218506,
      "rewards/margins": 0.971238911151886,
      "rewards/rejected": -2.962462902069092,
      "step": 14810
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.402548965598688e-08,
      "logits/chosen": -1.1414233446121216,
      "logits/rejected": -0.9460879564285278,
      "logps/chosen": -413.87188720703125,
      "logps/rejected": -520.7072143554688,
      "loss": 0.4791,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.0924458503723145,
      "rewards/margins": 1.1005178689956665,
      "rewards/rejected": -3.1929638385772705,
      "step": 14820
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.3427993149998375e-08,
      "logits/chosen": -1.2117063999176025,
      "logits/rejected": -1.0982780456542969,
      "logps/chosen": -437.361083984375,
      "logps/rejected": -491.078125,
      "loss": 0.4716,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9532392024993896,
      "rewards/margins": 1.0908714532852173,
      "rewards/rejected": -3.0441105365753174,
      "step": 14830
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.2843467821658518e-08,
      "logits/chosen": -1.2357401847839355,
      "logits/rejected": -1.0914043188095093,
      "logps/chosen": -398.68353271484375,
      "logps/rejected": -521.7628784179688,
      "loss": 0.4396,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7246061563491821,
      "rewards/margins": 1.2344787120819092,
      "rewards/rejected": -2.959084987640381,
      "step": 14840
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.2271916720137666e-08,
      "logits/chosen": -1.4540129899978638,
      "logits/rejected": -1.137785792350769,
      "logps/chosen": -503.82501220703125,
      "logps/rejected": -536.3612060546875,
      "loss": 0.5983,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1805543899536133,
      "rewards/margins": 0.786711573600769,
      "rewards/rejected": -2.967266082763672,
      "step": 14850
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.171334282692671e-08,
      "logits/chosen": -1.2715617418289185,
      "logits/rejected": -1.0421197414398193,
      "logps/chosen": -478.9646911621094,
      "logps/rejected": -586.7308959960938,
      "loss": 0.396,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9636627435684204,
      "rewards/margins": 1.3839813470840454,
      "rewards/rejected": -3.3476436138153076,
      "step": 14860
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.116774905582041e-08,
      "logits/chosen": -1.1228783130645752,
      "logits/rejected": -0.9871054887771606,
      "logps/chosen": -374.7401428222656,
      "logps/rejected": -438.5972595214844,
      "loss": 0.5955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.8992159366607666,
      "rewards/margins": 0.7656039595603943,
      "rewards/rejected": -2.6648201942443848,
      "step": 14870
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.0635138252902966e-08,
      "logits/chosen": -1.2849915027618408,
      "logits/rejected": -1.150407075881958,
      "logps/chosen": -429.98321533203125,
      "logps/rejected": -527.2002563476562,
      "loss": 0.4301,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.995692491531372,
      "rewards/margins": 1.1934545040130615,
      "rewards/rejected": -3.1891469955444336,
      "step": 14880
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.0115513196533589e-08,
      "logits/chosen": -1.2216876745224,
      "logits/rejected": -1.0180718898773193,
      "logps/chosen": -453.65777587890625,
      "logps/rejected": -507.4833068847656,
      "loss": 0.5853,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9104763269424438,
      "rewards/margins": 0.7113370895385742,
      "rewards/rejected": -2.6218130588531494,
      "step": 14890
    },
    {
      "epoch": 0.97,
      "learning_rate": 9.608876597330952e-09,
      "logits/chosen": -1.1359317302703857,
      "logits/rejected": -0.8752700090408325,
      "logps/chosen": -544.45068359375,
      "logps/rejected": -649.2925415039062,
      "loss": 0.516,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.5782792568206787,
      "rewards/margins": 1.1652708053588867,
      "rewards/rejected": -3.7435498237609863,
      "step": 14900
    },
    {
      "epoch": 0.97,
      "eval_logits/chosen": -1.0542715787887573,
      "eval_logits/rejected": -0.9249005317687988,
      "eval_logps/chosen": -432.13201904296875,
      "eval_logps/rejected": -520.3480834960938,
      "eval_loss": 0.48537370562553406,
      "eval_rewards/accuracies": 0.6700000166893005,
      "eval_rewards/chosen": -2.0012712478637695,
      "eval_rewards/margins": 1.086090087890625,
      "eval_rewards/rejected": -3.0873613357543945,
      "eval_runtime": 732.6647,
      "eval_samples_per_second": 2.73,
      "eval_steps_per_second": 1.365,
      "step": 14900
    },
    {
      "epoch": 0.98,
      "learning_rate": 9.115231098159594e-09,
      "logits/chosen": -1.2317795753479004,
      "logits/rejected": -1.197540283203125,
      "logps/chosen": -445.7748107910156,
      "logps/rejected": -509.39288330078125,
      "loss": 0.5767,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9246104955673218,
      "rewards/margins": 0.8392006754875183,
      "rewards/rejected": -2.7638113498687744,
      "step": 14910
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.634579274116317e-09,
      "logits/chosen": -1.0000141859054565,
      "logits/rejected": -1.0341566801071167,
      "logps/chosen": -363.2093200683594,
      "logps/rejected": -465.8623046875,
      "loss": 0.6092,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.7315008640289307,
      "rewards/margins": 0.8204340934753418,
      "rewards/rejected": -2.5519349575042725,
      "step": 14920
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.166923632516865e-09,
      "logits/chosen": -1.2367180585861206,
      "logits/rejected": -1.001817226409912,
      "logps/chosen": -417.31402587890625,
      "logps/rejected": -633.681884765625,
      "loss": 0.3654,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.8958278894424438,
      "rewards/margins": 1.787407636642456,
      "rewards/rejected": -3.6832358837127686,
      "step": 14930
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.712266612881492e-09,
      "logits/chosen": -0.8679588437080383,
      "logits/rejected": -0.7478830218315125,
      "logps/chosen": -344.3149719238281,
      "logps/rejected": -457.35089111328125,
      "loss": 0.4323,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5907829999923706,
      "rewards/margins": 1.1791170835494995,
      "rewards/rejected": -2.769899845123291,
      "step": 14940
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.270610586924687e-09,
      "logits/chosen": -1.2399359941482544,
      "logits/rejected": -1.1076147556304932,
      "logps/chosen": -456.895751953125,
      "logps/rejected": -520.5778198242188,
      "loss": 0.4983,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.919304609298706,
      "rewards/margins": 1.0776231288909912,
      "rewards/rejected": -2.996927499771118,
      "step": 14950
    },
    {
      "epoch": 0.98,
      "learning_rate": 6.841957858539916e-09,
      "logits/chosen": -1.0050383806228638,
      "logits/rejected": -0.9870063662528992,
      "logps/chosen": -399.08941650390625,
      "logps/rejected": -478.54241943359375,
      "loss": 0.5808,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.324338912963867,
      "rewards/margins": 0.6860220432281494,
      "rewards/rejected": -3.0103607177734375,
      "step": 14960
    },
    {
      "epoch": 0.98,
      "learning_rate": 6.426310663790181e-09,
      "logits/chosen": -0.9850949048995972,
      "logits/rejected": -0.6555286645889282,
      "logps/chosen": -460.75439453125,
      "logps/rejected": -519.1944580078125,
      "loss": 0.5963,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.237785816192627,
      "rewards/margins": 0.8990850448608398,
      "rewards/rejected": -3.136870861053467,
      "step": 14970
    },
    {
      "epoch": 0.98,
      "learning_rate": 6.023671170894696e-09,
      "logits/chosen": -1.3116604089736938,
      "logits/rejected": -0.957747757434845,
      "logps/chosen": -485.274658203125,
      "logps/rejected": -561.7952880859375,
      "loss": 0.3114,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.929460883140564,
      "rewards/margins": 1.6330032348632812,
      "rewards/rejected": -3.5624642372131348,
      "step": 14980
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.634041480218344e-09,
      "logits/chosen": -1.0634709596633911,
      "logits/rejected": -1.2313354015350342,
      "logps/chosen": -450.6940002441406,
      "logps/rejected": -563.3707885742188,
      "loss": 0.4649,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9762039184570312,
      "rewards/margins": 1.1711724996566772,
      "rewards/rejected": -3.147376298904419,
      "step": 14990
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.257423624260849e-09,
      "logits/chosen": -1.2537717819213867,
      "logits/rejected": -0.9475277066230774,
      "logps/chosen": -469.4458923339844,
      "logps/rejected": -545.3299560546875,
      "loss": 0.5369,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.0612242221832275,
      "rewards/margins": 1.1912734508514404,
      "rewards/rejected": -3.252497911453247,
      "step": 15000
    },
    {
      "epoch": 0.98,
      "eval_logits/chosen": -1.0536617040634155,
      "eval_logits/rejected": -0.924413800239563,
      "eval_logps/chosen": -432.14788818359375,
      "eval_logps/rejected": -520.3270874023438,
      "eval_loss": 0.4853664040565491,
      "eval_rewards/accuracies": 0.6704999804496765,
      "eval_rewards/chosen": -2.0014290809631348,
      "eval_rewards/margins": 1.085722804069519,
      "eval_rewards/rejected": -3.0871517658233643,
      "eval_runtime": 731.2965,
      "eval_samples_per_second": 2.735,
      "eval_steps_per_second": 1.367,
      "step": 15000
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.893819567644564e-09,
      "logits/chosen": -0.9579187631607056,
      "logits/rejected": -0.9031054377555847,
      "logps/chosen": -366.45526123046875,
      "logps/rejected": -449.436767578125,
      "loss": 0.5813,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7845852375030518,
      "rewards/margins": 0.7280542850494385,
      "rewards/rejected": -2.5126395225524902,
      "step": 15010
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.543231207107257e-09,
      "logits/chosen": -0.8927432894706726,
      "logits/rejected": -0.96844881772995,
      "logps/chosen": -475.09466552734375,
      "logps/rejected": -545.72900390625,
      "loss": 0.5459,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.1727311611175537,
      "rewards/margins": 0.9424299001693726,
      "rewards/rejected": -3.115161180496216,
      "step": 15020
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.205660371488785e-09,
      "logits/chosen": -1.2815234661102295,
      "logits/rejected": -1.1932264566421509,
      "logps/chosen": -508.7298278808594,
      "logps/rejected": -565.9403686523438,
      "loss": 0.5658,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.245765209197998,
      "rewards/margins": 0.9900045394897461,
      "rewards/rejected": -3.235769271850586,
      "step": 15030
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.88110882172471e-09,
      "logits/chosen": -1.0577032566070557,
      "logits/rejected": -1.0624340772628784,
      "logps/chosen": -433.12603759765625,
      "logps/rejected": -525.9588012695312,
      "loss": 0.5161,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1242759227752686,
      "rewards/margins": 0.9746935963630676,
      "rewards/rejected": -3.0989696979522705,
      "step": 15040
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.569578250834371e-09,
      "logits/chosen": -0.9758235812187195,
      "logits/rejected": -1.0453417301177979,
      "logps/chosen": -473.77239990234375,
      "logps/rejected": -589.0740356445312,
      "loss": 0.3695,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.7179462909698486,
      "rewards/margins": 1.5306434631347656,
      "rewards/rejected": -3.2485899925231934,
      "step": 15050
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.2710702839139353e-09,
      "logits/chosen": -1.129165530204773,
      "logits/rejected": -1.040967345237732,
      "logps/chosen": -408.28240966796875,
      "logps/rejected": -488.89935302734375,
      "loss": 0.5936,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.036830425262451,
      "rewards/margins": 0.7134383320808411,
      "rewards/rejected": -2.7502689361572266,
      "step": 15060
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.9855864781272448e-09,
      "logits/chosen": -1.214477300643921,
      "logits/rejected": -1.325339674949646,
      "logps/chosen": -396.2386169433594,
      "logps/rejected": -524.412353515625,
      "loss": 0.528,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.8456494808197021,
      "rewards/margins": 0.9337970614433289,
      "rewards/rejected": -2.779446601867676,
      "step": 15070
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.7131283226977665e-09,
      "logits/chosen": -1.1110106706619263,
      "logits/rejected": -1.1910868883132935,
      "logps/chosen": -415.4817810058594,
      "logps/rejected": -538.5015869140625,
      "loss": 0.4812,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9603351354599,
      "rewards/margins": 1.0916963815689087,
      "rewards/rejected": -3.0520312786102295,
      "step": 15080
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.4536972389008205e-09,
      "logits/chosen": -1.0061264038085938,
      "logits/rejected": -0.9592840075492859,
      "logps/chosen": -424.15185546875,
      "logps/rejected": -516.12890625,
      "loss": 0.4262,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.870168924331665,
      "rewards/margins": 1.2966797351837158,
      "rewards/rejected": -3.166848659515381,
      "step": 15090
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.20729458005553e-09,
      "logits/chosen": -0.9116916656494141,
      "logits/rejected": -0.6472299695014954,
      "logps/chosen": -385.2953186035156,
      "logps/rejected": -491.0865783691406,
      "loss": 0.442,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9088608026504517,
      "rewards/margins": 1.279929518699646,
      "rewards/rejected": -3.1887905597686768,
      "step": 15100
    },
    {
      "epoch": 0.99,
      "eval_logits/chosen": -1.0546156167984009,
      "eval_logits/rejected": -0.9254134893417358,
      "eval_logps/chosen": -432.0098571777344,
      "eval_logps/rejected": -520.1915283203125,
      "eval_loss": 0.4853096008300781,
      "eval_rewards/accuracies": 0.671500027179718,
      "eval_rewards/chosen": -2.0000486373901367,
      "eval_rewards/margins": 1.0857477188110352,
      "eval_rewards/rejected": -3.0857961177825928,
      "eval_runtime": 732.0844,
      "eval_samples_per_second": 2.732,
      "eval_steps_per_second": 1.366,
      "step": 15100
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.9739216315192712e-09,
      "logits/chosen": -1.0977634191513062,
      "logits/rejected": -0.8465323448181152,
      "logps/chosen": -429.33892822265625,
      "logps/rejected": -495.4786682128906,
      "loss": 0.5072,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0003013610839844,
      "rewards/margins": 0.8983712196350098,
      "rewards/rejected": -2.898672580718994,
      "step": 15110
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.7535796106796231e-09,
      "logits/chosen": -1.1453430652618408,
      "logits/rejected": -0.8692857623100281,
      "logps/chosen": -476.8145446777344,
      "logps/rejected": -496.99481201171875,
      "loss": 0.4946,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.0127811431884766,
      "rewards/margins": 1.0051124095916748,
      "rewards/rejected": -3.0178937911987305,
      "step": 15120
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.5462696669482636e-09,
      "logits/chosen": -1.240791916847229,
      "logits/rejected": -1.0452940464019775,
      "logps/chosen": -411.5857849121094,
      "logps/rejected": -510.7543029785156,
      "loss": 0.5611,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.903436303138733,
      "rewards/margins": 0.9152453541755676,
      "rewards/rejected": -2.8186817169189453,
      "step": 15130
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.3519928817556927e-09,
      "logits/chosen": -1.0189902782440186,
      "logits/rejected": -0.8792413473129272,
      "logps/chosen": -352.0376281738281,
      "logps/rejected": -434.4034729003906,
      "loss": 0.5348,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.786441445350647,
      "rewards/margins": 0.8181886672973633,
      "rewards/rejected": -2.6046297550201416,
      "step": 15140
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.1707502685448512e-09,
      "logits/chosen": -1.2458264827728271,
      "logits/rejected": -1.0184621810913086,
      "logps/chosen": -460.77850341796875,
      "logps/rejected": -509.570068359375,
      "loss": 0.5115,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.385084629058838,
      "rewards/margins": 1.0698010921478271,
      "rewards/rejected": -3.454885959625244,
      "step": 15150
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.002542772765569e-09,
      "logits/chosen": -0.9062987565994263,
      "logits/rejected": -0.7371143698692322,
      "logps/chosen": -374.9570617675781,
      "logps/rejected": -443.7781677246094,
      "loss": 0.4719,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.863532304763794,
      "rewards/margins": 1.0712873935699463,
      "rewards/rejected": -2.9348196983337402,
      "step": 15160
    },
    {
      "epoch": 0.99,
      "learning_rate": 8.473712718709559e-10,
      "logits/chosen": -1.0819716453552246,
      "logits/rejected": -1.0894006490707397,
      "logps/chosen": -406.87353515625,
      "logps/rejected": -448.4830017089844,
      "loss": 0.5974,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.131286144256592,
      "rewards/margins": 0.52387535572052,
      "rewards/rejected": -2.6551613807678223,
      "step": 15170
    },
    {
      "epoch": 0.99,
      "learning_rate": 7.052365753112966e-10,
      "logits/chosen": -0.8242332339286804,
      "logits/rejected": -0.8488165736198425,
      "logps/chosen": -464.1226501464844,
      "logps/rejected": -555.234130859375,
      "loss": 0.5364,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.2880654335021973,
      "rewards/margins": 1.098871111869812,
      "rewards/rejected": -3.386936902999878,
      "step": 15180
    },
    {
      "epoch": 0.99,
      "learning_rate": 5.761394245307195e-10,
      "logits/chosen": -0.8750588297843933,
      "logits/rejected": -0.9825660586357117,
      "logps/chosen": -440.34344482421875,
      "logps/rejected": -505.38800048828125,
      "loss": 0.5564,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.0326123237609863,
      "rewards/margins": 0.6625606417655945,
      "rewards/rejected": -2.6951727867126465,
      "step": 15190
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.6008049296358826e-10,
      "logits/chosen": -1.1716618537902832,
      "logits/rejected": -1.0131809711456299,
      "logps/chosen": -359.4949645996094,
      "logps/rejected": -432.78948974609375,
      "loss": 0.4814,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.734723687171936,
      "rewards/margins": 1.0153820514678955,
      "rewards/rejected": -2.750105619430542,
      "step": 15200
    },
    {
      "epoch": 0.99,
      "eval_logits/chosen": -1.0581306219100952,
      "eval_logits/rejected": -0.9285818338394165,
      "eval_logps/chosen": -431.9893493652344,
      "eval_logps/rejected": -520.1320190429688,
      "eval_loss": 0.4854483902454376,
      "eval_rewards/accuracies": 0.671999990940094,
      "eval_rewards/chosen": -1.999843716621399,
      "eval_rewards/margins": 1.0853569507598877,
      "eval_rewards/rejected": -3.085200548171997,
      "eval_runtime": 730.109,
      "eval_samples_per_second": 2.739,
      "eval_steps_per_second": 1.37,
      "step": 15200
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.5706038603006146e-10,
      "logits/chosen": -1.0735647678375244,
      "logits/rejected": -1.0667097568511963,
      "logps/chosen": -445.7269592285156,
      "logps/rejected": -559.1290283203125,
      "loss": 0.4524,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5611655712127686,
      "rewards/margins": 1.276451826095581,
      "rewards/rejected": -2.8376173973083496,
      "step": 15210
    },
    {
      "epoch": 1.0,
      "learning_rate": 2.670796411333165e-10,
      "logits/chosen": -1.4009530544281006,
      "logits/rejected": -1.077488660812378,
      "logps/chosen": -432.26788330078125,
      "logps/rejected": -525.7315673828125,
      "loss": 0.503,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.1245641708374023,
      "rewards/margins": 1.0576976537704468,
      "rewards/rejected": -3.1822619438171387,
      "step": 15220
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.9013872765677455e-10,
      "logits/chosen": -1.1325209140777588,
      "logits/rejected": -1.025263786315918,
      "logps/chosen": -384.40570068359375,
      "logps/rejected": -450.3807678222656,
      "loss": 0.5252,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.64892578125,
      "rewards/margins": 0.785096287727356,
      "rewards/rejected": -2.4340221881866455,
      "step": 15230
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.262380469624347e-10,
      "logits/chosen": -1.0579341650009155,
      "logits/rejected": -0.881651759147644,
      "logps/chosen": -395.76531982421875,
      "logps/rejected": -468.01776123046875,
      "loss": 0.4803,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9175841808319092,
      "rewards/margins": 0.9516212344169617,
      "rewards/rejected": -2.8692054748535156,
      "step": 15240
    },
    {
      "epoch": 1.0,
      "learning_rate": 7.53779323872661e-11,
      "logits/chosen": -1.0222140550613403,
      "logits/rejected": -0.9701115489006042,
      "logps/chosen": -397.0771789550781,
      "logps/rejected": -520.7330322265625,
      "loss": 0.6077,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.052137851715088,
      "rewards/margins": 1.1761195659637451,
      "rewards/rejected": -3.228257417678833,
      "step": 15250
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.7558649242652734e-11,
      "logits/chosen": -1.4560601711273193,
      "logits/rejected": -1.136359453201294,
      "logps/chosen": -625.4713134765625,
      "logps/rejected": -639.3870239257812,
      "loss": 0.5194,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.287875175476074,
      "rewards/margins": 0.9348654747009277,
      "rewards/rejected": -3.222740650177002,
      "step": 15260
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.2780394812450526e-11,
      "logits/chosen": -0.8604592084884644,
      "logits/rejected": -0.9040239453315735,
      "logps/chosen": -468.42352294921875,
      "logps/rejected": -587.9859619140625,
      "loss": 0.4393,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.362797260284424,
      "rewards/margins": 1.1964504718780518,
      "rewards/rejected": -3.559248447418213,
      "step": 15270
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.0432983521546646e-12,
      "logits/chosen": -0.9416975975036621,
      "logits/rejected": -0.9361570477485657,
      "logps/chosen": -362.9610595703125,
      "logps/rejected": -484.21112060546875,
      "loss": 0.4863,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7949488162994385,
      "rewards/margins": 1.024044394493103,
      "rewards/rejected": -2.818993091583252,
      "step": 15280
    },
    {
      "epoch": 1.0,
      "step": 15284,
      "total_flos": 0.0,
      "train_loss": 0.5233479982519362,
      "train_runtime": 176392.38,
      "train_samples_per_second": 0.347,
      "train_steps_per_second": 0.087
    }
  ],
  "logging_steps": 10,
  "max_steps": 15284,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}