{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 14.915254237288135,
  "eval_steps": 1,
  "global_step": 880,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01694915254237288,
      "grad_norm": 40.30728845506363,
      "learning_rate": 4.2372881355932205e-09,
      "logits/chosen": 12.842013359069824,
      "logits/rejected": 13.082613945007324,
      "logps/chosen": -18.68050193786621,
      "logps/rejected": -30.006702423095703,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03389830508474576,
      "grad_norm": 38.041564508830845,
      "learning_rate": 8.474576271186441e-09,
      "logits/chosen": 10.079428672790527,
      "logits/rejected": 10.317561149597168,
      "logps/chosen": -20.233402252197266,
      "logps/rejected": -21.939817428588867,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.05084745762711865,
      "grad_norm": 40.36008813830181,
      "learning_rate": 1.2711864406779661e-08,
      "logits/chosen": 14.052921295166016,
      "logits/rejected": 15.504006385803223,
      "logps/chosen": -16.064619064331055,
      "logps/rejected": -29.048044204711914,
      "loss": 0.6998,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.026119917631149292,
      "rewards/margins": -0.0660426914691925,
      "rewards/rejected": 0.03992277383804321,
      "step": 3
    },
    {
      "epoch": 0.06779661016949153,
      "grad_norm": 36.63114570098774,
      "learning_rate": 1.6949152542372882e-08,
      "logits/chosen": 11.676168441772461,
      "logits/rejected": 12.226595878601074,
      "logps/chosen": -15.098368644714355,
      "logps/rejected": -23.02960205078125,
      "loss": 0.7011,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0063827671110630035,
      "rewards/margins": 0.014015134423971176,
      "rewards/rejected": -0.02039790153503418,
      "step": 4
    },
    {
      "epoch": 0.0847457627118644,
      "grad_norm": 42.846639257495035,
      "learning_rate": 2.11864406779661e-08,
      "logits/chosen": 13.691095352172852,
      "logits/rejected": 13.234848976135254,
      "logps/chosen": -18.44330596923828,
      "logps/rejected": -15.939766883850098,
      "loss": 0.7118,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.05990014597773552,
      "rewards/margins": 0.0980239287018776,
      "rewards/rejected": -0.038123778998851776,
      "step": 5
    },
    {
      "epoch": 0.1016949152542373,
      "grad_norm": 41.95119341285687,
      "learning_rate": 2.5423728813559323e-08,
      "logits/chosen": 9.710973739624023,
      "logits/rejected": 9.734879493713379,
      "logps/chosen": -19.200092315673828,
      "logps/rejected": -23.185443878173828,
      "loss": 0.707,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014187633991241455,
      "rewards/margins": 0.03218716382980347,
      "rewards/rejected": -0.01799952983856201,
      "step": 6
    },
    {
      "epoch": 0.11864406779661017,
      "grad_norm": 42.934384992123036,
      "learning_rate": 2.966101694915254e-08,
      "logits/chosen": 12.107307434082031,
      "logits/rejected": 12.784415245056152,
      "logps/chosen": -16.704126358032227,
      "logps/rejected": -20.03870391845703,
      "loss": 0.7033,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06281542778015137,
      "rewards/margins": 0.17079591751098633,
      "rewards/rejected": -0.10798049718141556,
      "step": 7
    },
    {
      "epoch": 0.13559322033898305,
      "grad_norm": 37.640968099147095,
      "learning_rate": 3.3898305084745764e-08,
      "logits/chosen": 12.114370346069336,
      "logits/rejected": 12.843629837036133,
      "logps/chosen": -14.356505393981934,
      "logps/rejected": -23.001556396484375,
      "loss": 0.7016,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.04248759523034096,
      "rewards/margins": -0.09302316606044769,
      "rewards/rejected": 0.050535574555397034,
      "step": 8
    },
    {
      "epoch": 0.15254237288135594,
      "grad_norm": 48.59143625607344,
      "learning_rate": 3.813559322033898e-08,
      "logits/chosen": 13.338932991027832,
      "logits/rejected": 13.014775276184082,
      "logps/chosen": -20.295446395874023,
      "logps/rejected": -17.377910614013672,
      "loss": 0.7236,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.012785300612449646,
      "rewards/margins": 0.025210216641426086,
      "rewards/rejected": -0.01242491602897644,
      "step": 9
    },
    {
      "epoch": 0.1694915254237288,
      "grad_norm": 42.21481624311643,
      "learning_rate": 4.23728813559322e-08,
      "logits/chosen": 10.676657676696777,
      "logits/rejected": 10.866190910339355,
      "logps/chosen": -22.40692138671875,
      "logps/rejected": -17.18635368347168,
      "loss": 0.7366,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0834585577249527,
      "rewards/margins": -0.16166189312934875,
      "rewards/rejected": 0.07820333540439606,
      "step": 10
    },
    {
      "epoch": 0.1864406779661017,
      "grad_norm": 44.24683815055113,
      "learning_rate": 4.661016949152542e-08,
      "logits/chosen": 11.689413070678711,
      "logits/rejected": 12.616247177124023,
      "logps/chosen": -16.787853240966797,
      "logps/rejected": -25.714006423950195,
      "loss": 0.7008,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.051930248737335205,
      "rewards/margins": -0.04377424716949463,
      "rewards/rejected": 0.09570449590682983,
      "step": 11
    },
    {
      "epoch": 0.2033898305084746,
      "grad_norm": 36.42082486581824,
      "learning_rate": 5.0847457627118645e-08,
      "logits/chosen": 10.625275611877441,
      "logits/rejected": 10.755789756774902,
      "logps/chosen": -13.972813606262207,
      "logps/rejected": -18.619009017944336,
      "loss": 0.6921,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0011137649416923523,
      "rewards/margins": -0.010056130588054657,
      "rewards/rejected": 0.008942365646362305,
      "step": 12
    },
    {
      "epoch": 0.22033898305084745,
      "grad_norm": 39.783625967921616,
      "learning_rate": 5.508474576271186e-08,
      "logits/chosen": 11.280006408691406,
      "logits/rejected": 12.170235633850098,
      "logps/chosen": -15.202749252319336,
      "logps/rejected": -19.611961364746094,
      "loss": 0.7027,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.04294466972351074,
      "rewards/margins": -0.056273579597473145,
      "rewards/rejected": 0.013328909873962402,
      "step": 13
    },
    {
      "epoch": 0.23728813559322035,
      "grad_norm": 43.360099635584334,
      "learning_rate": 5.932203389830508e-08,
      "logits/chosen": 7.622957706451416,
      "logits/rejected": 9.428025245666504,
      "logps/chosen": -18.667062759399414,
      "logps/rejected": -33.28583908081055,
      "loss": 0.6832,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.021295249462127686,
      "rewards/margins": 0.09253796935081482,
      "rewards/rejected": -0.07124271988868713,
      "step": 14
    },
    {
      "epoch": 0.2542372881355932,
      "grad_norm": 39.17087374081227,
      "learning_rate": 6.35593220338983e-08,
      "logits/chosen": 15.24155044555664,
      "logits/rejected": 16.208242416381836,
      "logps/chosen": -13.249297142028809,
      "logps/rejected": -23.505441665649414,
      "loss": 0.7163,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.02330087125301361,
      "rewards/margins": -0.07975521683692932,
      "rewards/rejected": 0.05645434558391571,
      "step": 15
    },
    {
      "epoch": 0.2711864406779661,
      "grad_norm": 37.50466933990464,
      "learning_rate": 6.779661016949153e-08,
      "logits/chosen": 14.139371871948242,
      "logits/rejected": 14.819007873535156,
      "logps/chosen": -16.831674575805664,
      "logps/rejected": -22.343116760253906,
      "loss": 0.6819,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.03721078485250473,
      "rewards/margins": -0.026429735124111176,
      "rewards/rejected": -0.010781049728393555,
      "step": 16
    },
    {
      "epoch": 0.288135593220339,
      "grad_norm": 39.00712547599067,
      "learning_rate": 7.203389830508475e-08,
      "logits/chosen": 14.457000732421875,
      "logits/rejected": 14.57475757598877,
      "logps/chosen": -13.015029907226562,
      "logps/rejected": -21.924034118652344,
      "loss": 0.6667,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08990475535392761,
      "rewards/margins": 0.14169014990329742,
      "rewards/rejected": -0.05178540199995041,
      "step": 17
    },
    {
      "epoch": 0.3050847457627119,
      "grad_norm": 44.410867327593046,
      "learning_rate": 7.627118644067796e-08,
      "logits/chosen": 12.978434562683105,
      "logits/rejected": 13.062633514404297,
      "logps/chosen": -12.442479133605957,
      "logps/rejected": -17.96292495727539,
      "loss": 0.7077,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06292188912630081,
      "rewards/margins": -0.007752574980258942,
      "rewards/rejected": 0.07067446410655975,
      "step": 18
    },
    {
      "epoch": 0.3220338983050847,
      "grad_norm": 38.86515880597673,
      "learning_rate": 8.050847457627117e-08,
      "logits/chosen": 10.975082397460938,
      "logits/rejected": 11.75900936126709,
      "logps/chosen": -11.997967720031738,
      "logps/rejected": -24.670589447021484,
      "loss": 0.7101,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.018813543021678925,
      "rewards/margins": -0.012066647410392761,
      "rewards/rejected": -0.006746895611286163,
      "step": 19
    },
    {
      "epoch": 0.3389830508474576,
      "grad_norm": 37.07719478540159,
      "learning_rate": 8.47457627118644e-08,
      "logits/chosen": 13.550518989562988,
      "logits/rejected": 13.391486167907715,
      "logps/chosen": -17.554624557495117,
      "logps/rejected": -20.00838279724121,
      "loss": 0.693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.066666379570961,
      "rewards/margins": 0.024734124541282654,
      "rewards/rejected": 0.041932255029678345,
      "step": 20
    },
    {
      "epoch": 0.3559322033898305,
      "grad_norm": 38.29971669543194,
      "learning_rate": 8.898305084745762e-08,
      "logits/chosen": 13.924101829528809,
      "logits/rejected": 14.325352668762207,
      "logps/chosen": -14.366829872131348,
      "logps/rejected": -21.499202728271484,
      "loss": 0.7111,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.044451721012592316,
      "rewards/margins": -0.024806134402751923,
      "rewards/rejected": 0.06925785541534424,
      "step": 21
    },
    {
      "epoch": 0.3728813559322034,
      "grad_norm": 41.353983577755464,
      "learning_rate": 9.322033898305084e-08,
      "logits/chosen": 9.174437522888184,
      "logits/rejected": 9.49307918548584,
      "logps/chosen": -13.705348014831543,
      "logps/rejected": -17.275392532348633,
      "loss": 0.7085,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0074497610330581665,
      "rewards/margins": -0.07652243971824646,
      "rewards/rejected": 0.0690726786851883,
      "step": 22
    },
    {
      "epoch": 0.3898305084745763,
      "grad_norm": 43.114227330572284,
      "learning_rate": 9.745762711864407e-08,
      "logits/chosen": 12.173064231872559,
      "logits/rejected": 12.370023727416992,
      "logps/chosen": -16.37883758544922,
      "logps/rejected": -20.038183212280273,
      "loss": 0.723,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01693063974380493,
      "rewards/margins": 0.010861068964004517,
      "rewards/rejected": -0.027791708707809448,
      "step": 23
    },
    {
      "epoch": 0.4067796610169492,
      "grad_norm": 38.83032139356952,
      "learning_rate": 1.0169491525423729e-07,
      "logits/chosen": 15.312846183776855,
      "logits/rejected": 15.3582181930542,
      "logps/chosen": -20.627126693725586,
      "logps/rejected": -27.445995330810547,
      "loss": 0.6751,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07566741108894348,
      "rewards/margins": 0.06957697868347168,
      "rewards/rejected": 0.006090432405471802,
      "step": 24
    },
    {
      "epoch": 0.423728813559322,
      "grad_norm": 43.67807182666561,
      "learning_rate": 1.059322033898305e-07,
      "logits/chosen": 11.813891410827637,
      "logits/rejected": 11.837639808654785,
      "logps/chosen": -21.68389892578125,
      "logps/rejected": -25.150630950927734,
      "loss": 0.6891,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.08590184897184372,
      "rewards/margins": 0.07415612787008286,
      "rewards/rejected": 0.011745721101760864,
      "step": 25
    },
    {
      "epoch": 0.4406779661016949,
      "grad_norm": 41.746172744282724,
      "learning_rate": 1.1016949152542372e-07,
      "logits/chosen": 10.105422973632812,
      "logits/rejected": 10.882453918457031,
      "logps/chosen": -14.849566459655762,
      "logps/rejected": -21.765117645263672,
      "loss": 0.7372,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004259124398231506,
      "rewards/margins": 0.01350797712802887,
      "rewards/rejected": -0.009248852729797363,
      "step": 26
    },
    {
      "epoch": 0.4576271186440678,
      "grad_norm": 42.56394623827792,
      "learning_rate": 1.1440677966101695e-07,
      "logits/chosen": 13.506670951843262,
      "logits/rejected": 13.843586921691895,
      "logps/chosen": -19.2850399017334,
      "logps/rejected": -22.344873428344727,
      "loss": 0.7004,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.05585460364818573,
      "rewards/margins": -0.10725802183151245,
      "rewards/rejected": 0.05140341818332672,
      "step": 27
    },
    {
      "epoch": 0.4745762711864407,
      "grad_norm": 36.639123230654995,
      "learning_rate": 1.1864406779661017e-07,
      "logits/chosen": 14.286264419555664,
      "logits/rejected": 14.345661163330078,
      "logps/chosen": -15.126670837402344,
      "logps/rejected": -21.23917007446289,
      "loss": 0.6815,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.012283587828278542,
      "rewards/margins": 0.06251242011785507,
      "rewards/rejected": -0.05022883415222168,
      "step": 28
    },
    {
      "epoch": 0.4915254237288136,
      "grad_norm": 43.447579941943744,
      "learning_rate": 1.228813559322034e-07,
      "logits/chosen": 12.693879127502441,
      "logits/rejected": 12.80766773223877,
      "logps/chosen": -16.297504425048828,
      "logps/rejected": -23.56643295288086,
      "loss": 0.6999,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.020490556955337524,
      "rewards/margins": 0.07709893584251404,
      "rewards/rejected": -0.09758949279785156,
      "step": 29
    },
    {
      "epoch": 0.5084745762711864,
      "grad_norm": 41.412620150940505,
      "learning_rate": 1.271186440677966e-07,
      "logits/chosen": 10.882184982299805,
      "logits/rejected": 11.457561492919922,
      "logps/chosen": -18.117883682250977,
      "logps/rejected": -24.392854690551758,
      "loss": 0.6949,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.05693977326154709,
      "rewards/margins": -0.06227093189954758,
      "rewards/rejected": 0.005331158638000488,
      "step": 30
    },
    {
      "epoch": 0.5254237288135594,
      "grad_norm": 41.04485757895064,
      "learning_rate": 1.3135593220338984e-07,
      "logits/chosen": 12.045116424560547,
      "logits/rejected": 12.250951766967773,
      "logps/chosen": -18.459415435791016,
      "logps/rejected": -24.026748657226562,
      "loss": 0.7062,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.022956043481826782,
      "rewards/margins": -0.023535877466201782,
      "rewards/rejected": 0.046491920948028564,
      "step": 31
    },
    {
      "epoch": 0.5423728813559322,
      "grad_norm": 76.97626958402756,
      "learning_rate": 1.3559322033898305e-07,
      "logits/chosen": 12.790939331054688,
      "logits/rejected": 12.999430656433105,
      "logps/chosen": -18.866657257080078,
      "logps/rejected": -24.532800674438477,
      "loss": 0.7016,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008551269769668579,
      "rewards/margins": -0.07439977675676346,
      "rewards/rejected": 0.06584850698709488,
      "step": 32
    },
    {
      "epoch": 0.559322033898305,
      "grad_norm": 41.233124075932956,
      "learning_rate": 1.3983050847457625e-07,
      "logits/chosen": 11.019987106323242,
      "logits/rejected": 11.895480155944824,
      "logps/chosen": -17.390188217163086,
      "logps/rejected": -27.735076904296875,
      "loss": 0.7024,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07621224224567413,
      "rewards/margins": -0.00863146036863327,
      "rewards/rejected": 0.0848436951637268,
      "step": 33
    },
    {
      "epoch": 0.576271186440678,
      "grad_norm": 41.51281273691135,
      "learning_rate": 1.440677966101695e-07,
      "logits/chosen": 10.961234092712402,
      "logits/rejected": 11.117724418640137,
      "logps/chosen": -15.324450492858887,
      "logps/rejected": -22.6516056060791,
      "loss": 0.7064,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03333364427089691,
      "rewards/margins": -0.046573787927627563,
      "rewards/rejected": 0.013240143656730652,
      "step": 34
    },
    {
      "epoch": 0.5932203389830508,
      "grad_norm": 38.46980857022322,
      "learning_rate": 1.483050847457627e-07,
      "logits/chosen": 11.720096588134766,
      "logits/rejected": 13.81384563446045,
      "logps/chosen": -11.844643592834473,
      "logps/rejected": -26.574565887451172,
      "loss": 0.7136,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01212693378329277,
      "rewards/margins": -0.1099301129579544,
      "rewards/rejected": 0.09780317544937134,
      "step": 35
    },
    {
      "epoch": 0.6101694915254238,
      "grad_norm": 36.14892188798069,
      "learning_rate": 1.5254237288135593e-07,
      "logits/chosen": 13.399576187133789,
      "logits/rejected": 12.95138168334961,
      "logps/chosen": -13.442176818847656,
      "logps/rejected": -18.09130859375,
      "loss": 0.6871,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01939527690410614,
      "rewards/margins": -0.010730020701885223,
      "rewards/rejected": 0.030125297605991364,
      "step": 36
    },
    {
      "epoch": 0.6271186440677966,
      "grad_norm": 40.851396852891206,
      "learning_rate": 1.5677966101694915e-07,
      "logits/chosen": 14.05972671508789,
      "logits/rejected": 14.02302360534668,
      "logps/chosen": -19.48711395263672,
      "logps/rejected": -20.447059631347656,
      "loss": 0.703,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.016420789062976837,
      "rewards/margins": -0.0325637087225914,
      "rewards/rejected": 0.016142919659614563,
      "step": 37
    },
    {
      "epoch": 0.6440677966101694,
      "grad_norm": 37.67420037057592,
      "learning_rate": 1.6101694915254234e-07,
      "logits/chosen": 14.11151123046875,
      "logits/rejected": 15.1259765625,
      "logps/chosen": -19.008934020996094,
      "logps/rejected": -34.84490966796875,
      "loss": 0.6951,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.008656233549118042,
      "rewards/margins": -0.0032239556312561035,
      "rewards/rejected": 0.011880189180374146,
      "step": 38
    },
    {
      "epoch": 0.6610169491525424,
      "grad_norm": 37.70147259149854,
      "learning_rate": 1.6525423728813559e-07,
      "logits/chosen": 11.573564529418945,
      "logits/rejected": 12.407613754272461,
      "logps/chosen": -25.639665603637695,
      "logps/rejected": -27.497907638549805,
      "loss": 0.7096,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07243013381958008,
      "rewards/margins": 0.015660464763641357,
      "rewards/rejected": -0.08809059858322144,
      "step": 39
    },
    {
      "epoch": 0.6779661016949152,
      "grad_norm": 37.72600351271015,
      "learning_rate": 1.694915254237288e-07,
      "logits/chosen": 12.521224021911621,
      "logits/rejected": 13.355399131774902,
      "logps/chosen": -15.367606163024902,
      "logps/rejected": -24.378122329711914,
      "loss": 0.7133,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.029328536242246628,
      "rewards/margins": -0.07064150273799896,
      "rewards/rejected": 0.09997004270553589,
      "step": 40
    },
    {
      "epoch": 0.6949152542372882,
      "grad_norm": 37.200999329078535,
      "learning_rate": 1.7372881355932202e-07,
      "logits/chosen": 9.892135620117188,
      "logits/rejected": 11.508831977844238,
      "logps/chosen": -16.929044723510742,
      "logps/rejected": -24.869295120239258,
      "loss": 0.6806,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.057470276951789856,
      "rewards/margins": 0.0028195232152938843,
      "rewards/rejected": -0.06028980016708374,
      "step": 41
    },
    {
      "epoch": 0.711864406779661,
      "grad_norm": 41.45400661954368,
      "learning_rate": 1.7796610169491524e-07,
      "logits/chosen": 12.402385711669922,
      "logits/rejected": 12.750102996826172,
      "logps/chosen": -14.469609260559082,
      "logps/rejected": -22.454177856445312,
      "loss": 0.7085,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06580530107021332,
      "rewards/margins": 0.16378699243068695,
      "rewards/rejected": -0.09798169136047363,
      "step": 42
    },
    {
      "epoch": 0.7288135593220338,
      "grad_norm": 41.67437074393324,
      "learning_rate": 1.8220338983050846e-07,
      "logits/chosen": 14.246957778930664,
      "logits/rejected": 14.030410766601562,
      "logps/chosen": -27.281932830810547,
      "logps/rejected": -22.738405227661133,
      "loss": 0.6912,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.020992666482925415,
      "rewards/margins": 0.07494118809700012,
      "rewards/rejected": -0.09593385457992554,
      "step": 43
    },
    {
      "epoch": 0.7457627118644068,
      "grad_norm": 36.371842747457514,
      "learning_rate": 1.8644067796610168e-07,
      "logits/chosen": 12.301523208618164,
      "logits/rejected": 12.70305347442627,
      "logps/chosen": -20.010610580444336,
      "logps/rejected": -29.714580535888672,
      "loss": 0.6922,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.12373033165931702,
      "rewards/margins": 0.15592673420906067,
      "rewards/rejected": -0.03219640254974365,
      "step": 44
    },
    {
      "epoch": 0.7627118644067796,
      "grad_norm": 41.8781105096038,
      "learning_rate": 1.906779661016949e-07,
      "logits/chosen": 10.72525405883789,
      "logits/rejected": 12.510992050170898,
      "logps/chosen": -14.862523078918457,
      "logps/rejected": -27.508800506591797,
      "loss": 0.6838,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.029734574258327484,
      "rewards/margins": 0.07138457149267197,
      "rewards/rejected": -0.04164999723434448,
      "step": 45
    },
    {
      "epoch": 0.7796610169491526,
      "grad_norm": 41.90756110852291,
      "learning_rate": 1.9491525423728814e-07,
      "logits/chosen": 11.77270221710205,
      "logits/rejected": 11.579182624816895,
      "logps/chosen": -18.880056381225586,
      "logps/rejected": -22.22597885131836,
      "loss": 0.7062,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0413464680314064,
      "rewards/margins": 0.0769132599234581,
      "rewards/rejected": -0.0355667918920517,
      "step": 46
    },
    {
      "epoch": 0.7966101694915254,
      "grad_norm": 62.77035926211305,
      "learning_rate": 1.9915254237288134e-07,
      "logits/chosen": 12.305220603942871,
      "logits/rejected": 12.062664031982422,
      "logps/chosen": -18.956796646118164,
      "logps/rejected": -20.161949157714844,
      "loss": 0.7016,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03725249692797661,
      "rewards/margins": 0.011303797364234924,
      "rewards/rejected": 0.025948703289031982,
      "step": 47
    },
    {
      "epoch": 0.8135593220338984,
      "grad_norm": 37.86640870800892,
      "learning_rate": 2.0338983050847458e-07,
      "logits/chosen": 13.177642822265625,
      "logits/rejected": 13.93821907043457,
      "logps/chosen": -16.465944290161133,
      "logps/rejected": -26.467010498046875,
      "loss": 0.6881,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.035428911447525024,
      "rewards/margins": -0.01987355947494507,
      "rewards/rejected": 0.05530247092247009,
      "step": 48
    },
    {
      "epoch": 0.8305084745762712,
      "grad_norm": 37.25491848473269,
      "learning_rate": 2.076271186440678e-07,
      "logits/chosen": 14.861462593078613,
      "logits/rejected": 14.734480857849121,
      "logps/chosen": -15.433595657348633,
      "logps/rejected": -17.87074089050293,
      "loss": 0.7049,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03526681661605835,
      "rewards/margins": -0.05743015184998512,
      "rewards/rejected": 0.022163331508636475,
      "step": 49
    },
    {
      "epoch": 0.847457627118644,
      "grad_norm": 41.88806594120029,
      "learning_rate": 2.11864406779661e-07,
      "logits/chosen": 10.575276374816895,
      "logits/rejected": 11.21845817565918,
      "logps/chosen": -14.93527889251709,
      "logps/rejected": -25.54239273071289,
      "loss": 0.703,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.06612725555896759,
      "rewards/margins": -0.07982112467288971,
      "rewards/rejected": 0.01369386911392212,
      "step": 50
    },
    {
      "epoch": 0.864406779661017,
      "grad_norm": 40.46403881382111,
      "learning_rate": 2.1610169491525424e-07,
      "logits/chosen": 12.37446117401123,
      "logits/rejected": 13.336200714111328,
      "logps/chosen": -17.579078674316406,
      "logps/rejected": -32.97596740722656,
      "loss": 0.6947,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008079767227172852,
      "rewards/margins": 0.00566767156124115,
      "rewards/rejected": 0.0024120956659317017,
      "step": 51
    },
    {
      "epoch": 0.8813559322033898,
      "grad_norm": 39.18280033903859,
      "learning_rate": 2.2033898305084743e-07,
      "logits/chosen": 13.351238250732422,
      "logits/rejected": 14.134299278259277,
      "logps/chosen": -16.299585342407227,
      "logps/rejected": -24.282123565673828,
      "loss": 0.6835,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08001573383808136,
      "rewards/margins": 0.03988678753376007,
      "rewards/rejected": 0.04012894630432129,
      "step": 52
    },
    {
      "epoch": 0.8983050847457628,
      "grad_norm": 39.754504822625954,
      "learning_rate": 2.2457627118644068e-07,
      "logits/chosen": 13.946189880371094,
      "logits/rejected": 14.154667854309082,
      "logps/chosen": -15.623710632324219,
      "logps/rejected": -18.68361473083496,
      "loss": 0.6901,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02733871340751648,
      "rewards/margins": 0.13955903053283691,
      "rewards/rejected": -0.1668977439403534,
      "step": 53
    },
    {
      "epoch": 0.9152542372881356,
      "grad_norm": 37.66072122883095,
      "learning_rate": 2.288135593220339e-07,
      "logits/chosen": 15.305002212524414,
      "logits/rejected": 15.318461418151855,
      "logps/chosen": -17.736726760864258,
      "logps/rejected": -18.221403121948242,
      "loss": 0.6973,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.10565029084682465,
      "rewards/margins": -0.06389039754867554,
      "rewards/rejected": -0.04175989329814911,
      "step": 54
    },
    {
      "epoch": 0.9322033898305084,
      "grad_norm": 35.612257753723746,
      "learning_rate": 2.330508474576271e-07,
      "logits/chosen": 15.089719772338867,
      "logits/rejected": 15.363324165344238,
      "logps/chosen": -17.88437843322754,
      "logps/rejected": -21.706897735595703,
      "loss": 0.6827,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.053910866379737854,
      "rewards/margins": 0.10098430514335632,
      "rewards/rejected": -0.04707343876361847,
      "step": 55
    },
    {
      "epoch": 0.9491525423728814,
      "grad_norm": 39.18257877542564,
      "learning_rate": 2.3728813559322033e-07,
      "logits/chosen": 13.049365997314453,
      "logits/rejected": 13.170905113220215,
      "logps/chosen": -15.466392517089844,
      "logps/rejected": -19.350955963134766,
      "loss": 0.7121,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.08173052966594696,
      "rewards/margins": -0.09363162517547607,
      "rewards/rejected": 0.011901095509529114,
      "step": 56
    },
    {
      "epoch": 0.9661016949152542,
      "grad_norm": 40.96891813392811,
      "learning_rate": 2.4152542372881355e-07,
      "logits/chosen": 13.86726188659668,
      "logits/rejected": 14.41377067565918,
      "logps/chosen": -17.831151962280273,
      "logps/rejected": -28.358530044555664,
      "loss": 0.7051,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.056873977184295654,
      "rewards/margins": 0.04540741443634033,
      "rewards/rejected": 0.011466562747955322,
      "step": 57
    },
    {
      "epoch": 0.9830508474576272,
      "grad_norm": 40.64462852147455,
      "learning_rate": 2.457627118644068e-07,
      "logits/chosen": 13.420578956604004,
      "logits/rejected": 13.936662673950195,
      "logps/chosen": -22.40713119506836,
      "logps/rejected": -22.56110191345215,
      "loss": 0.7096,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.023797959089279175,
      "rewards/margins": -0.01631149649620056,
      "rewards/rejected": 0.040109455585479736,
      "step": 58
    },
    {
      "epoch": 1.0,
      "grad_norm": 42.59747961209585,
      "learning_rate": 2.5e-07,
      "logits/chosen": 12.881525039672852,
      "logits/rejected": 13.721171379089355,
      "logps/chosen": -16.626148223876953,
      "logps/rejected": -25.509788513183594,
      "loss": 0.694,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.028950288891792297,
      "rewards/margins": 0.006730042397975922,
      "rewards/rejected": -0.03568033128976822,
      "step": 59
    },
    {
      "epoch": 1.0169491525423728,
      "grad_norm": 39.65797571647544,
      "learning_rate": 2.542372881355932e-07,
      "logits/chosen": 14.147932052612305,
      "logits/rejected": 15.005398750305176,
      "logps/chosen": -12.525471687316895,
      "logps/rejected": -23.025493621826172,
      "loss": 0.6843,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01412774994969368,
      "rewards/margins": 0.024437706917524338,
      "rewards/rejected": -0.03856545686721802,
      "step": 60
    },
    {
      "epoch": 1.0338983050847457,
      "grad_norm": 36.46992678492697,
      "learning_rate": 2.584745762711864e-07,
      "logits/chosen": 12.175691604614258,
      "logits/rejected": 12.264796257019043,
      "logps/chosen": -17.057971954345703,
      "logps/rejected": -21.540843963623047,
      "loss": 0.6934,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.048048973083496094,
      "rewards/margins": 0.03411996364593506,
      "rewards/rejected": -0.08216893672943115,
      "step": 61
    },
    {
      "epoch": 1.0508474576271187,
      "grad_norm": 38.82225863120416,
      "learning_rate": 2.6271186440677967e-07,
      "logits/chosen": 10.932394027709961,
      "logits/rejected": 11.038849830627441,
      "logps/chosen": -16.846797943115234,
      "logps/rejected": -23.29388999938965,
      "loss": 0.6939,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03787383437156677,
      "rewards/margins": 0.04288873076438904,
      "rewards/rejected": -0.005014896392822266,
      "step": 62
    },
    {
      "epoch": 1.0677966101694916,
      "grad_norm": 39.49407859666274,
      "learning_rate": 2.6694915254237286e-07,
      "logits/chosen": 10.65343952178955,
      "logits/rejected": 11.759461402893066,
      "logps/chosen": -16.778003692626953,
      "logps/rejected": -26.33823585510254,
      "loss": 0.6776,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.09588789939880371,
      "rewards/margins": 0.12244513630867004,
      "rewards/rejected": -0.026557236909866333,
      "step": 63
    },
    {
      "epoch": 1.0847457627118644,
      "grad_norm": 39.740758663676,
      "learning_rate": 2.711864406779661e-07,
      "logits/chosen": 12.711525917053223,
      "logits/rejected": 13.348038673400879,
      "logps/chosen": -17.374736785888672,
      "logps/rejected": -24.012853622436523,
      "loss": 0.6737,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009046778082847595,
      "rewards/margins": 0.059755921363830566,
      "rewards/rejected": -0.06880269944667816,
      "step": 64
    },
    {
      "epoch": 1.1016949152542372,
      "grad_norm": 43.94207160987874,
      "learning_rate": 2.754237288135593e-07,
      "logits/chosen": 11.751956939697266,
      "logits/rejected": 13.209794044494629,
      "logps/chosen": -17.514083862304688,
      "logps/rejected": -27.080425262451172,
      "loss": 0.6797,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12722863256931305,
      "rewards/margins": -0.018149808049201965,
      "rewards/rejected": -0.10907882452011108,
      "step": 65
    },
    {
      "epoch": 1.11864406779661,
      "grad_norm": 35.43215229240541,
      "learning_rate": 2.796610169491525e-07,
      "logits/chosen": 12.129363059997559,
      "logits/rejected": 12.394501686096191,
      "logps/chosen": -18.410751342773438,
      "logps/rejected": -22.589263916015625,
      "loss": 0.6714,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.015718191862106323,
      "rewards/margins": 0.04346385598182678,
      "rewards/rejected": -0.02774566411972046,
      "step": 66
    },
    {
      "epoch": 1.1355932203389831,
      "grad_norm": 37.13327137852814,
      "learning_rate": 2.838983050847458e-07,
      "logits/chosen": 10.463054656982422,
      "logits/rejected": 10.192320823669434,
      "logps/chosen": -14.659592628479004,
      "logps/rejected": -16.757896423339844,
      "loss": 0.669,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00866326130926609,
      "rewards/margins": 0.055536217987537384,
      "rewards/rejected": -0.06419947743415833,
      "step": 67
    },
    {
      "epoch": 1.152542372881356,
      "grad_norm": 40.430671120170146,
      "learning_rate": 2.88135593220339e-07,
      "logits/chosen": 14.019149780273438,
      "logits/rejected": 14.417764663696289,
      "logps/chosen": -16.717283248901367,
      "logps/rejected": -24.413545608520508,
      "loss": 0.6584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12998425960540771,
      "rewards/margins": 0.2277044951915741,
      "rewards/rejected": -0.09772025048732758,
      "step": 68
    },
    {
      "epoch": 1.1694915254237288,
      "grad_norm": 40.58226171654033,
      "learning_rate": 2.923728813559322e-07,
      "logits/chosen": 12.504399299621582,
      "logits/rejected": 13.594255447387695,
      "logps/chosen": -14.431526184082031,
      "logps/rejected": -22.050336837768555,
      "loss": 0.6733,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.043449416756629944,
      "rewards/margins": 0.060832902789115906,
      "rewards/rejected": -0.10428231954574585,
      "step": 69
    },
    {
      "epoch": 1.1864406779661016,
      "grad_norm": 39.39764264695123,
      "learning_rate": 2.966101694915254e-07,
      "logits/chosen": 11.229328155517578,
      "logits/rejected": 11.79920768737793,
      "logps/chosen": -15.364906311035156,
      "logps/rejected": -24.188587188720703,
      "loss": 0.6667,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.16962946951389313,
      "rewards/margins": 0.2086838036775589,
      "rewards/rejected": -0.03905433416366577,
      "step": 70
    },
    {
      "epoch": 1.2033898305084745,
      "grad_norm": 40.31701080664346,
      "learning_rate": 3.008474576271186e-07,
      "logits/chosen": 11.156949043273926,
      "logits/rejected": 11.680891036987305,
      "logps/chosen": -16.79814910888672,
      "logps/rejected": -25.06736946105957,
      "loss": 0.6683,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.05000852048397064,
      "rewards/margins": 0.0933413952589035,
      "rewards/rejected": -0.04333287477493286,
      "step": 71
    },
    {
      "epoch": 1.2203389830508475,
      "grad_norm": 41.58246062784223,
      "learning_rate": 3.0508474576271186e-07,
      "logits/chosen": 8.323310852050781,
      "logits/rejected": 9.706480026245117,
      "logps/chosen": -18.133089065551758,
      "logps/rejected": -26.048744201660156,
      "loss": 0.6549,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04978783428668976,
      "rewards/margins": 0.22536294162273407,
      "rewards/rejected": -0.1755751073360443,
      "step": 72
    },
    {
      "epoch": 1.2372881355932204,
      "grad_norm": 37.559888806481496,
      "learning_rate": 3.093220338983051e-07,
      "logits/chosen": 13.401594161987305,
      "logits/rejected": 13.73277759552002,
      "logps/chosen": -14.402023315429688,
      "logps/rejected": -19.22127914428711,
      "loss": 0.6612,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16372399032115936,
      "rewards/margins": 0.29817506670951843,
      "rewards/rejected": -0.13445107638835907,
      "step": 73
    },
    {
      "epoch": 1.2542372881355932,
      "grad_norm": 40.882004689136416,
      "learning_rate": 3.135593220338983e-07,
      "logits/chosen": 13.300498008728027,
      "logits/rejected": 13.367609977722168,
      "logps/chosen": -16.360227584838867,
      "logps/rejected": -22.51313018798828,
      "loss": 0.6598,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.06130232661962509,
      "rewards/margins": 0.21569877862930298,
      "rewards/rejected": -0.15439645946025848,
      "step": 74
    },
    {
      "epoch": 1.271186440677966,
      "grad_norm": 37.29775386481398,
      "learning_rate": 3.177966101694915e-07,
      "logits/chosen": 9.930148124694824,
      "logits/rejected": 9.965421676635742,
      "logps/chosen": -13.893769264221191,
      "logps/rejected": -17.500465393066406,
      "loss": 0.6518,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.06109648942947388,
      "rewards/margins": 0.09164294600486755,
      "rewards/rejected": -0.030546456575393677,
      "step": 75
    },
    {
      "epoch": 1.288135593220339,
      "grad_norm": 35.354986913884105,
      "learning_rate": 3.220338983050847e-07,
      "logits/chosen": 13.425969123840332,
      "logits/rejected": 14.32559585571289,
      "logps/chosen": -15.175914764404297,
      "logps/rejected": -24.598207473754883,
      "loss": 0.6717,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0012704432010650635,
      "rewards/margins": 0.02735494077205658,
      "rewards/rejected": -0.028625383973121643,
      "step": 76
    },
    {
      "epoch": 1.305084745762712,
      "grad_norm": 36.98175660192275,
      "learning_rate": 3.26271186440678e-07,
      "logits/chosen": 9.431652069091797,
      "logits/rejected": 9.38131332397461,
      "logps/chosen": -21.996917724609375,
      "logps/rejected": -30.023685455322266,
      "loss": 0.6569,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13959163427352905,
      "rewards/margins": 0.22159650921821594,
      "rewards/rejected": -0.08200487494468689,
      "step": 77
    },
    {
      "epoch": 1.3220338983050848,
      "grad_norm": 36.359735380010825,
      "learning_rate": 3.3050847457627117e-07,
      "logits/chosen": 14.359407424926758,
      "logits/rejected": 15.139140129089355,
      "logps/chosen": -15.036028861999512,
      "logps/rejected": -24.142406463623047,
      "loss": 0.6529,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.014202922582626343,
      "rewards/margins": 0.19232788681983948,
      "rewards/rejected": -0.17812496423721313,
      "step": 78
    },
    {
      "epoch": 1.3389830508474576,
      "grad_norm": 36.50503808890011,
      "learning_rate": 3.3474576271186436e-07,
      "logits/chosen": 14.310426712036133,
      "logits/rejected": 14.55799674987793,
      "logps/chosen": -12.61948013305664,
      "logps/rejected": -20.105886459350586,
      "loss": 0.6574,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005430340766906738,
      "rewards/margins": 0.12721706926822662,
      "rewards/rejected": -0.12178672850131989,
      "step": 79
    },
    {
      "epoch": 1.3559322033898304,
      "grad_norm": 35.78916311248173,
      "learning_rate": 3.389830508474576e-07,
      "logits/chosen": 12.969365119934082,
      "logits/rejected": 13.552573204040527,
      "logps/chosen": -15.17340087890625,
      "logps/rejected": -27.757129669189453,
      "loss": 0.6573,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05384726822376251,
      "rewards/margins": 0.14679737389087677,
      "rewards/rejected": -0.09295010566711426,
      "step": 80
    },
    {
      "epoch": 1.3728813559322033,
      "grad_norm": 38.99240126520527,
      "learning_rate": 3.432203389830508e-07,
      "logits/chosen": 14.033243179321289,
      "logits/rejected": 14.331340789794922,
      "logps/chosen": -14.390337944030762,
      "logps/rejected": -22.8853702545166,
      "loss": 0.6488,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.07335115224123001,
      "rewards/margins": 0.18050891160964966,
      "rewards/rejected": -0.10715775191783905,
      "step": 81
    },
    {
      "epoch": 1.3898305084745763,
      "grad_norm": 34.14892965471411,
      "learning_rate": 3.4745762711864405e-07,
      "logits/chosen": 11.835970878601074,
      "logits/rejected": 11.749543190002441,
      "logps/chosen": -11.856498718261719,
      "logps/rejected": -14.143364906311035,
      "loss": 0.6574,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.04997224733233452,
      "rewards/margins": 0.04305719956755638,
      "rewards/rejected": 0.006915047764778137,
      "step": 82
    },
    {
      "epoch": 1.4067796610169492,
      "grad_norm": 39.274641594335634,
      "learning_rate": 3.516949152542373e-07,
      "logits/chosen": 10.360257148742676,
      "logits/rejected": 10.089805603027344,
      "logps/chosen": -18.05191421508789,
      "logps/rejected": -20.790409088134766,
      "loss": 0.6483,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03167504072189331,
      "rewards/margins": 0.13742826879024506,
      "rewards/rejected": -0.10575322806835175,
      "step": 83
    },
    {
      "epoch": 1.423728813559322,
      "grad_norm": 36.85996674163347,
      "learning_rate": 3.559322033898305e-07,
      "logits/chosen": 10.419514656066895,
      "logits/rejected": 10.655095100402832,
      "logps/chosen": -17.017175674438477,
      "logps/rejected": -19.419795989990234,
      "loss": 0.6639,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04058393836021423,
      "rewards/margins": 0.0065583735704422,
      "rewards/rejected": -0.04714231193065643,
      "step": 84
    },
    {
      "epoch": 1.4406779661016949,
      "grad_norm": 35.17113062069321,
      "learning_rate": 3.601694915254237e-07,
      "logits/chosen": 13.433987617492676,
      "logits/rejected": 13.448290824890137,
      "logps/chosen": -15.061188697814941,
      "logps/rejected": -16.228702545166016,
      "loss": 0.6367,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.028515294194221497,
      "rewards/margins": 0.07839739322662354,
      "rewards/rejected": -0.04988209158182144,
      "step": 85
    },
    {
      "epoch": 1.457627118644068,
      "grad_norm": 37.41104165649846,
      "learning_rate": 3.644067796610169e-07,
      "logits/chosen": 11.649856567382812,
      "logits/rejected": 12.419816970825195,
      "logps/chosen": -15.746871948242188,
      "logps/rejected": -28.98259162902832,
      "loss": 0.6567,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.020991250872612,
      "rewards/margins": 0.15705639123916626,
      "rewards/rejected": -0.13606514036655426,
      "step": 86
    },
    {
      "epoch": 1.4745762711864407,
      "grad_norm": 35.42707515007882,
      "learning_rate": 3.6864406779661017e-07,
      "logits/chosen": 10.078230857849121,
      "logits/rejected": 10.944649696350098,
      "logps/chosen": -15.349961280822754,
      "logps/rejected": -23.0633487701416,
      "loss": 0.6476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008510768413543701,
      "rewards/margins": 0.13419035077095032,
      "rewards/rejected": -0.12567958235740662,
      "step": 87
    },
    {
      "epoch": 1.4915254237288136,
      "grad_norm": 35.46644556499606,
      "learning_rate": 3.7288135593220336e-07,
      "logits/chosen": 11.770813941955566,
      "logits/rejected": 12.585672378540039,
      "logps/chosen": -18.352783203125,
      "logps/rejected": -22.152597427368164,
      "loss": 0.6159,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.1304049789905548,
      "rewards/margins": 0.36608466506004333,
      "rewards/rejected": -0.23567967116832733,
      "step": 88
    },
    {
      "epoch": 1.5084745762711864,
      "grad_norm": 37.29455949159622,
      "learning_rate": 3.771186440677966e-07,
      "logits/chosen": 13.56704330444336,
      "logits/rejected": 13.756218910217285,
      "logps/chosen": -18.772235870361328,
      "logps/rejected": -23.98781967163086,
      "loss": 0.6373,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0235111266374588,
      "rewards/margins": 0.19346098601818085,
      "rewards/rejected": -0.16994985938072205,
      "step": 89
    },
    {
      "epoch": 1.5254237288135593,
      "grad_norm": 37.279643088142166,
      "learning_rate": 3.813559322033898e-07,
      "logits/chosen": 10.409061431884766,
      "logits/rejected": 11.294801712036133,
      "logps/chosen": -20.020545959472656,
      "logps/rejected": -27.11080551147461,
      "loss": 0.6548,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.03669746220111847,
      "rewards/margins": 0.003694683313369751,
      "rewards/rejected": -0.04039214551448822,
      "step": 90
    },
    {
      "epoch": 1.542372881355932,
      "grad_norm": 34.92132206920193,
      "learning_rate": 3.8559322033898304e-07,
      "logits/chosen": 10.68431282043457,
      "logits/rejected": 10.871668815612793,
      "logps/chosen": -18.425750732421875,
      "logps/rejected": -19.080305099487305,
      "loss": 0.6373,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04941455274820328,
      "rewards/margins": 0.01522158831357956,
      "rewards/rejected": -0.06463614106178284,
      "step": 91
    },
    {
      "epoch": 1.559322033898305,
      "grad_norm": 35.67266173202364,
      "learning_rate": 3.898305084745763e-07,
      "logits/chosen": 11.35604476928711,
      "logits/rejected": 11.843755722045898,
      "logps/chosen": -17.879486083984375,
      "logps/rejected": -24.5284423828125,
      "loss": 0.612,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03402914106845856,
      "rewards/margins": 0.12276534736156464,
      "rewards/rejected": -0.08873620629310608,
      "step": 92
    },
    {
      "epoch": 1.576271186440678,
      "grad_norm": 37.54947919804327,
      "learning_rate": 3.940677966101695e-07,
      "logits/chosen": 11.453618049621582,
      "logits/rejected": 12.393528938293457,
      "logps/chosen": -14.59844970703125,
      "logps/rejected": -22.819143295288086,
      "loss": 0.6599,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05744030699133873,
      "rewards/margins": 0.21809090673923492,
      "rewards/rejected": -0.1606505960226059,
      "step": 93
    },
    {
      "epoch": 1.5932203389830508,
      "grad_norm": 34.33945131326431,
      "learning_rate": 3.9830508474576267e-07,
      "logits/chosen": 9.380054473876953,
      "logits/rejected": 9.780099868774414,
      "logps/chosen": -13.068021774291992,
      "logps/rejected": -24.602806091308594,
      "loss": 0.6245,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02326585352420807,
      "rewards/margins": 0.2321978360414505,
      "rewards/rejected": -0.20893198251724243,
      "step": 94
    },
    {
      "epoch": 1.6101694915254239,
      "grad_norm": 35.81592033651009,
      "learning_rate": 4.025423728813559e-07,
      "logits/chosen": 13.495811462402344,
      "logits/rejected": 13.81532096862793,
      "logps/chosen": -13.080314636230469,
      "logps/rejected": -18.98967170715332,
      "loss": 0.6407,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.07077131420373917,
      "rewards/margins": 0.2651638686656952,
      "rewards/rejected": -0.19439256191253662,
      "step": 95
    },
    {
      "epoch": 1.6271186440677967,
      "grad_norm": 34.53874606260594,
      "learning_rate": 4.0677966101694916e-07,
      "logits/chosen": 12.902689933776855,
      "logits/rejected": 13.325467109680176,
      "logps/chosen": -18.66386604309082,
      "logps/rejected": -23.48880958557129,
      "loss": 0.6075,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.015904970467090607,
      "rewards/margins": 0.06729435175657272,
      "rewards/rejected": -0.05138938128948212,
      "step": 96
    },
    {
      "epoch": 1.6440677966101696,
      "grad_norm": 35.02426892448023,
      "learning_rate": 4.1101694915254236e-07,
      "logits/chosen": 9.74704360961914,
      "logits/rejected": 10.310001373291016,
      "logps/chosen": -12.350079536437988,
      "logps/rejected": -20.870689392089844,
      "loss": 0.6058,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03719270974397659,
      "rewards/margins": 0.30710333585739136,
      "rewards/rejected": -0.26991063356399536,
      "step": 97
    },
    {
      "epoch": 1.6610169491525424,
      "grad_norm": 62.01619203898106,
      "learning_rate": 4.152542372881356e-07,
      "logits/chosen": 10.409793853759766,
      "logits/rejected": 11.300240516662598,
      "logps/chosen": -17.81023406982422,
      "logps/rejected": -24.685707092285156,
      "loss": 0.591,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.07331323623657227,
      "rewards/margins": 0.28021639585494995,
      "rewards/rejected": -0.20690315961837769,
      "step": 98
    },
    {
      "epoch": 1.6779661016949152,
      "grad_norm": 33.31574704614062,
      "learning_rate": 4.194915254237288e-07,
      "logits/chosen": 16.979007720947266,
      "logits/rejected": 16.530698776245117,
      "logps/chosen": -17.609045028686523,
      "logps/rejected": -22.1740665435791,
      "loss": 0.618,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.09865903854370117,
      "rewards/margins": 0.20120608806610107,
      "rewards/rejected": -0.1025470495223999,
      "step": 99
    },
    {
      "epoch": 1.694915254237288,
      "grad_norm": 37.008794239474405,
      "learning_rate": 4.23728813559322e-07,
      "logits/chosen": 12.573817253112793,
      "logits/rejected": 12.280284881591797,
      "logps/chosen": -22.748950958251953,
      "logps/rejected": -24.286212921142578,
      "loss": 0.6158,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005324997007846832,
      "rewards/margins": 0.3028494119644165,
      "rewards/rejected": -0.30817440152168274,
      "step": 100
    },
    {
      "epoch": 1.711864406779661,
      "grad_norm": 34.22685156094786,
      "learning_rate": 4.279661016949153e-07,
      "logits/chosen": 8.986577033996582,
      "logits/rejected": 9.49609661102295,
      "logps/chosen": -16.046968460083008,
      "logps/rejected": -25.25179672241211,
      "loss": 0.5904,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.020370006561279297,
      "rewards/margins": 0.3425019681453705,
      "rewards/rejected": -0.3628719747066498,
      "step": 101
    },
    {
      "epoch": 1.7288135593220337,
      "grad_norm": 34.625932958965485,
      "learning_rate": 4.322033898305085e-07,
      "logits/chosen": 11.384140968322754,
      "logits/rejected": 11.360179901123047,
      "logps/chosen": -17.27088165283203,
      "logps/rejected": -22.397926330566406,
      "loss": 0.6012,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01983724534511566,
      "rewards/margins": 0.28960052132606506,
      "rewards/rejected": -0.2697632908821106,
      "step": 102
    },
    {
      "epoch": 1.7457627118644068,
      "grad_norm": 33.33748508633769,
      "learning_rate": 4.3644067796610167e-07,
      "logits/chosen": 14.142353057861328,
      "logits/rejected": 14.965060234069824,
      "logps/chosen": -14.294002532958984,
      "logps/rejected": -25.333721160888672,
      "loss": 0.5844,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.02870616316795349,
      "rewards/margins": 0.4857754111289978,
      "rewards/rejected": -0.4570692479610443,
      "step": 103
    },
    {
      "epoch": 1.7627118644067796,
      "grad_norm": 39.42019607658545,
      "learning_rate": 4.4067796610169486e-07,
      "logits/chosen": 12.695120811462402,
      "logits/rejected": 12.690081596374512,
      "logps/chosen": -13.946575164794922,
      "logps/rejected": -14.654245376586914,
      "loss": 0.6185,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.004654925316572189,
      "rewards/margins": 0.08387665450572968,
      "rewards/rejected": -0.08853158354759216,
      "step": 104
    },
    {
      "epoch": 1.7796610169491527,
      "grad_norm": 35.38737658039196,
      "learning_rate": 4.449152542372881e-07,
      "logits/chosen": 13.033133506774902,
      "logits/rejected": 13.914467811584473,
      "logps/chosen": -18.670257568359375,
      "logps/rejected": -28.337072372436523,
      "loss": 0.5595,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.012542501091957092,
      "rewards/margins": 0.43251490592956543,
      "rewards/rejected": -0.41997238993644714,
      "step": 105
    },
    {
      "epoch": 1.7966101694915255,
      "grad_norm": 31.47996004507577,
      "learning_rate": 4.4915254237288135e-07,
      "logits/chosen": 10.95296859741211,
      "logits/rejected": 11.121237754821777,
      "logps/chosen": -16.975109100341797,
      "logps/rejected": -25.316640853881836,
      "loss": 0.5541,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.012887578457593918,
      "rewards/margins": 0.3660765588283539,
      "rewards/rejected": -0.35318896174430847,
      "step": 106
    },
    {
      "epoch": 1.8135593220338984,
      "grad_norm": 36.40410354328316,
      "learning_rate": 4.5338983050847454e-07,
      "logits/chosen": 10.065115928649902,
      "logits/rejected": 10.710315704345703,
      "logps/chosen": -15.07365608215332,
      "logps/rejected": -26.562572479248047,
      "loss": 0.6034,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011696398258209229,
      "rewards/margins": 0.1393248438835144,
      "rewards/rejected": -0.12762844562530518,
      "step": 107
    },
    {
      "epoch": 1.8305084745762712,
      "grad_norm": 40.6534803962061,
      "learning_rate": 4.576271186440678e-07,
      "logits/chosen": 11.191512107849121,
      "logits/rejected": 12.057967185974121,
      "logps/chosen": -14.929410934448242,
      "logps/rejected": -24.907493591308594,
      "loss": 0.5917,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07028765976428986,
      "rewards/margins": 0.2717167139053345,
      "rewards/rejected": -0.34200435876846313,
      "step": 108
    },
    {
      "epoch": 1.847457627118644,
      "grad_norm": 35.71218397639331,
      "learning_rate": 4.61864406779661e-07,
      "logits/chosen": 12.564529418945312,
      "logits/rejected": 13.02096176147461,
      "logps/chosen": -10.957157135009766,
      "logps/rejected": -21.63454818725586,
      "loss": 0.5709,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1165485829114914,
      "rewards/margins": 0.4494689106941223,
      "rewards/rejected": -0.3329203128814697,
      "step": 109
    },
    {
      "epoch": 1.8644067796610169,
      "grad_norm": 33.929367751303374,
      "learning_rate": 4.661016949152542e-07,
      "logits/chosen": 8.212858200073242,
      "logits/rejected": 8.061684608459473,
      "logps/chosen": -26.250083923339844,
      "logps/rejected": -25.81487274169922,
      "loss": 0.5885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013301417231559753,
      "rewards/margins": 0.05666793882846832,
      "rewards/rejected": -0.04336652159690857,
      "step": 110
    },
    {
      "epoch": 1.8813559322033897,
      "grad_norm": 32.019942469686896,
      "learning_rate": 4.7033898305084747e-07,
      "logits/chosen": 12.412970542907715,
      "logits/rejected": 13.044310569763184,
      "logps/chosen": -14.468311309814453,
      "logps/rejected": -22.58787727355957,
      "loss": 0.5516,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0577264279127121,
      "rewards/margins": 0.42321252822875977,
      "rewards/rejected": -0.3654860854148865,
      "step": 111
    },
    {
      "epoch": 1.8983050847457628,
      "grad_norm": 35.55786831096516,
      "learning_rate": 4.7457627118644066e-07,
      "logits/chosen": 8.835311889648438,
      "logits/rejected": 9.934072494506836,
      "logps/chosen": -15.43493938446045,
      "logps/rejected": -21.87274742126465,
      "loss": 0.6047,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010160937905311584,
      "rewards/margins": 0.2345259189605713,
      "rewards/rejected": -0.2243649810552597,
      "step": 112
    },
    {
      "epoch": 1.9152542372881356,
      "grad_norm": 34.291753623304935,
      "learning_rate": 4.788135593220339e-07,
      "logits/chosen": 10.39708137512207,
      "logits/rejected": 10.226729393005371,
      "logps/chosen": -17.666790008544922,
      "logps/rejected": -19.35904312133789,
      "loss": 0.5398,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.05581867694854736,
      "rewards/margins": 0.19152891635894775,
      "rewards/rejected": -0.24734759330749512,
      "step": 113
    },
    {
      "epoch": 1.9322033898305084,
      "grad_norm": 33.6944592067948,
      "learning_rate": 4.830508474576271e-07,
      "logits/chosen": 13.301194190979004,
      "logits/rejected": 13.128783226013184,
      "logps/chosen": -21.99412727355957,
      "logps/rejected": -30.18555450439453,
      "loss": 0.5636,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.03493595868349075,
      "rewards/margins": 0.6104806661605835,
      "rewards/rejected": -0.5755447149276733,
      "step": 114
    },
    {
      "epoch": 1.9491525423728815,
      "grad_norm": 31.824406268281404,
      "learning_rate": 4.872881355932203e-07,
      "logits/chosen": 13.88214111328125,
      "logits/rejected": 13.710270881652832,
      "logps/chosen": -21.656864166259766,
      "logps/rejected": -28.074872970581055,
      "loss": 0.5808,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06213666498661041,
      "rewards/margins": 0.45388251543045044,
      "rewards/rejected": -0.5160191655158997,
      "step": 115
    },
    {
      "epoch": 1.9661016949152543,
      "grad_norm": 36.84167639719694,
      "learning_rate": 4.915254237288136e-07,
      "logits/chosen": 9.90043830871582,
      "logits/rejected": 10.525725364685059,
      "logps/chosen": -16.76810073852539,
      "logps/rejected": -24.763050079345703,
      "loss": 0.5318,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.12131154537200928,
      "rewards/margins": 0.5713745355606079,
      "rewards/rejected": -0.450063019990921,
      "step": 116
    },
    {
      "epoch": 1.9830508474576272,
      "grad_norm": 43.015222191808384,
      "learning_rate": 4.957627118644068e-07,
      "logits/chosen": 10.298654556274414,
      "logits/rejected": 10.836167335510254,
      "logps/chosen": -19.98606300354004,
      "logps/rejected": -29.31884765625,
      "loss": 0.5295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06146101653575897,
      "rewards/margins": 0.4207366108894348,
      "rewards/rejected": -0.35927557945251465,
      "step": 117
    },
    {
      "epoch": 2.0,
      "grad_norm": 33.830726998774026,
      "learning_rate": 5e-07,
      "logits/chosen": 10.221606254577637,
      "logits/rejected": 10.766395568847656,
      "logps/chosen": -17.46257209777832,
      "logps/rejected": -23.293987274169922,
      "loss": 0.5567,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.017913732677698135,
      "rewards/margins": 0.30477598309516907,
      "rewards/rejected": -0.28686225414276123,
      "step": 118
    },
    {
      "epoch": 2.016949152542373,
      "grad_norm": 31.598979188153844,
      "learning_rate": 4.99998906143358e-07,
      "logits/chosen": 7.365565776824951,
      "logits/rejected": 8.237077713012695,
      "logps/chosen": -13.421714782714844,
      "logps/rejected": -24.525100708007812,
      "loss": 0.5111,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09821398556232452,
      "rewards/margins": 0.6104830503463745,
      "rewards/rejected": -0.5122690796852112,
      "step": 119
    },
    {
      "epoch": 2.0338983050847457,
      "grad_norm": 28.9174685908656,
      "learning_rate": 4.999956245830044e-07,
      "logits/chosen": 8.895161628723145,
      "logits/rejected": 9.429058074951172,
      "logps/chosen": -16.168237686157227,
      "logps/rejected": -23.326154708862305,
      "loss": 0.5228,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0778268426656723,
      "rewards/margins": 0.5315461158752441,
      "rewards/rejected": -0.45371925830841064,
      "step": 120
    },
    {
      "epoch": 2.0508474576271185,
      "grad_norm": 32.2647349268431,
      "learning_rate": 4.999901553476555e-07,
      "logits/chosen": 11.467917442321777,
      "logits/rejected": 10.708782196044922,
      "logps/chosen": -22.234786987304688,
      "logps/rejected": -18.131317138671875,
      "loss": 0.5482,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0419849194586277,
      "rewards/margins": 0.25038909912109375,
      "rewards/rejected": -0.29237401485443115,
      "step": 121
    },
    {
      "epoch": 2.0677966101694913,
      "grad_norm": 34.7212502994755,
      "learning_rate": 4.999824984851718e-07,
      "logits/chosen": 8.989114761352539,
      "logits/rejected": 9.167284965515137,
      "logps/chosen": -21.553335189819336,
      "logps/rejected": -29.138168334960938,
      "loss": 0.5211,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06739462912082672,
      "rewards/margins": 0.38520726561546326,
      "rewards/rejected": -0.31781265139579773,
      "step": 122
    },
    {
      "epoch": 2.084745762711864,
      "grad_norm": 31.945568508747726,
      "learning_rate": 4.999726540625574e-07,
      "logits/chosen": 9.047769546508789,
      "logits/rejected": 9.909135818481445,
      "logps/chosen": -15.000102043151855,
      "logps/rejected": -26.59886932373047,
      "loss": 0.5061,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.098088338971138,
      "rewards/margins": 0.3822152018547058,
      "rewards/rejected": -0.2841268479824066,
      "step": 123
    },
    {
      "epoch": 2.1016949152542375,
      "grad_norm": 27.47436739327594,
      "learning_rate": 4.999606221659594e-07,
      "logits/chosen": 8.615792274475098,
      "logits/rejected": 8.573972702026367,
      "logps/chosen": -21.705352783203125,
      "logps/rejected": -26.503461837768555,
      "loss": 0.519,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.11717559397220612,
      "rewards/margins": 0.4941335618495941,
      "rewards/rejected": -0.3769579529762268,
      "step": 124
    },
    {
      "epoch": 2.1186440677966103,
      "grad_norm": 28.57759693816656,
      "learning_rate": 4.999464029006672e-07,
      "logits/chosen": 10.453192710876465,
      "logits/rejected": 10.818021774291992,
      "logps/chosen": -15.647000312805176,
      "logps/rejected": -22.32992935180664,
      "loss": 0.5042,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1543671041727066,
      "rewards/margins": 0.44498467445373535,
      "rewards/rejected": -0.29061758518218994,
      "step": 125
    },
    {
      "epoch": 2.135593220338983,
      "grad_norm": 30.8599676767818,
      "learning_rate": 4.999299963911115e-07,
      "logits/chosen": 9.664230346679688,
      "logits/rejected": 9.966415405273438,
      "logps/chosen": -15.471524238586426,
      "logps/rejected": -18.966232299804688,
      "loss": 0.5189,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1293969601392746,
      "rewards/margins": 0.45817211270332336,
      "rewards/rejected": -0.3287751376628876,
      "step": 126
    },
    {
      "epoch": 2.152542372881356,
      "grad_norm": 28.153225851680524,
      "learning_rate": 4.999114027808631e-07,
      "logits/chosen": 10.365848541259766,
      "logits/rejected": 10.544573783874512,
      "logps/chosen": -18.896303176879883,
      "logps/rejected": -26.077632904052734,
      "loss": 0.4968,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06244945526123047,
      "rewards/margins": 0.8896008133888245,
      "rewards/rejected": -0.8271512985229492,
      "step": 127
    },
    {
      "epoch": 2.169491525423729,
      "grad_norm": 29.077836625260247,
      "learning_rate": 4.998906222326321e-07,
      "logits/chosen": 11.856369972229004,
      "logits/rejected": 12.338065147399902,
      "logps/chosen": -19.74888801574707,
      "logps/rejected": -26.943958282470703,
      "loss": 0.5112,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18331018090248108,
      "rewards/margins": 0.8261761665344238,
      "rewards/rejected": -0.6428660154342651,
      "step": 128
    },
    {
      "epoch": 2.1864406779661016,
      "grad_norm": 30.101971609588308,
      "learning_rate": 4.99867654928266e-07,
      "logits/chosen": 9.577162742614746,
      "logits/rejected": 9.2794771194458,
      "logps/chosen": -19.739227294921875,
      "logps/rejected": -26.39651107788086,
      "loss": 0.4757,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08600222319364548,
      "rewards/margins": 0.5597529411315918,
      "rewards/rejected": -0.4737507402896881,
      "step": 129
    },
    {
      "epoch": 2.2033898305084745,
      "grad_norm": 27.97003250483979,
      "learning_rate": 4.998425010687483e-07,
      "logits/chosen": 9.935083389282227,
      "logits/rejected": 10.445450782775879,
      "logps/chosen": -18.21001434326172,
      "logps/rejected": -24.57656478881836,
      "loss": 0.5066,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.17998579144477844,
      "rewards/margins": 0.8225247859954834,
      "rewards/rejected": -0.6425389051437378,
      "step": 130
    },
    {
      "epoch": 2.2203389830508473,
      "grad_norm": 28.70117722604828,
      "learning_rate": 4.998151608741969e-07,
      "logits/chosen": 8.155485153198242,
      "logits/rejected": 8.829066276550293,
      "logps/chosen": -19.724557876586914,
      "logps/rejected": -32.21281433105469,
      "loss": 0.4693,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10095170885324478,
      "rewards/margins": 0.9277392625808716,
      "rewards/rejected": -0.8267876505851746,
      "step": 131
    },
    {
      "epoch": 2.23728813559322,
      "grad_norm": 30.67778967336796,
      "learning_rate": 4.997856345838614e-07,
      "logits/chosen": 7.782173156738281,
      "logits/rejected": 8.034370422363281,
      "logps/chosen": -15.949851036071777,
      "logps/rejected": -20.475975036621094,
      "loss": 0.5104,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03952084109187126,
      "rewards/margins": 0.616716206073761,
      "rewards/rejected": -0.577195405960083,
      "step": 132
    },
    {
      "epoch": 2.2542372881355934,
      "grad_norm": 30.80137068187178,
      "learning_rate": 4.997539224561225e-07,
      "logits/chosen": 11.369648933410645,
      "logits/rejected": 11.718732833862305,
      "logps/chosen": -13.169281005859375,
      "logps/rejected": -17.166488647460938,
      "loss": 0.4788,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1415635049343109,
      "rewards/margins": 0.5353420972824097,
      "rewards/rejected": -0.39377859234809875,
      "step": 133
    },
    {
      "epoch": 2.2711864406779663,
      "grad_norm": 31.80509416960202,
      "learning_rate": 4.99720024768488e-07,
      "logits/chosen": 8.444438934326172,
      "logits/rejected": 8.495518684387207,
      "logps/chosen": -21.16027069091797,
      "logps/rejected": -23.736560821533203,
      "loss": 0.4245,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.12956289947032928,
      "rewards/margins": 0.5659047365188599,
      "rewards/rejected": -0.4363418519496918,
      "step": 134
    },
    {
      "epoch": 2.288135593220339,
      "grad_norm": 32.67234861176432,
      "learning_rate": 4.996839418175918e-07,
      "logits/chosen": 10.731569290161133,
      "logits/rejected": 12.128774642944336,
      "logps/chosen": -16.912437438964844,
      "logps/rejected": -32.4586296081543,
      "loss": 0.4672,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.042983584105968475,
      "rewards/margins": 0.8427398800849915,
      "rewards/rejected": -0.8857234716415405,
      "step": 135
    },
    {
      "epoch": 2.305084745762712,
      "grad_norm": 27.925436246552373,
      "learning_rate": 4.996456739191904e-07,
      "logits/chosen": 11.008123397827148,
      "logits/rejected": 12.00696086883545,
      "logps/chosen": -12.97161865234375,
      "logps/rejected": -22.966861724853516,
      "loss": 0.456,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14307719469070435,
      "rewards/margins": 0.8332260847091675,
      "rewards/rejected": -0.6901488900184631,
      "step": 136
    },
    {
      "epoch": 2.3220338983050848,
      "grad_norm": 27.57837493625319,
      "learning_rate": 4.996052214081608e-07,
      "logits/chosen": 11.080045700073242,
      "logits/rejected": 10.920424461364746,
      "logps/chosen": -16.392833709716797,
      "logps/rejected": -25.646282196044922,
      "loss": 0.4793,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.03493800759315491,
      "rewards/margins": 0.7717962861061096,
      "rewards/rejected": -0.7368583083152771,
      "step": 137
    },
    {
      "epoch": 2.3389830508474576,
      "grad_norm": 30.84970177656098,
      "learning_rate": 4.995625846384966e-07,
      "logits/chosen": 12.923776626586914,
      "logits/rejected": 13.309206008911133,
      "logps/chosen": -18.304767608642578,
      "logps/rejected": -26.07758903503418,
      "loss": 0.4991,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.07253877818584442,
      "rewards/margins": 0.41423463821411133,
      "rewards/rejected": -0.3416959047317505,
      "step": 138
    },
    {
      "epoch": 2.3559322033898304,
      "grad_norm": 28.87895085350674,
      "learning_rate": 4.995177639833061e-07,
      "logits/chosen": 10.714326858520508,
      "logits/rejected": 11.218597412109375,
      "logps/chosen": -12.854103088378906,
      "logps/rejected": -20.783885955810547,
      "loss": 0.477,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.16060581803321838,
      "rewards/margins": 0.5592792630195618,
      "rewards/rejected": -0.398673415184021,
      "step": 139
    },
    {
      "epoch": 2.3728813559322033,
      "grad_norm": 30.345207712006474,
      "learning_rate": 4.994707598348084e-07,
      "logits/chosen": 9.185007095336914,
      "logits/rejected": 10.09119701385498,
      "logps/chosen": -16.681596755981445,
      "logps/rejected": -28.037395477294922,
      "loss": 0.4592,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07563964277505875,
      "rewards/margins": 0.6644760966300964,
      "rewards/rejected": -0.5888364315032959,
      "step": 140
    },
    {
      "epoch": 2.389830508474576,
      "grad_norm": 29.900980845912017,
      "learning_rate": 4.994215726043297e-07,
      "logits/chosen": 11.37492561340332,
      "logits/rejected": 12.306166648864746,
      "logps/chosen": -18.86722183227539,
      "logps/rejected": -27.48049545288086,
      "loss": 0.4599,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.03138112276792526,
      "rewards/margins": 0.8968455791473389,
      "rewards/rejected": -0.865464448928833,
      "step": 141
    },
    {
      "epoch": 2.406779661016949,
      "grad_norm": 28.184632132634288,
      "learning_rate": 4.993702027223003e-07,
      "logits/chosen": 10.223265647888184,
      "logits/rejected": 11.089990615844727,
      "logps/chosen": -16.124814987182617,
      "logps/rejected": -24.316085815429688,
      "loss": 0.4277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1649247407913208,
      "rewards/margins": 1.0841801166534424,
      "rewards/rejected": -0.9192553758621216,
      "step": 142
    },
    {
      "epoch": 2.423728813559322,
      "grad_norm": 33.65023165087564,
      "learning_rate": 4.993166506382505e-07,
      "logits/chosen": 12.015296936035156,
      "logits/rejected": 12.360418319702148,
      "logps/chosen": -11.458662033081055,
      "logps/rejected": -24.14784812927246,
      "loss": 0.4601,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12457229942083359,
      "rewards/margins": 0.8391572833061218,
      "rewards/rejected": -0.7145849466323853,
      "step": 143
    },
    {
      "epoch": 2.440677966101695,
      "grad_norm": 27.026839644788712,
      "learning_rate": 4.992609168208068e-07,
      "logits/chosen": 10.368545532226562,
      "logits/rejected": 10.083795547485352,
      "logps/chosen": -23.548145294189453,
      "logps/rejected": -22.070770263671875,
      "loss": 0.4604,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2950201630592346,
      "rewards/margins": 0.7546678781509399,
      "rewards/rejected": -0.4596477150917053,
      "step": 144
    },
    {
      "epoch": 2.457627118644068,
      "grad_norm": 26.060817977279918,
      "learning_rate": 4.992030017576875e-07,
      "logits/chosen": 10.199317932128906,
      "logits/rejected": 10.922442436218262,
      "logps/chosen": -21.961441040039062,
      "logps/rejected": -35.883018493652344,
      "loss": 0.4145,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1837531328201294,
      "rewards/margins": 1.1913630962371826,
      "rewards/rejected": -1.0076100826263428,
      "step": 145
    },
    {
      "epoch": 2.4745762711864407,
      "grad_norm": 30.504477408509327,
      "learning_rate": 4.991429059556989e-07,
      "logits/chosen": 9.347443580627441,
      "logits/rejected": 9.603809356689453,
      "logps/chosen": -18.638919830322266,
      "logps/rejected": -22.002826690673828,
      "loss": 0.4603,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0920301228761673,
      "rewards/margins": 0.9097036123275757,
      "rewards/rejected": -0.8176735043525696,
      "step": 146
    },
    {
      "epoch": 2.4915254237288136,
      "grad_norm": 26.196734416575048,
      "learning_rate": 4.990806299407305e-07,
      "logits/chosen": 8.564815521240234,
      "logits/rejected": 9.3782377243042,
      "logps/chosen": -14.529531478881836,
      "logps/rejected": -19.932464599609375,
      "loss": 0.4439,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.14292752742767334,
      "rewards/margins": 0.5559223890304565,
      "rewards/rejected": -0.4129948318004608,
      "step": 147
    },
    {
      "epoch": 2.5084745762711864,
      "grad_norm": 28.9947309009371,
      "learning_rate": 4.990161742577506e-07,
      "logits/chosen": 9.976179122924805,
      "logits/rejected": 10.230119705200195,
      "logps/chosen": -14.522598266601562,
      "logps/rejected": -24.451900482177734,
      "loss": 0.4331,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.01258106529712677,
      "rewards/margins": 0.8219106197357178,
      "rewards/rejected": -0.809329628944397,
      "step": 148
    },
    {
      "epoch": 2.5254237288135593,
      "grad_norm": 28.301260499378476,
      "learning_rate": 4.989495394708015e-07,
      "logits/chosen": 8.76701831817627,
      "logits/rejected": 9.332403182983398,
      "logps/chosen": -24.048688888549805,
      "logps/rejected": -22.38104820251465,
      "loss": 0.4394,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04988780617713928,
      "rewards/margins": 0.7242330312728882,
      "rewards/rejected": -0.6743452548980713,
      "step": 149
    },
    {
      "epoch": 2.542372881355932,
      "grad_norm": 24.87074559473272,
      "learning_rate": 4.988807261629942e-07,
      "logits/chosen": 11.337067604064941,
      "logits/rejected": 11.21312427520752,
      "logps/chosen": -18.84625244140625,
      "logps/rejected": -23.458078384399414,
      "loss": 0.4181,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.005762174725532532,
      "rewards/margins": 1.0975971221923828,
      "rewards/rejected": -1.0918350219726562,
      "step": 150
    },
    {
      "epoch": 2.559322033898305,
      "grad_norm": 25.943700529763216,
      "learning_rate": 4.988097349365039e-07,
      "logits/chosen": 8.649271965026855,
      "logits/rejected": 10.232512474060059,
      "logps/chosen": -19.395671844482422,
      "logps/rejected": -26.689014434814453,
      "loss": 0.4141,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16895315051078796,
      "rewards/margins": 0.9023088216781616,
      "rewards/rejected": -0.7333556413650513,
      "step": 151
    },
    {
      "epoch": 2.576271186440678,
      "grad_norm": 26.560543547978344,
      "learning_rate": 4.987365664125646e-07,
      "logits/chosen": 11.920557975769043,
      "logits/rejected": 12.487735748291016,
      "logps/chosen": -16.76218032836914,
      "logps/rejected": -19.075769424438477,
      "loss": 0.4424,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.09761041402816772,
      "rewards/margins": 0.5322891473770142,
      "rewards/rejected": -0.43467870354652405,
      "step": 152
    },
    {
      "epoch": 2.593220338983051,
      "grad_norm": 30.875491021836197,
      "learning_rate": 4.986612212314632e-07,
      "logits/chosen": 8.095218658447266,
      "logits/rejected": 8.630035400390625,
      "logps/chosen": -16.40205192565918,
      "logps/rejected": -29.01576805114746,
      "loss": 0.5034,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0655764490365982,
      "rewards/margins": 1.2725319862365723,
      "rewards/rejected": -1.2069554328918457,
      "step": 153
    },
    {
      "epoch": 2.610169491525424,
      "grad_norm": 29.106013409554382,
      "learning_rate": 4.985837000525343e-07,
      "logits/chosen": 11.021102905273438,
      "logits/rejected": 10.358692169189453,
      "logps/chosen": -15.820294380187988,
      "logps/rejected": -19.943557739257812,
      "loss": 0.4508,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.012390628457069397,
      "rewards/margins": 0.5218388438224792,
      "rewards/rejected": -0.5342295169830322,
      "step": 154
    },
    {
      "epoch": 2.6271186440677967,
      "grad_norm": 25.043691622956985,
      "learning_rate": 4.985040035541542e-07,
      "logits/chosen": 9.689821243286133,
      "logits/rejected": 11.35682201385498,
      "logps/chosen": -16.085561752319336,
      "logps/rejected": -28.019329071044922,
      "loss": 0.3964,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18151240050792694,
      "rewards/margins": 1.1647062301635742,
      "rewards/rejected": -0.9831939935684204,
      "step": 155
    },
    {
      "epoch": 2.6440677966101696,
      "grad_norm": 26.21644326627851,
      "learning_rate": 4.984221324337356e-07,
      "logits/chosen": 9.743395805358887,
      "logits/rejected": 11.021498680114746,
      "logps/chosen": -14.556093215942383,
      "logps/rejected": -26.45406723022461,
      "loss": 0.4057,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07891668379306793,
      "rewards/margins": 0.9213628172874451,
      "rewards/rejected": -1.0002795457839966,
      "step": 156
    },
    {
      "epoch": 2.6610169491525424,
      "grad_norm": 29.80638488183626,
      "learning_rate": 4.983380874077204e-07,
      "logits/chosen": 8.874907493591309,
      "logits/rejected": 8.924434661865234,
      "logps/chosen": -13.57056999206543,
      "logps/rejected": -17.829633712768555,
      "loss": 0.4866,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1548992097377777,
      "rewards/margins": 0.7773313522338867,
      "rewards/rejected": -0.6224321126937866,
      "step": 157
    },
    {
      "epoch": 2.6779661016949152,
      "grad_norm": 26.481655223143644,
      "learning_rate": 4.982518692115743e-07,
      "logits/chosen": 9.572946548461914,
      "logits/rejected": 10.248663902282715,
      "logps/chosen": -14.427574157714844,
      "logps/rejected": -22.97412109375,
      "loss": 0.4233,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3066959083080292,
      "rewards/margins": 1.1530815362930298,
      "rewards/rejected": -0.8463855981826782,
      "step": 158
    },
    {
      "epoch": 2.694915254237288,
      "grad_norm": 30.392105942262955,
      "learning_rate": 4.981634785997801e-07,
      "logits/chosen": 11.983063697814941,
      "logits/rejected": 12.59067440032959,
      "logps/chosen": -17.3765869140625,
      "logps/rejected": -25.998476028442383,
      "loss": 0.4306,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.10719990730285645,
      "rewards/margins": 0.8148340582847595,
      "rewards/rejected": -0.922033965587616,
      "step": 159
    },
    {
      "epoch": 2.711864406779661,
      "grad_norm": 27.76495608776217,
      "learning_rate": 4.980729163458311e-07,
      "logits/chosen": 8.870572090148926,
      "logits/rejected": 9.155264854431152,
      "logps/chosen": -20.706480026245117,
      "logps/rejected": -22.279741287231445,
      "loss": 0.4485,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08153115212917328,
      "rewards/margins": 0.7684078216552734,
      "rewards/rejected": -0.686876654624939,
      "step": 160
    },
    {
      "epoch": 2.7288135593220337,
      "grad_norm": 28.299451351438137,
      "learning_rate": 4.979801832422243e-07,
      "logits/chosen": 9.896560668945312,
      "logits/rejected": 10.647369384765625,
      "logps/chosen": -14.198293685913086,
      "logps/rejected": -23.360374450683594,
      "loss": 0.4107,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.050837576389312744,
      "rewards/margins": 0.5405330657958984,
      "rewards/rejected": -0.5913706421852112,
      "step": 161
    },
    {
      "epoch": 2.7457627118644066,
      "grad_norm": 26.33726622019465,
      "learning_rate": 4.978852801004533e-07,
      "logits/chosen": 9.247544288635254,
      "logits/rejected": 9.513944625854492,
      "logps/chosen": -14.692225456237793,
      "logps/rejected": -21.092857360839844,
      "loss": 0.4495,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.126656174659729,
      "rewards/margins": 0.5739098787307739,
      "rewards/rejected": -0.44725367426872253,
      "step": 162
    },
    {
      "epoch": 2.7627118644067794,
      "grad_norm": 27.340813928397274,
      "learning_rate": 4.977882077510018e-07,
      "logits/chosen": 9.752120971679688,
      "logits/rejected": 11.007214546203613,
      "logps/chosen": -13.380044937133789,
      "logps/rejected": -25.285293579101562,
      "loss": 0.4236,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09852191060781479,
      "rewards/margins": 1.5012712478637695,
      "rewards/rejected": -1.4027493000030518,
      "step": 163
    },
    {
      "epoch": 2.7796610169491527,
      "grad_norm": 25.850905007942515,
      "learning_rate": 4.976889670433355e-07,
      "logits/chosen": 8.906452178955078,
      "logits/rejected": 8.845396995544434,
      "logps/chosen": -23.392776489257812,
      "logps/rejected": -22.667905807495117,
      "loss": 0.3735,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09475763142108917,
      "rewards/margins": 1.1101325750350952,
      "rewards/rejected": -1.0153748989105225,
      "step": 164
    },
    {
      "epoch": 2.7966101694915255,
      "grad_norm": 25.80939076026458,
      "learning_rate": 4.975875588458953e-07,
      "logits/chosen": 11.2208833694458,
      "logits/rejected": 11.42241382598877,
      "logps/chosen": -18.877132415771484,
      "logps/rejected": -20.48996353149414,
      "loss": 0.4493,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.040170177817344666,
      "rewards/margins": 0.5747877359390259,
      "rewards/rejected": -0.6149579286575317,
      "step": 165
    },
    {
      "epoch": 2.8135593220338984,
      "grad_norm": 26.338313528073403,
      "learning_rate": 4.974839840460894e-07,
      "logits/chosen": 10.968280792236328,
      "logits/rejected": 11.683061599731445,
      "logps/chosen": -8.583036422729492,
      "logps/rejected": -19.372709274291992,
      "loss": 0.3715,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.21730396151542664,
      "rewards/margins": 1.3301661014556885,
      "rewards/rejected": -1.1128621101379395,
      "step": 166
    },
    {
      "epoch": 2.830508474576271,
      "grad_norm": 26.738734324592283,
      "learning_rate": 4.973782435502858e-07,
      "logits/chosen": 8.941149711608887,
      "logits/rejected": 10.029899597167969,
      "logps/chosen": -18.587181091308594,
      "logps/rejected": -29.493885040283203,
      "loss": 0.4358,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08800230920314789,
      "rewards/margins": 1.2188138961791992,
      "rewards/rejected": -1.3068161010742188,
      "step": 167
    },
    {
      "epoch": 2.847457627118644,
      "grad_norm": 25.571243928734713,
      "learning_rate": 4.97270338283804e-07,
      "logits/chosen": 8.122725486755371,
      "logits/rejected": 8.439363479614258,
      "logps/chosen": -12.261382102966309,
      "logps/rejected": -17.34757423400879,
      "loss": 0.4122,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1635618358850479,
      "rewards/margins": 0.41292399168014526,
      "rewards/rejected": -0.24936217069625854,
      "step": 168
    },
    {
      "epoch": 2.864406779661017,
      "grad_norm": 26.732405795819066,
      "learning_rate": 4.97160269190907e-07,
      "logits/chosen": 11.752440452575684,
      "logits/rejected": 12.06600284576416,
      "logps/chosen": -14.426056861877441,
      "logps/rejected": -20.560089111328125,
      "loss": 0.4442,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10870569944381714,
      "rewards/margins": 0.657209038734436,
      "rewards/rejected": -0.5485032796859741,
      "step": 169
    },
    {
      "epoch": 2.8813559322033897,
      "grad_norm": 37.82985353920003,
      "learning_rate": 4.970480372347933e-07,
      "logits/chosen": 10.382537841796875,
      "logits/rejected": 10.585658073425293,
      "logps/chosen": -13.967865943908691,
      "logps/rejected": -22.570960998535156,
      "loss": 0.4372,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.07470311224460602,
      "rewards/margins": 0.851300835609436,
      "rewards/rejected": -0.7765977382659912,
      "step": 170
    },
    {
      "epoch": 2.898305084745763,
      "grad_norm": 28.963422726648655,
      "learning_rate": 4.969336433975886e-07,
      "logits/chosen": 11.298312187194824,
      "logits/rejected": 12.328322410583496,
      "logps/chosen": -14.665711402893066,
      "logps/rejected": -29.004274368286133,
      "loss": 0.418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05677422136068344,
      "rewards/margins": 1.126150131225586,
      "rewards/rejected": -1.0693758726119995,
      "step": 171
    },
    {
      "epoch": 2.915254237288136,
      "grad_norm": 25.04801638294687,
      "learning_rate": 4.968170886803361e-07,
      "logits/chosen": 8.335662841796875,
      "logits/rejected": 9.045802116394043,
      "logps/chosen": -15.8262300491333,
      "logps/rejected": -25.64813804626465,
      "loss": 0.3991,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.28148353099823,
      "rewards/margins": 0.9742909073829651,
      "rewards/rejected": -0.6928073167800903,
      "step": 172
    },
    {
      "epoch": 2.9322033898305087,
      "grad_norm": 27.65516270956161,
      "learning_rate": 4.966983741029893e-07,
      "logits/chosen": 7.764376163482666,
      "logits/rejected": 7.787277698516846,
      "logps/chosen": -17.45782470703125,
      "logps/rejected": -28.34349250793457,
      "loss": 0.3965,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.016701295971870422,
      "rewards/margins": 0.8749082684516907,
      "rewards/rejected": -0.8582069277763367,
      "step": 173
    },
    {
      "epoch": 2.9491525423728815,
      "grad_norm": 27.335474568003058,
      "learning_rate": 4.965775007044019e-07,
      "logits/chosen": 9.209973335266113,
      "logits/rejected": 10.886381149291992,
      "logps/chosen": -17.583267211914062,
      "logps/rejected": -26.150978088378906,
      "loss": 0.3574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3135643005371094,
      "rewards/margins": 1.6306042671203613,
      "rewards/rejected": -1.3170397281646729,
      "step": 174
    },
    {
      "epoch": 2.9661016949152543,
      "grad_norm": 27.369409853348216,
      "learning_rate": 4.964544695423193e-07,
      "logits/chosen": 9.178210258483887,
      "logits/rejected": 9.29438304901123,
      "logps/chosen": -13.898372650146484,
      "logps/rejected": -18.40739631652832,
      "loss": 0.4116,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16570331156253815,
      "rewards/margins": 0.9794048070907593,
      "rewards/rejected": -0.8137016296386719,
      "step": 175
    },
    {
      "epoch": 2.983050847457627,
      "grad_norm": 27.08632667503853,
      "learning_rate": 4.963292816933691e-07,
      "logits/chosen": 7.745831489562988,
      "logits/rejected": 8.334796905517578,
      "logps/chosen": -19.82839012145996,
      "logps/rejected": -26.07370376586914,
      "loss": 0.425,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10253079980611801,
      "rewards/margins": 0.9082773923873901,
      "rewards/rejected": -0.8057465553283691,
      "step": 176
    },
    {
      "epoch": 3.0,
      "grad_norm": 29.11312831933624,
      "learning_rate": 4.96201938253052e-07,
      "logits/chosen": 8.67463207244873,
      "logits/rejected": 8.769617080688477,
      "logps/chosen": -19.712432861328125,
      "logps/rejected": -29.56153106689453,
      "loss": 0.4492,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.009731769561767578,
      "rewards/margins": 1.0045030117034912,
      "rewards/rejected": -1.0142347812652588,
      "step": 177
    },
    {
      "epoch": 3.016949152542373,
      "grad_norm": 22.70341027925011,
      "learning_rate": 4.960724403357314e-07,
      "logits/chosen": 9.667706489562988,
      "logits/rejected": 9.530271530151367,
      "logps/chosen": -14.369274139404297,
      "logps/rejected": -23.56820297241211,
      "loss": 0.3558,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0965251475572586,
      "rewards/margins": 1.0754098892211914,
      "rewards/rejected": -0.9788846969604492,
      "step": 178
    },
    {
      "epoch": 3.0338983050847457,
      "grad_norm": 22.012454849538084,
      "learning_rate": 4.959407890746248e-07,
      "logits/chosen": 8.7022123336792,
      "logits/rejected": 8.974809646606445,
      "logps/chosen": -14.963942527770996,
      "logps/rejected": -24.484926223754883,
      "loss": 0.3825,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15482452511787415,
      "rewards/margins": 1.3257172107696533,
      "rewards/rejected": -1.1708927154541016,
      "step": 179
    },
    {
      "epoch": 3.0508474576271185,
      "grad_norm": 21.871293164666387,
      "learning_rate": 4.958069856217929e-07,
      "logits/chosen": 10.28845500946045,
      "logits/rejected": 9.987916946411133,
      "logps/chosen": -14.2538480758667,
      "logps/rejected": -19.900346755981445,
      "loss": 0.3447,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1215810775756836,
      "rewards/margins": 1.0722662210464478,
      "rewards/rejected": -0.9506851434707642,
      "step": 180
    },
    {
      "epoch": 3.0677966101694913,
      "grad_norm": 24.133472988977683,
      "learning_rate": 4.956710311481302e-07,
      "logits/chosen": 7.17443323135376,
      "logits/rejected": 7.678508281707764,
      "logps/chosen": -14.993780136108398,
      "logps/rejected": -25.741018295288086,
      "loss": 0.372,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11443566530942917,
      "rewards/margins": 1.444988489151001,
      "rewards/rejected": -1.3305529356002808,
      "step": 181
    },
    {
      "epoch": 3.084745762711864,
      "grad_norm": 24.14418324234125,
      "learning_rate": 4.955329268433542e-07,
      "logits/chosen": 6.731716156005859,
      "logits/rejected": 7.250002861022949,
      "logps/chosen": -18.923925399780273,
      "logps/rejected": -23.595857620239258,
      "loss": 0.3804,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.052950143814086914,
      "rewards/margins": 1.2683604955673218,
      "rewards/rejected": -1.3213107585906982,
      "step": 182
    },
    {
      "epoch": 3.1016949152542375,
      "grad_norm": 23.24825847333165,
      "learning_rate": 4.953926739159956e-07,
      "logits/chosen": 10.804813385009766,
      "logits/rejected": 11.33828067779541,
      "logps/chosen": -18.222864151000977,
      "logps/rejected": -26.791757583618164,
      "loss": 0.3568,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.029467307031154633,
      "rewards/margins": 1.3857711553573608,
      "rewards/rejected": -1.3563038110733032,
      "step": 183
    },
    {
      "epoch": 3.1186440677966103,
      "grad_norm": 23.624561156451218,
      "learning_rate": 4.952502735933869e-07,
      "logits/chosen": 7.371200084686279,
      "logits/rejected": 8.635663986206055,
      "logps/chosen": -16.824323654174805,
      "logps/rejected": -30.131465911865234,
      "loss": 0.3962,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.26038092374801636,
      "rewards/margins": 1.377197027206421,
      "rewards/rejected": -1.1168160438537598,
      "step": 184
    },
    {
      "epoch": 3.135593220338983,
      "grad_norm": 23.0617388330513,
      "learning_rate": 4.951057271216525e-07,
      "logits/chosen": 7.263133525848389,
      "logits/rejected": 8.648643493652344,
      "logps/chosen": -14.994542121887207,
      "logps/rejected": -27.363725662231445,
      "loss": 0.3559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04362674802541733,
      "rewards/margins": 0.8833621144294739,
      "rewards/rejected": -0.8397355079650879,
      "step": 185
    },
    {
      "epoch": 3.152542372881356,
      "grad_norm": 24.060190276119712,
      "learning_rate": 4.949590357656974e-07,
      "logits/chosen": 10.067876815795898,
      "logits/rejected": 10.368553161621094,
      "logps/chosen": -18.041959762573242,
      "logps/rejected": -30.241945266723633,
      "loss": 0.3332,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.26056158542633057,
      "rewards/margins": 1.1221503019332886,
      "rewards/rejected": -0.8615886569023132,
      "step": 186
    },
    {
      "epoch": 3.169491525423729,
      "grad_norm": 21.596576629334546,
      "learning_rate": 4.948102008091962e-07,
      "logits/chosen": 8.81076431274414,
      "logits/rejected": 9.11528205871582,
      "logps/chosen": -16.617202758789062,
      "logps/rejected": -25.771406173706055,
      "loss": 0.3458,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04481735825538635,
      "rewards/margins": 1.1783089637756348,
      "rewards/rejected": -1.1334917545318604,
      "step": 187
    },
    {
      "epoch": 3.1864406779661016,
      "grad_norm": 23.319342589454678,
      "learning_rate": 4.946592235545815e-07,
      "logits/chosen": 6.276343822479248,
      "logits/rejected": 7.280974388122559,
      "logps/chosen": -25.35882568359375,
      "logps/rejected": -31.435470581054688,
      "loss": 0.3524,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13174700736999512,
      "rewards/margins": 1.1706089973449707,
      "rewards/rejected": -1.0388619899749756,
      "step": 188
    },
    {
      "epoch": 3.2033898305084745,
      "grad_norm": 24.334730171525816,
      "learning_rate": 4.945061053230333e-07,
      "logits/chosen": 6.7825822830200195,
      "logits/rejected": 7.950262069702148,
      "logps/chosen": -19.224178314208984,
      "logps/rejected": -34.59772872924805,
      "loss": 0.3604,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.13449785113334656,
      "rewards/margins": 1.6669206619262695,
      "rewards/rejected": -1.5324227809906006,
      "step": 189
    },
    {
      "epoch": 3.2203389830508473,
      "grad_norm": 21.339498583686392,
      "learning_rate": 4.943508474544666e-07,
      "logits/chosen": 7.709277629852295,
      "logits/rejected": 8.046525001525879,
      "logps/chosen": -11.371763229370117,
      "logps/rejected": -21.591466903686523,
      "loss": 0.3242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2736116051673889,
      "rewards/margins": 1.5210593938827515,
      "rewards/rejected": -1.2474478483200073,
      "step": 190
    },
    {
      "epoch": 3.23728813559322,
      "grad_norm": 21.53073528694387,
      "learning_rate": 4.941934513075204e-07,
      "logits/chosen": 6.447499752044678,
      "logits/rejected": 7.401057720184326,
      "logps/chosen": -24.705169677734375,
      "logps/rejected": -27.498699188232422,
      "loss": 0.325,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24285036325454712,
      "rewards/margins": 1.7285178899765015,
      "rewards/rejected": -1.4856674671173096,
      "step": 191
    },
    {
      "epoch": 3.2542372881355934,
      "grad_norm": 20.761722419548054,
      "learning_rate": 4.94033918259545e-07,
      "logits/chosen": 11.910299301147461,
      "logits/rejected": 11.728422164916992,
      "logps/chosen": -14.798544883728027,
      "logps/rejected": -23.423885345458984,
      "loss": 0.3595,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3081997036933899,
      "rewards/margins": 1.2061128616333008,
      "rewards/rejected": -0.8979132175445557,
      "step": 192
    },
    {
      "epoch": 3.2711864406779663,
      "grad_norm": 22.02765764075306,
      "learning_rate": 4.938722497065909e-07,
      "logits/chosen": 9.125901222229004,
      "logits/rejected": 9.558362007141113,
      "logps/chosen": -16.233884811401367,
      "logps/rejected": -20.573768615722656,
      "loss": 0.3395,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1345757693052292,
      "rewards/margins": 1.391578197479248,
      "rewards/rejected": -1.257002592086792,
      "step": 193
    },
    {
      "epoch": 3.288135593220339,
      "grad_norm": 23.161966059680974,
      "learning_rate": 4.937084470633958e-07,
      "logits/chosen": 5.163336753845215,
      "logits/rejected": 5.996748447418213,
      "logps/chosen": -19.100372314453125,
      "logps/rejected": -24.20922088623047,
      "loss": 0.3378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20368611812591553,
      "rewards/margins": 1.46111261844635,
      "rewards/rejected": -1.2574265003204346,
      "step": 194
    },
    {
      "epoch": 3.305084745762712,
      "grad_norm": 29.992204991018934,
      "learning_rate": 4.935425117633726e-07,
      "logits/chosen": 9.052857398986816,
      "logits/rejected": 8.928924560546875,
      "logps/chosen": -15.425498962402344,
      "logps/rejected": -21.158906936645508,
      "loss": 0.3513,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.12333399057388306,
      "rewards/margins": 0.8104948401451111,
      "rewards/rejected": -0.687160849571228,
      "step": 195
    },
    {
      "epoch": 3.3220338983050848,
      "grad_norm": 22.847542397883526,
      "learning_rate": 4.933744452585966e-07,
      "logits/chosen": 7.322593688964844,
      "logits/rejected": 7.687209129333496,
      "logps/chosen": -14.15726375579834,
      "logps/rejected": -21.306930541992188,
      "loss": 0.371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2583553194999695,
      "rewards/margins": 1.3991026878356934,
      "rewards/rejected": -1.1407474279403687,
      "step": 196
    },
    {
      "epoch": 3.3389830508474576,
      "grad_norm": 22.873574991222572,
      "learning_rate": 4.932042490197933e-07,
      "logits/chosen": 7.417664051055908,
      "logits/rejected": 7.723471641540527,
      "logps/chosen": -24.805866241455078,
      "logps/rejected": -26.682741165161133,
      "loss": 0.3619,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.21670067310333252,
      "rewards/margins": 1.4350523948669434,
      "rewards/rejected": -1.2183517217636108,
      "step": 197
    },
    {
      "epoch": 3.3559322033898304,
      "grad_norm": 24.56769186369395,
      "learning_rate": 4.930319245363248e-07,
      "logits/chosen": 9.720744132995605,
      "logits/rejected": 10.568902015686035,
      "logps/chosen": -10.5819091796875,
      "logps/rejected": -24.263601303100586,
      "loss": 0.3567,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09235741198062897,
      "rewards/margins": 1.4750256538391113,
      "rewards/rejected": -1.3826682567596436,
      "step": 198
    },
    {
      "epoch": 3.3728813559322033,
      "grad_norm": 23.233797057466784,
      "learning_rate": 4.928574733161775e-07,
      "logits/chosen": 11.545450210571289,
      "logits/rejected": 12.52687931060791,
      "logps/chosen": -16.03731918334961,
      "logps/rejected": -26.412817001342773,
      "loss": 0.3599,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.010396137833595276,
      "rewards/margins": 1.0698754787445068,
      "rewards/rejected": -1.0594793558120728,
      "step": 199
    },
    {
      "epoch": 3.389830508474576,
      "grad_norm": 20.139690046673707,
      "learning_rate": 4.926808968859483e-07,
      "logits/chosen": 5.908224105834961,
      "logits/rejected": 6.287415981292725,
      "logps/chosen": -15.103047370910645,
      "logps/rejected": -20.081340789794922,
      "loss": 0.2854,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.27523258328437805,
      "rewards/margins": 1.3817555904388428,
      "rewards/rejected": -1.1065233945846558,
      "step": 200
    },
    {
      "epoch": 3.406779661016949,
      "grad_norm": 23.94893902926556,
      "learning_rate": 4.925021967908316e-07,
      "logits/chosen": 9.42378044128418,
      "logits/rejected": 10.111106872558594,
      "logps/chosen": -12.101956367492676,
      "logps/rejected": -23.762901306152344,
      "loss": 0.3612,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05877889692783356,
      "rewards/margins": 0.8194180727005005,
      "rewards/rejected": -0.8781968951225281,
      "step": 201
    },
    {
      "epoch": 3.423728813559322,
      "grad_norm": 20.9327603969309,
      "learning_rate": 4.923213745946059e-07,
      "logits/chosen": 8.8514986038208,
      "logits/rejected": 10.125353813171387,
      "logps/chosen": -12.044666290283203,
      "logps/rejected": -28.45659637451172,
      "loss": 0.3327,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.20615214109420776,
      "rewards/margins": 1.9639978408813477,
      "rewards/rejected": -1.7578457593917847,
      "step": 202
    },
    {
      "epoch": 3.440677966101695,
      "grad_norm": 23.527829904491835,
      "learning_rate": 4.921384318796193e-07,
      "logits/chosen": 7.121918678283691,
      "logits/rejected": 8.05240249633789,
      "logps/chosen": -19.336584091186523,
      "logps/rejected": -23.552419662475586,
      "loss": 0.3402,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11570586264133453,
      "rewards/margins": 1.2959182262420654,
      "rewards/rejected": -1.180212378501892,
      "step": 203
    },
    {
      "epoch": 3.457627118644068,
      "grad_norm": 19.749222830827275,
      "learning_rate": 4.919533702467771e-07,
      "logits/chosen": 6.370368480682373,
      "logits/rejected": 7.441535472869873,
      "logps/chosen": -16.786794662475586,
      "logps/rejected": -27.888469696044922,
      "loss": 0.2868,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.22812122106552124,
      "rewards/margins": 2.133049488067627,
      "rewards/rejected": -1.9049283266067505,
      "step": 204
    },
    {
      "epoch": 3.4745762711864407,
      "grad_norm": 22.127296567931705,
      "learning_rate": 4.91766191315526e-07,
      "logits/chosen": 8.205850601196289,
      "logits/rejected": 8.747001647949219,
      "logps/chosen": -21.583003997802734,
      "logps/rejected": -30.38113021850586,
      "loss": 0.319,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.11991026997566223,
      "rewards/margins": 1.666430950164795,
      "rewards/rejected": -1.546520709991455,
      "step": 205
    },
    {
      "epoch": 3.4915254237288136,
      "grad_norm": 27.870162386057064,
      "learning_rate": 4.915768967238417e-07,
      "logits/chosen": 10.85152816772461,
      "logits/rejected": 11.19784164428711,
      "logps/chosen": -16.432479858398438,
      "logps/rejected": -20.672014236450195,
      "loss": 0.3583,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.08660467714071274,
      "rewards/margins": 0.7723907828330994,
      "rewards/rejected": -0.6857861280441284,
      "step": 206
    },
    {
      "epoch": 3.5084745762711864,
      "grad_norm": 21.105675455485688,
      "learning_rate": 4.913854881282131e-07,
      "logits/chosen": 9.626623153686523,
      "logits/rejected": 10.561471939086914,
      "logps/chosen": -13.190305709838867,
      "logps/rejected": -27.12738800048828,
      "loss": 0.3139,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.09586920589208603,
      "rewards/margins": 1.6659499406814575,
      "rewards/rejected": -1.5700807571411133,
      "step": 207
    },
    {
      "epoch": 3.5254237288135593,
      "grad_norm": 21.947799986422574,
      "learning_rate": 4.91191967203629e-07,
      "logits/chosen": 8.077613830566406,
      "logits/rejected": 8.295317649841309,
      "logps/chosen": -15.712980270385742,
      "logps/rejected": -27.68941879272461,
      "loss": 0.3188,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1660003513097763,
      "rewards/margins": 1.33102548122406,
      "rewards/rejected": -1.165025234222412,
      "step": 208
    },
    {
      "epoch": 3.542372881355932,
      "grad_norm": 21.54421619455441,
      "learning_rate": 4.909963356435624e-07,
      "logits/chosen": 7.969567775726318,
      "logits/rejected": 8.85141658782959,
      "logps/chosen": -11.287786483764648,
      "logps/rejected": -26.90688133239746,
      "loss": 0.3329,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15023496747016907,
      "rewards/margins": 2.192362070083618,
      "rewards/rejected": -2.0421271324157715,
      "step": 209
    },
    {
      "epoch": 3.559322033898305,
      "grad_norm": 23.389960730472435,
      "learning_rate": 4.907985951599563e-07,
      "logits/chosen": 9.967187881469727,
      "logits/rejected": 10.385942459106445,
      "logps/chosen": -15.627347946166992,
      "logps/rejected": -25.37283706665039,
      "loss": 0.3403,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.147053062915802,
      "rewards/margins": 1.3340773582458496,
      "rewards/rejected": -1.481130599975586,
      "step": 210
    },
    {
      "epoch": 3.576271186440678,
      "grad_norm": 20.386193598695463,
      "learning_rate": 4.905987474832087e-07,
      "logits/chosen": 6.471014499664307,
      "logits/rejected": 8.414436340332031,
      "logps/chosen": -19.985212326049805,
      "logps/rejected": -30.850446701049805,
      "loss": 0.3001,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.35499030351638794,
      "rewards/margins": 1.4340555667877197,
      "rewards/rejected": -1.079065203666687,
      "step": 211
    },
    {
      "epoch": 3.593220338983051,
      "grad_norm": 19.86096413206925,
      "learning_rate": 4.903967943621573e-07,
      "logits/chosen": 7.104785919189453,
      "logits/rejected": 6.808341979980469,
      "logps/chosen": -21.855426788330078,
      "logps/rejected": -26.186508178710938,
      "loss": 0.2714,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.16494721174240112,
      "rewards/margins": 1.7304604053497314,
      "rewards/rejected": -1.5655131340026855,
      "step": 212
    },
    {
      "epoch": 3.610169491525424,
      "grad_norm": 19.972878939684275,
      "learning_rate": 4.901927375640642e-07,
      "logits/chosen": 7.536691188812256,
      "logits/rejected": 8.184225082397461,
      "logps/chosen": -17.092975616455078,
      "logps/rejected": -27.26826286315918,
      "loss": 0.3071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1841067671775818,
      "rewards/margins": 1.6666710376739502,
      "rewards/rejected": -1.4825642108917236,
      "step": 213
    },
    {
      "epoch": 3.6271186440677967,
      "grad_norm": 20.907070751662914,
      "learning_rate": 4.899865788746005e-07,
      "logits/chosen": 6.0391011238098145,
      "logits/rejected": 6.892394065856934,
      "logps/chosen": -19.163902282714844,
      "logps/rejected": -22.660507202148438,
      "loss": 0.3244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2965112328529358,
      "rewards/margins": 2.102067470550537,
      "rewards/rejected": -1.8055561780929565,
      "step": 214
    },
    {
      "epoch": 3.6440677966101696,
      "grad_norm": 18.794923971592116,
      "learning_rate": 4.897783200978305e-07,
      "logits/chosen": 6.708567142486572,
      "logits/rejected": 8.018110275268555,
      "logps/chosen": -17.882518768310547,
      "logps/rejected": -25.35689353942871,
      "loss": 0.2939,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06995850056409836,
      "rewards/margins": 1.7058115005493164,
      "rewards/rejected": -1.6358528137207031,
      "step": 215
    },
    {
      "epoch": 3.6610169491525424,
      "grad_norm": 22.113554379144286,
      "learning_rate": 4.895679630561963e-07,
      "logits/chosen": 8.934822082519531,
      "logits/rejected": 9.462247848510742,
      "logps/chosen": -13.299782752990723,
      "logps/rejected": -22.16495704650879,
      "loss": 0.3266,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2283388376235962,
      "rewards/margins": 1.3375617265701294,
      "rewards/rejected": -1.1092228889465332,
      "step": 216
    },
    {
      "epoch": 3.6779661016949152,
      "grad_norm": 22.146172855918522,
      "learning_rate": 4.893555095905013e-07,
      "logits/chosen": 4.558166027069092,
      "logits/rejected": 5.124869346618652,
      "logps/chosen": -23.284637451171875,
      "logps/rejected": -31.29628562927246,
      "loss": 0.3132,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2708703279495239,
      "rewards/margins": 1.6527708768844604,
      "rewards/rejected": -1.3819005489349365,
      "step": 217
    },
    {
      "epoch": 3.694915254237288,
      "grad_norm": 20.522636710839876,
      "learning_rate": 4.891409615598949e-07,
      "logits/chosen": 7.233565807342529,
      "logits/rejected": 7.809101581573486,
      "logps/chosen": -17.282150268554688,
      "logps/rejected": -21.183683395385742,
      "loss": 0.3113,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.11144751310348511,
      "rewards/margins": 1.5106884241104126,
      "rewards/rejected": -1.3992410898208618,
      "step": 218
    },
    {
      "epoch": 3.711864406779661,
      "grad_norm": 19.979629024830988,
      "learning_rate": 4.889243208418549e-07,
      "logits/chosen": 7.521520614624023,
      "logits/rejected": 7.7953314781188965,
      "logps/chosen": -14.246883392333984,
      "logps/rejected": -23.500028610229492,
      "loss": 0.3296,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.27966731786727905,
      "rewards/margins": 1.612604022026062,
      "rewards/rejected": -1.3329367637634277,
      "step": 219
    },
    {
      "epoch": 3.7288135593220337,
      "grad_norm": 21.796523646749833,
      "learning_rate": 4.88705589332173e-07,
      "logits/chosen": 9.764825820922852,
      "logits/rejected": 9.947137832641602,
      "logps/chosen": -13.865436553955078,
      "logps/rejected": -18.25220489501953,
      "loss": 0.3321,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22284789383411407,
      "rewards/margins": 0.9826896786689758,
      "rewards/rejected": -0.7598418593406677,
      "step": 220
    },
    {
      "epoch": 3.7457627118644066,
      "grad_norm": 24.185750088279356,
      "learning_rate": 4.884847689449361e-07,
      "logits/chosen": 9.337504386901855,
      "logits/rejected": 10.483132362365723,
      "logps/chosen": -19.560300827026367,
      "logps/rejected": -31.120304107666016,
      "loss": 0.3651,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04826126992702484,
      "rewards/margins": 1.525299310684204,
      "rewards/rejected": -1.477038025856018,
      "step": 221
    },
    {
      "epoch": 3.7627118644067794,
      "grad_norm": 20.57645440620922,
      "learning_rate": 4.88261861612511e-07,
      "logits/chosen": 7.008860111236572,
      "logits/rejected": 7.396633625030518,
      "logps/chosen": -19.443256378173828,
      "logps/rejected": -21.94890594482422,
      "loss": 0.2738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2511497139930725,
      "rewards/margins": 1.7133121490478516,
      "rewards/rejected": -1.4621624946594238,
      "step": 222
    },
    {
      "epoch": 3.7796610169491527,
      "grad_norm": 42.36323422895596,
      "learning_rate": 4.880368692855273e-07,
      "logits/chosen": 5.257360935211182,
      "logits/rejected": 6.08449649810791,
      "logps/chosen": -24.111492156982422,
      "logps/rejected": -28.312646865844727,
      "loss": 0.3026,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22682160139083862,
      "rewards/margins": 1.9936718940734863,
      "rewards/rejected": -1.766850233078003,
      "step": 223
    },
    {
      "epoch": 3.7966101694915255,
      "grad_norm": 21.57536476218224,
      "learning_rate": 4.878097939328596e-07,
      "logits/chosen": 10.081559181213379,
      "logits/rejected": 9.927122116088867,
      "logps/chosen": -17.91533088684082,
      "logps/rejected": -24.432580947875977,
      "loss": 0.3221,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.07144062221050262,
      "rewards/margins": 1.6415483951568604,
      "rewards/rejected": -1.712989091873169,
      "step": 224
    },
    {
      "epoch": 3.8135593220338984,
      "grad_norm": 22.918857955226564,
      "learning_rate": 4.875806375416109e-07,
      "logits/chosen": 9.115406036376953,
      "logits/rejected": 9.300793647766113,
      "logps/chosen": -22.0445499420166,
      "logps/rejected": -24.975872039794922,
      "loss": 0.3145,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.03266003727912903,
      "rewards/margins": 1.8592348098754883,
      "rewards/rejected": -1.8265748023986816,
      "step": 225
    },
    {
      "epoch": 3.830508474576271,
      "grad_norm": 22.493511216167683,
      "learning_rate": 4.873494021170954e-07,
      "logits/chosen": 7.798854351043701,
      "logits/rejected": 8.126043319702148,
      "logps/chosen": -15.902989387512207,
      "logps/rejected": -21.789674758911133,
      "loss": 0.3353,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.045627400279045105,
      "rewards/margins": 1.554894208908081,
      "rewards/rejected": -1.5092668533325195,
      "step": 226
    },
    {
      "epoch": 3.847457627118644,
      "grad_norm": 20.36579881861922,
      "learning_rate": 4.871160896828199e-07,
      "logits/chosen": 6.323782444000244,
      "logits/rejected": 7.2763142585754395,
      "logps/chosen": -20.429576873779297,
      "logps/rejected": -27.979249954223633,
      "loss": 0.3174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04189814627170563,
      "rewards/margins": 1.8435866832733154,
      "rewards/rejected": -1.801688551902771,
      "step": 227
    },
    {
      "epoch": 3.864406779661017,
      "grad_norm": 24.774639537029763,
      "learning_rate": 4.868807022804678e-07,
      "logits/chosen": 9.863037109375,
      "logits/rejected": 11.282732963562012,
      "logps/chosen": -18.87766456604004,
      "logps/rejected": -35.58513259887695,
      "loss": 0.3485,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.13600066304206848,
      "rewards/margins": 2.0307259559631348,
      "rewards/rejected": -2.16672682762146,
      "step": 228
    },
    {
      "epoch": 3.8813559322033897,
      "grad_norm": 22.966813457902724,
      "learning_rate": 4.866432419698792e-07,
      "logits/chosen": 5.345111846923828,
      "logits/rejected": 5.739473342895508,
      "logps/chosen": -14.497096061706543,
      "logps/rejected": -19.52205467224121,
      "loss": 0.3202,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11341910064220428,
      "rewards/margins": 1.1143789291381836,
      "rewards/rejected": -1.000959873199463,
      "step": 229
    },
    {
      "epoch": 3.898305084745763,
      "grad_norm": 20.655199996849774,
      "learning_rate": 4.864037108290347e-07,
      "logits/chosen": 7.576197624206543,
      "logits/rejected": 8.149455070495605,
      "logps/chosen": -13.794957160949707,
      "logps/rejected": -35.497589111328125,
      "loss": 0.3118,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09785757213830948,
      "rewards/margins": 2.4956488609313965,
      "rewards/rejected": -2.3977913856506348,
      "step": 230
    },
    {
      "epoch": 3.915254237288136,
      "grad_norm": 22.031674230554742,
      "learning_rate": 4.86162110954036e-07,
      "logits/chosen": 5.174468040466309,
      "logits/rejected": 5.88414192199707,
      "logps/chosen": -14.31143569946289,
      "logps/rejected": -18.397750854492188,
      "loss": 0.3161,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.34032052755355835,
      "rewards/margins": 0.8846842646598816,
      "rewards/rejected": -0.5443637371063232,
      "step": 231
    },
    {
      "epoch": 3.9322033898305087,
      "grad_norm": 21.411141660381265,
      "learning_rate": 4.859184444590881e-07,
      "logits/chosen": 9.678922653198242,
      "logits/rejected": 10.129996299743652,
      "logps/chosen": -15.969707489013672,
      "logps/rejected": -21.065452575683594,
      "loss": 0.2914,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13621875643730164,
      "rewards/margins": 0.9995884895324707,
      "rewards/rejected": -0.8633697032928467,
      "step": 232
    },
    {
      "epoch": 3.9491525423728815,
      "grad_norm": 23.327719096582616,
      "learning_rate": 4.856727134764809e-07,
      "logits/chosen": 6.851404190063477,
      "logits/rejected": 7.505338668823242,
      "logps/chosen": -13.464282035827637,
      "logps/rejected": -23.67028045654297,
      "loss": 0.3166,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0010511577129364014,
      "rewards/margins": 1.9928821325302124,
      "rewards/rejected": -1.9939334392547607,
      "step": 233
    },
    {
      "epoch": 3.9661016949152543,
      "grad_norm": 22.34328312174226,
      "learning_rate": 4.8542492015657e-07,
      "logits/chosen": 11.095281600952148,
      "logits/rejected": 11.888551712036133,
      "logps/chosen": -18.534252166748047,
      "logps/rejected": -27.535181045532227,
      "loss": 0.3389,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1114778071641922,
      "rewards/margins": 2.0294349193573,
      "rewards/rejected": -1.9179573059082031,
      "step": 234
    },
    {
      "epoch": 3.983050847457627,
      "grad_norm": 22.460277454690832,
      "learning_rate": 4.851750666677583e-07,
      "logits/chosen": 7.924036026000977,
      "logits/rejected": 8.230151176452637,
      "logps/chosen": -15.009639739990234,
      "logps/rejected": -24.392051696777344,
      "loss": 0.3365,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17272941768169403,
      "rewards/margins": 1.2254295349121094,
      "rewards/rejected": -1.0527000427246094,
      "step": 235
    },
    {
      "epoch": 4.0,
      "grad_norm": 22.886063239469433,
      "learning_rate": 4.849231551964771e-07,
      "logits/chosen": 5.907500267028809,
      "logits/rejected": 6.998715877532959,
      "logps/chosen": -20.924211502075195,
      "logps/rejected": -26.191280364990234,
      "loss": 0.2911,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.018931731581687927,
      "rewards/margins": 2.093313217163086,
      "rewards/rejected": -2.1122450828552246,
      "step": 236
    },
    {
      "epoch": 4.016949152542373,
      "grad_norm": 16.11228313336153,
      "learning_rate": 4.846691879471666e-07,
      "logits/chosen": 6.13948917388916,
      "logits/rejected": 7.0086822509765625,
      "logps/chosen": -21.371328353881836,
      "logps/rejected": -26.916461944580078,
      "loss": 0.218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2001882642507553,
      "rewards/margins": 2.128901481628418,
      "rewards/rejected": -1.928713321685791,
      "step": 237
    },
    {
      "epoch": 4.033898305084746,
      "grad_norm": 19.987499022105034,
      "learning_rate": 4.844131671422569e-07,
      "logits/chosen": 6.478044033050537,
      "logits/rejected": 6.766986846923828,
      "logps/chosen": -16.220653533935547,
      "logps/rejected": -19.514848709106445,
      "loss": 0.3162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3637089133262634,
      "rewards/margins": 1.3370646238327026,
      "rewards/rejected": -0.9733555912971497,
      "step": 238
    },
    {
      "epoch": 4.0508474576271185,
      "grad_norm": 18.688925017295187,
      "learning_rate": 4.841550950221485e-07,
      "logits/chosen": 7.050149440765381,
      "logits/rejected": 7.562372207641602,
      "logps/chosen": -20.11605453491211,
      "logps/rejected": -26.2134952545166,
      "loss": 0.2711,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16213461756706238,
      "rewards/margins": 1.712576985359192,
      "rewards/rejected": -1.5504424571990967,
      "step": 239
    },
    {
      "epoch": 4.067796610169491,
      "grad_norm": 18.92762993148661,
      "learning_rate": 4.838949738451928e-07,
      "logits/chosen": 7.53150749206543,
      "logits/rejected": 7.950225830078125,
      "logps/chosen": -19.729782104492188,
      "logps/rejected": -30.533475875854492,
      "loss": 0.2874,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2349463552236557,
      "rewards/margins": 2.648629665374756,
      "rewards/rejected": -2.4136834144592285,
      "step": 240
    },
    {
      "epoch": 4.084745762711864,
      "grad_norm": 18.885623463835536,
      "learning_rate": 4.836328058876717e-07,
      "logits/chosen": 7.093875885009766,
      "logits/rejected": 8.201071739196777,
      "logps/chosen": -17.072683334350586,
      "logps/rejected": -24.49260902404785,
      "loss": 0.2928,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.017010435461997986,
      "rewards/margins": 1.231659173965454,
      "rewards/rejected": -1.214648723602295,
      "step": 241
    },
    {
      "epoch": 4.101694915254237,
      "grad_norm": 18.79125519072356,
      "learning_rate": 4.833685934437787e-07,
      "logits/chosen": 6.813934803009033,
      "logits/rejected": 7.2115678787231445,
      "logps/chosen": -17.01702117919922,
      "logps/rejected": -29.53050422668457,
      "loss": 0.2546,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4310588240623474,
      "rewards/margins": 1.8695611953735352,
      "rewards/rejected": -1.438502311706543,
      "step": 242
    },
    {
      "epoch": 4.11864406779661,
      "grad_norm": 16.887866681775407,
      "learning_rate": 4.831023388255979e-07,
      "logits/chosen": 5.699078559875488,
      "logits/rejected": 6.792886734008789,
      "logps/chosen": -18.427654266357422,
      "logps/rejected": -24.72570037841797,
      "loss": 0.2358,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15313753485679626,
      "rewards/margins": 2.126829147338867,
      "rewards/rejected": -1.973691701889038,
      "step": 243
    },
    {
      "epoch": 4.135593220338983,
      "grad_norm": 17.68436179219868,
      "learning_rate": 4.828340443630846e-07,
      "logits/chosen": 6.9619526863098145,
      "logits/rejected": 7.495880126953125,
      "logps/chosen": -16.34537124633789,
      "logps/rejected": -22.424455642700195,
      "loss": 0.3041,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.48435744643211365,
      "rewards/margins": 1.6667134761810303,
      "rewards/rejected": -1.1823559999465942,
      "step": 244
    },
    {
      "epoch": 4.1525423728813555,
      "grad_norm": 19.718183477610275,
      "learning_rate": 4.825637124040441e-07,
      "logits/chosen": 9.444064140319824,
      "logits/rejected": 9.68178653717041,
      "logps/chosen": -17.95220947265625,
      "logps/rejected": -29.515003204345703,
      "loss": 0.2785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34623271226882935,
      "rewards/margins": 1.8744040727615356,
      "rewards/rejected": -1.5281713008880615,
      "step": 245
    },
    {
      "epoch": 4.169491525423728,
      "grad_norm": 19.424536492921106,
      "learning_rate": 4.822913453141117e-07,
      "logits/chosen": 7.507587432861328,
      "logits/rejected": 7.846378326416016,
      "logps/chosen": -15.093152046203613,
      "logps/rejected": -24.109743118286133,
      "loss": 0.2887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22552460432052612,
      "rewards/margins": 1.9814910888671875,
      "rewards/rejected": -1.7559665441513062,
      "step": 246
    },
    {
      "epoch": 4.186440677966102,
      "grad_norm": 19.10137459530579,
      "learning_rate": 4.820169454767318e-07,
      "logits/chosen": 8.067806243896484,
      "logits/rejected": 9.25924301147461,
      "logps/chosen": -16.840450286865234,
      "logps/rejected": -28.04216957092285,
      "loss": 0.2616,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35099655389785767,
      "rewards/margins": 2.54416561126709,
      "rewards/rejected": -2.193168878555298,
      "step": 247
    },
    {
      "epoch": 4.203389830508475,
      "grad_norm": 18.565041100888553,
      "learning_rate": 4.81740515293137e-07,
      "logits/chosen": 8.024762153625488,
      "logits/rejected": 8.494616508483887,
      "logps/chosen": -17.160768508911133,
      "logps/rejected": -26.250389099121094,
      "loss": 0.2831,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26058316230773926,
      "rewards/margins": 1.5284281969070435,
      "rewards/rejected": -1.2678451538085938,
      "step": 248
    },
    {
      "epoch": 4.220338983050848,
      "grad_norm": 17.696717834540067,
      "learning_rate": 4.814620571823274e-07,
      "logits/chosen": 5.288139343261719,
      "logits/rejected": 5.49987268447876,
      "logps/chosen": -21.20608901977539,
      "logps/rejected": -30.10845947265625,
      "loss": 0.284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.29643139243125916,
      "rewards/margins": 2.4248037338256836,
      "rewards/rejected": -2.1283726692199707,
      "step": 249
    },
    {
      "epoch": 4.237288135593221,
      "grad_norm": 16.08433980916575,
      "learning_rate": 4.811815735810489e-07,
      "logits/chosen": 6.320197105407715,
      "logits/rejected": 7.34105110168457,
      "logps/chosen": -18.912696838378906,
      "logps/rejected": -30.96662139892578,
      "loss": 0.2491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30656683444976807,
      "rewards/margins": 2.7686104774475098,
      "rewards/rejected": -2.462043285369873,
      "step": 250
    },
    {
      "epoch": 4.254237288135593,
      "grad_norm": 16.436104483894702,
      "learning_rate": 4.808990669437724e-07,
      "logits/chosen": 3.874683380126953,
      "logits/rejected": 4.041537761688232,
      "logps/chosen": -13.296972274780273,
      "logps/rejected": -26.234445571899414,
      "loss": 0.2308,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20190800726413727,
      "rewards/margins": 2.277385711669922,
      "rewards/rejected": -2.0754778385162354,
      "step": 251
    },
    {
      "epoch": 4.271186440677966,
      "grad_norm": 19.96329758104426,
      "learning_rate": 4.806145397426719e-07,
      "logits/chosen": 8.5095853805542,
      "logits/rejected": 8.628890991210938,
      "logps/chosen": -11.131439208984375,
      "logps/rejected": -20.39238166809082,
      "loss": 0.277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16035281121730804,
      "rewards/margins": 2.0513343811035156,
      "rewards/rejected": -1.890981674194336,
      "step": 252
    },
    {
      "epoch": 4.288135593220339,
      "grad_norm": 18.10784435727316,
      "learning_rate": 4.803279944676032e-07,
      "logits/chosen": 7.8867902755737305,
      "logits/rejected": 7.66014289855957,
      "logps/chosen": -11.152649879455566,
      "logps/rejected": -24.569089889526367,
      "loss": 0.2508,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1465606689453125,
      "rewards/margins": 1.89396071434021,
      "rewards/rejected": -1.747400164604187,
      "step": 253
    },
    {
      "epoch": 4.305084745762712,
      "grad_norm": 16.848798055022613,
      "learning_rate": 4.800394336260819e-07,
      "logits/chosen": 3.822847366333008,
      "logits/rejected": 5.139825820922852,
      "logps/chosen": -14.96919059753418,
      "logps/rejected": -27.31922149658203,
      "loss": 0.2692,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23743754625320435,
      "rewards/margins": 1.99288010597229,
      "rewards/rejected": -1.755442500114441,
      "step": 254
    },
    {
      "epoch": 4.322033898305085,
      "grad_norm": 17.12136208102501,
      "learning_rate": 4.797488597432616e-07,
      "logits/chosen": 4.065765380859375,
      "logits/rejected": 5.0273637771606445,
      "logps/chosen": -24.291616439819336,
      "logps/rejected": -23.00619888305664,
      "loss": 0.2292,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.25025373697280884,
      "rewards/margins": 1.9274500608444214,
      "rewards/rejected": -1.6771961450576782,
      "step": 255
    },
    {
      "epoch": 4.338983050847458,
      "grad_norm": 17.427062960507172,
      "learning_rate": 4.794562753619117e-07,
      "logits/chosen": 6.049924373626709,
      "logits/rejected": 5.929304122924805,
      "logps/chosen": -18.5858211517334,
      "logps/rejected": -23.01677703857422,
      "loss": 0.2702,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24605974555015564,
      "rewards/margins": 1.5662342309951782,
      "rewards/rejected": -1.3201744556427002,
      "step": 256
    },
    {
      "epoch": 4.3559322033898304,
      "grad_norm": 15.920672408874744,
      "learning_rate": 4.791616830423949e-07,
      "logits/chosen": 7.087856292724609,
      "logits/rejected": 7.3572678565979,
      "logps/chosen": -17.611812591552734,
      "logps/rejected": -23.177900314331055,
      "loss": 0.2335,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2877931594848633,
      "rewards/margins": 1.9493603706359863,
      "rewards/rejected": -1.6615670919418335,
      "step": 257
    },
    {
      "epoch": 4.372881355932203,
      "grad_norm": 16.27906455725465,
      "learning_rate": 4.788650853626456e-07,
      "logits/chosen": 4.602998733520508,
      "logits/rejected": 5.435512542724609,
      "logps/chosen": -18.157915115356445,
      "logps/rejected": -23.088634490966797,
      "loss": 0.2488,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42691361904144287,
      "rewards/margins": 2.0973167419433594,
      "rewards/rejected": -1.6704033613204956,
      "step": 258
    },
    {
      "epoch": 4.389830508474576,
      "grad_norm": 15.402435870137548,
      "learning_rate": 4.785664849181465e-07,
      "logits/chosen": 8.592238426208496,
      "logits/rejected": 9.519624710083008,
      "logps/chosen": -14.045475959777832,
      "logps/rejected": -23.337627410888672,
      "loss": 0.2581,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.038690753281116486,
      "rewards/margins": 1.2892788648605347,
      "rewards/rejected": -1.250588297843933,
      "step": 259
    },
    {
      "epoch": 4.406779661016949,
      "grad_norm": 16.299035928417254,
      "learning_rate": 4.78265884321906e-07,
      "logits/chosen": 5.048234939575195,
      "logits/rejected": 5.757306098937988,
      "logps/chosen": -14.810338973999023,
      "logps/rejected": -26.654769897460938,
      "loss": 0.264,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16818922758102417,
      "rewards/margins": 1.9285526275634766,
      "rewards/rejected": -1.7603633403778076,
      "step": 260
    },
    {
      "epoch": 4.423728813559322,
      "grad_norm": 17.024561674564683,
      "learning_rate": 4.779632862044361e-07,
      "logits/chosen": 5.772840976715088,
      "logits/rejected": 6.783274173736572,
      "logps/chosen": -17.9024715423584,
      "logps/rejected": -30.046398162841797,
      "loss": 0.2566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1782139241695404,
      "rewards/margins": 2.617436408996582,
      "rewards/rejected": -2.439222574234009,
      "step": 261
    },
    {
      "epoch": 4.440677966101695,
      "grad_norm": 17.896887917576542,
      "learning_rate": 4.776586932137283e-07,
      "logits/chosen": 8.3196382522583,
      "logits/rejected": 8.486194610595703,
      "logps/chosen": -18.671096801757812,
      "logps/rejected": -23.115036010742188,
      "loss": 0.262,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.37524986267089844,
      "rewards/margins": 1.8184455633163452,
      "rewards/rejected": -1.4431955814361572,
      "step": 262
    },
    {
      "epoch": 4.4576271186440675,
      "grad_norm": 18.02282203660666,
      "learning_rate": 4.773521080152311e-07,
      "logits/chosen": 5.8606696128845215,
      "logits/rejected": 6.721665382385254,
      "logps/chosen": -19.343944549560547,
      "logps/rejected": -32.12931442260742,
      "loss": 0.2426,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.45170512795448303,
      "rewards/margins": 2.2819502353668213,
      "rewards/rejected": -1.830245018005371,
      "step": 263
    },
    {
      "epoch": 4.47457627118644,
      "grad_norm": 16.54309012883112,
      "learning_rate": 4.770435332918267e-07,
      "logits/chosen": 6.439443588256836,
      "logits/rejected": 7.319515228271484,
      "logps/chosen": -18.117504119873047,
      "logps/rejected": -26.829072952270508,
      "loss": 0.2231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21523486077785492,
      "rewards/margins": 2.130596876144409,
      "rewards/rejected": -1.915362000465393,
      "step": 264
    },
    {
      "epoch": 4.491525423728813,
      "grad_norm": 17.419485402954614,
      "learning_rate": 4.76732971743807e-07,
      "logits/chosen": 7.095752716064453,
      "logits/rejected": 8.249744415283203,
      "logps/chosen": -15.102618217468262,
      "logps/rejected": -24.421607971191406,
      "loss": 0.2272,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3142382502555847,
      "rewards/margins": 2.8125152587890625,
      "rewards/rejected": -2.498276710510254,
      "step": 265
    },
    {
      "epoch": 4.508474576271187,
      "grad_norm": 18.562995753543007,
      "learning_rate": 4.7642042608885056e-07,
      "logits/chosen": 7.712843894958496,
      "logits/rejected": 8.154708862304688,
      "logps/chosen": -21.448400497436523,
      "logps/rejected": -29.982311248779297,
      "loss": 0.2839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.047302231192588806,
      "rewards/margins": 2.0715363025665283,
      "rewards/rejected": -2.0242340564727783,
      "step": 266
    },
    {
      "epoch": 4.52542372881356,
      "grad_norm": 17.86786243504552,
      "learning_rate": 4.761058990619986e-07,
      "logits/chosen": 5.361347675323486,
      "logits/rejected": 5.248828411102295,
      "logps/chosen": -18.133102416992188,
      "logps/rejected": -25.32653045654297,
      "loss": 0.2412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06450856477022171,
      "rewards/margins": 2.018827438354492,
      "rewards/rejected": -1.954318881034851,
      "step": 267
    },
    {
      "epoch": 4.5423728813559325,
      "grad_norm": 16.209189141288928,
      "learning_rate": 4.757893934156309e-07,
      "logits/chosen": 7.7050018310546875,
      "logits/rejected": 8.658272743225098,
      "logps/chosen": -15.19764232635498,
      "logps/rejected": -30.116540908813477,
      "loss": 0.2379,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.209306001663208,
      "rewards/margins": 2.709834575653076,
      "rewards/rejected": -2.500528573989868,
      "step": 268
    },
    {
      "epoch": 4.559322033898305,
      "grad_norm": 17.963829609972713,
      "learning_rate": 4.754709119194418e-07,
      "logits/chosen": 3.150035858154297,
      "logits/rejected": 3.4155702590942383,
      "logps/chosen": -16.21509552001953,
      "logps/rejected": -31.73999786376953,
      "loss": 0.2436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1859074831008911,
      "rewards/margins": 2.52579402923584,
      "rewards/rejected": -2.339886426925659,
      "step": 269
    },
    {
      "epoch": 4.576271186440678,
      "grad_norm": 18.147760096125833,
      "learning_rate": 4.7515045736041615e-07,
      "logits/chosen": 7.926346778869629,
      "logits/rejected": 9.348649024963379,
      "logps/chosen": -12.561721801757812,
      "logps/rejected": -29.205713272094727,
      "loss": 0.2694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32173627614974976,
      "rewards/margins": 2.6866581439971924,
      "rewards/rejected": -2.364922046661377,
      "step": 270
    },
    {
      "epoch": 4.593220338983051,
      "grad_norm": 16.51138027595473,
      "learning_rate": 4.748280325428048e-07,
      "logits/chosen": 10.498759269714355,
      "logits/rejected": 10.979084014892578,
      "logps/chosen": -17.349008560180664,
      "logps/rejected": -32.83536911010742,
      "loss": 0.2308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.198812797665596,
      "rewards/margins": 2.9069724082946777,
      "rewards/rejected": -2.7081594467163086,
      "step": 271
    },
    {
      "epoch": 4.610169491525424,
      "grad_norm": 17.18774465901935,
      "learning_rate": 4.745036402880999e-07,
      "logits/chosen": 3.7919700145721436,
      "logits/rejected": 4.364020347595215,
      "logps/chosen": -13.981499671936035,
      "logps/rejected": -24.345117568969727,
      "loss": 0.2668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21096213161945343,
      "rewards/margins": 2.6246230602264404,
      "rewards/rejected": -2.413661003112793,
      "step": 272
    },
    {
      "epoch": 4.627118644067797,
      "grad_norm": 18.245124366119207,
      "learning_rate": 4.741772834350104e-07,
      "logits/chosen": 6.9369378089904785,
      "logits/rejected": 7.000739574432373,
      "logps/chosen": -18.30843734741211,
      "logps/rejected": -26.522764205932617,
      "loss": 0.269,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.11697474122047424,
      "rewards/margins": 1.2881109714508057,
      "rewards/rejected": -1.1711363792419434,
      "step": 273
    },
    {
      "epoch": 4.6440677966101696,
      "grad_norm": 18.848939003698714,
      "learning_rate": 4.7384896483943726e-07,
      "logits/chosen": 4.844106197357178,
      "logits/rejected": 5.648234844207764,
      "logps/chosen": -11.51663589477539,
      "logps/rejected": -27.013900756835938,
      "loss": 0.2596,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3477116823196411,
      "rewards/margins": 2.468505859375,
      "rewards/rejected": -2.1207942962646484,
      "step": 274
    },
    {
      "epoch": 4.661016949152542,
      "grad_norm": 21.485542587702177,
      "learning_rate": 4.7351868737444825e-07,
      "logits/chosen": 9.079914093017578,
      "logits/rejected": 9.508895874023438,
      "logps/chosen": -14.13361930847168,
      "logps/rejected": -27.242555618286133,
      "loss": 0.3004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02514958381652832,
      "rewards/margins": 1.8603540658950806,
      "rewards/rejected": -1.8855036497116089,
      "step": 275
    },
    {
      "epoch": 4.677966101694915,
      "grad_norm": 19.670219619103424,
      "learning_rate": 4.7318645393025305e-07,
      "logits/chosen": 8.693038940429688,
      "logits/rejected": 9.391683578491211,
      "logps/chosen": -16.673078536987305,
      "logps/rejected": -21.615997314453125,
      "loss": 0.2778,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.019453592598438263,
      "rewards/margins": 1.658519983291626,
      "rewards/rejected": -1.6390663385391235,
      "step": 276
    },
    {
      "epoch": 4.694915254237288,
      "grad_norm": 16.137053552384206,
      "learning_rate": 4.7285226741417753e-07,
      "logits/chosen": 4.7442708015441895,
      "logits/rejected": 5.013760566711426,
      "logps/chosen": -19.061439514160156,
      "logps/rejected": -25.69558334350586,
      "loss": 0.2495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4043675363063812,
      "rewards/margins": 2.4568257331848145,
      "rewards/rejected": -2.0524580478668213,
      "step": 277
    },
    {
      "epoch": 4.711864406779661,
      "grad_norm": 17.739725289410767,
      "learning_rate": 4.7251613075063905e-07,
      "logits/chosen": 5.071934700012207,
      "logits/rejected": 6.222379684448242,
      "logps/chosen": -14.962982177734375,
      "logps/rejected": -25.778844833374023,
      "loss": 0.2445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3471611440181732,
      "rewards/margins": 2.654113531112671,
      "rewards/rejected": -2.306952476501465,
      "step": 278
    },
    {
      "epoch": 4.728813559322034,
      "grad_norm": 16.298662722486412,
      "learning_rate": 4.721780468811201e-07,
      "logits/chosen": 5.887434482574463,
      "logits/rejected": 6.384895324707031,
      "logps/chosen": -19.674272537231445,
      "logps/rejected": -23.1964111328125,
      "loss": 0.2212,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.23275047540664673,
      "rewards/margins": 1.6070315837860107,
      "rewards/rejected": -1.3742811679840088,
      "step": 279
    },
    {
      "epoch": 4.745762711864407,
      "grad_norm": 15.906115484751137,
      "learning_rate": 4.7183801876414286e-07,
      "logits/chosen": 7.360743045806885,
      "logits/rejected": 8.02312183380127,
      "logps/chosen": -15.797630310058594,
      "logps/rejected": -25.812213897705078,
      "loss": 0.2118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.017660900950431824,
      "rewards/margins": 2.3710832595825195,
      "rewards/rejected": -2.3534226417541504,
      "step": 280
    },
    {
      "epoch": 4.762711864406779,
      "grad_norm": 15.677627452398355,
      "learning_rate": 4.7149604937524356e-07,
      "logits/chosen": 3.6860146522521973,
      "logits/rejected": 4.597846031188965,
      "logps/chosen": -23.743751525878906,
      "logps/rejected": -34.87071228027344,
      "loss": 0.2401,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.40262383222579956,
      "rewards/margins": 2.2429513931274414,
      "rewards/rejected": -1.8403273820877075,
      "step": 281
    },
    {
      "epoch": 4.779661016949152,
      "grad_norm": 14.517687160323698,
      "learning_rate": 4.7115214170694616e-07,
      "logits/chosen": 5.888266563415527,
      "logits/rejected": 6.816344261169434,
      "logps/chosen": -15.760833740234375,
      "logps/rejected": -31.780643463134766,
      "loss": 0.2055,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04766558110713959,
      "rewards/margins": 2.9352927207946777,
      "rewards/rejected": -2.887627124786377,
      "step": 282
    },
    {
      "epoch": 4.796610169491525,
      "grad_norm": 18.000705159355824,
      "learning_rate": 4.70806298768736e-07,
      "logits/chosen": 7.001628398895264,
      "logits/rejected": 6.544361114501953,
      "logps/chosen": -16.813600540161133,
      "logps/rejected": -20.177221298217773,
      "loss": 0.2368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17308537662029266,
      "rewards/margins": 2.580078125,
      "rewards/rejected": -2.4069926738739014,
      "step": 283
    },
    {
      "epoch": 4.813559322033898,
      "grad_norm": 17.74785330269053,
      "learning_rate": 4.70458523587034e-07,
      "logits/chosen": 4.586512565612793,
      "logits/rejected": 5.216480255126953,
      "logps/chosen": -21.734012603759766,
      "logps/rejected": -38.459041595458984,
      "loss": 0.2345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13362450897693634,
      "rewards/margins": 2.968825578689575,
      "rewards/rejected": -2.8352010250091553,
      "step": 284
    },
    {
      "epoch": 4.830508474576272,
      "grad_norm": 17.69313020536604,
      "learning_rate": 4.701088192051695e-07,
      "logits/chosen": 4.594993591308594,
      "logits/rejected": 5.305399417877197,
      "logps/chosen": -19.230627059936523,
      "logps/rejected": -29.320083618164062,
      "loss": 0.2494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4186445474624634,
      "rewards/margins": 2.536355972290039,
      "rewards/rejected": -2.117711305618286,
      "step": 285
    },
    {
      "epoch": 4.847457627118644,
      "grad_norm": 15.219068766771,
      "learning_rate": 4.697571886833543e-07,
      "logits/chosen": 6.026544094085693,
      "logits/rejected": 6.7140607833862305,
      "logps/chosen": -15.21136474609375,
      "logps/rejected": -27.875221252441406,
      "loss": 0.215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03304705023765564,
      "rewards/margins": 2.2568628787994385,
      "rewards/rejected": -2.22381591796875,
      "step": 286
    },
    {
      "epoch": 4.864406779661017,
      "grad_norm": 15.548775123675558,
      "learning_rate": 4.6940363509865553e-07,
      "logits/chosen": 8.8172607421875,
      "logits/rejected": 9.029597282409668,
      "logps/chosen": -17.15589714050293,
      "logps/rejected": -25.47644805908203,
      "loss": 0.2311,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.38684672117233276,
      "rewards/margins": 2.146179676055908,
      "rewards/rejected": -1.7593328952789307,
      "step": 287
    },
    {
      "epoch": 4.88135593220339,
      "grad_norm": 16.621647617560352,
      "learning_rate": 4.6904816154496854e-07,
      "logits/chosen": 7.06538200378418,
      "logits/rejected": 8.518985748291016,
      "logps/chosen": -16.530170440673828,
      "logps/rejected": -25.592021942138672,
      "loss": 0.2418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1437515765428543,
      "rewards/margins": 2.5299487113952637,
      "rewards/rejected": -2.386197328567505,
      "step": 288
    },
    {
      "epoch": 4.898305084745763,
      "grad_norm": 18.20879716146644,
      "learning_rate": 4.6869077113299025e-07,
      "logits/chosen": 4.580350399017334,
      "logits/rejected": 5.193227767944336,
      "logps/chosen": -16.725788116455078,
      "logps/rejected": -27.487756729125977,
      "loss": 0.2752,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07869633287191391,
      "rewards/margins": 1.970123052597046,
      "rewards/rejected": -2.0488195419311523,
      "step": 289
    },
    {
      "epoch": 4.915254237288136,
      "grad_norm": 17.56550072326665,
      "learning_rate": 4.6833146699019177e-07,
      "logits/chosen": 7.187812328338623,
      "logits/rejected": 7.607518196105957,
      "logps/chosen": -15.682883262634277,
      "logps/rejected": -30.07357406616211,
      "loss": 0.272,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13676121830940247,
      "rewards/margins": 2.355900287628174,
      "rewards/rejected": -2.2191390991210938,
      "step": 290
    },
    {
      "epoch": 4.932203389830509,
      "grad_norm": 17.319715186835847,
      "learning_rate": 4.6797025226079074e-07,
      "logits/chosen": 8.21790885925293,
      "logits/rejected": 8.874201774597168,
      "logps/chosen": -16.58701515197754,
      "logps/rejected": -28.231704711914062,
      "loss": 0.2398,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.043866753578186035,
      "rewards/margins": 2.017036199569702,
      "rewards/rejected": -2.0609028339385986,
      "step": 291
    },
    {
      "epoch": 4.9491525423728815,
      "grad_norm": 17.60150177581467,
      "learning_rate": 4.676071301057243e-07,
      "logits/chosen": 4.695080757141113,
      "logits/rejected": 5.318776607513428,
      "logps/chosen": -15.804220199584961,
      "logps/rejected": -26.266937255859375,
      "loss": 0.262,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24425703287124634,
      "rewards/margins": 1.4589682817459106,
      "rewards/rejected": -1.214711308479309,
      "step": 292
    },
    {
      "epoch": 4.966101694915254,
      "grad_norm": 18.261705360854126,
      "learning_rate": 4.67242103702621e-07,
      "logits/chosen": 5.390717506408691,
      "logits/rejected": 6.542966842651367,
      "logps/chosen": -16.86713981628418,
      "logps/rejected": -26.163740158081055,
      "loss": 0.2747,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0493212565779686,
      "rewards/margins": 1.8711323738098145,
      "rewards/rejected": -1.8218111991882324,
      "step": 293
    },
    {
      "epoch": 4.983050847457627,
      "grad_norm": 16.247074569406255,
      "learning_rate": 4.668751762457733e-07,
      "logits/chosen": 6.2724609375,
      "logits/rejected": 6.9650468826293945,
      "logps/chosen": -14.587836265563965,
      "logps/rejected": -28.363473892211914,
      "loss": 0.2201,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.029139414429664612,
      "rewards/margins": 2.09466814994812,
      "rewards/rejected": -2.0655288696289062,
      "step": 294
    },
    {
      "epoch": 5.0,
      "grad_norm": 15.163845862231213,
      "learning_rate": 4.6650635094610966e-07,
      "logits/chosen": 6.010763168334961,
      "logits/rejected": 7.175766468048096,
      "logps/chosen": -15.043150901794434,
      "logps/rejected": -26.033666610717773,
      "loss": 0.2364,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40571334958076477,
      "rewards/margins": 2.3270514011383057,
      "rewards/rejected": -1.9213382005691528,
      "step": 295
    },
    {
      "epoch": 5.016949152542373,
      "grad_norm": 12.995315182536677,
      "learning_rate": 4.661356310311659e-07,
      "logits/chosen": 6.765958786010742,
      "logits/rejected": 7.221011161804199,
      "logps/chosen": -17.7678165435791,
      "logps/rejected": -30.081592559814453,
      "loss": 0.208,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16127540171146393,
      "rewards/margins": 2.3699498176574707,
      "rewards/rejected": -2.5312252044677734,
      "step": 296
    },
    {
      "epoch": 5.033898305084746,
      "grad_norm": 16.281282112966135,
      "learning_rate": 4.657630197450576e-07,
      "logits/chosen": 4.995595455169678,
      "logits/rejected": 6.14824104309082,
      "logps/chosen": -27.272743225097656,
      "logps/rejected": -37.27703857421875,
      "loss": 0.2147,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.37089285254478455,
      "rewards/margins": 3.0824501514434814,
      "rewards/rejected": -2.711557149887085,
      "step": 297
    },
    {
      "epoch": 5.0508474576271185,
      "grad_norm": 15.038788736271039,
      "learning_rate": 4.653885203484515e-07,
      "logits/chosen": 4.113032817840576,
      "logits/rejected": 4.685057163238525,
      "logps/chosen": -13.144742965698242,
      "logps/rejected": -19.484567642211914,
      "loss": 0.2289,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3587087392807007,
      "rewards/margins": 1.6604740619659424,
      "rewards/rejected": -1.3017653226852417,
      "step": 298
    },
    {
      "epoch": 5.067796610169491,
      "grad_norm": 13.341197053744226,
      "learning_rate": 4.6501213611853673e-07,
      "logits/chosen": 3.0600595474243164,
      "logits/rejected": 3.6969590187072754,
      "logps/chosen": -14.055743217468262,
      "logps/rejected": -30.396642684936523,
      "loss": 0.1703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2175249606370926,
      "rewards/margins": 2.968029022216797,
      "rewards/rejected": -2.750504493713379,
      "step": 299
    },
    {
      "epoch": 5.084745762711864,
      "grad_norm": 15.64944020606352,
      "learning_rate": 4.6463387034899643e-07,
      "logits/chosen": 7.053459167480469,
      "logits/rejected": 6.929786682128906,
      "logps/chosen": -17.727378845214844,
      "logps/rejected": -26.59695816040039,
      "loss": 0.2295,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2067881077528,
      "rewards/margins": 2.4637019634246826,
      "rewards/rejected": -2.256913661956787,
      "step": 300
    },
    {
      "epoch": 5.101694915254237,
      "grad_norm": 13.927587181216817,
      "learning_rate": 4.642537263499788e-07,
      "logits/chosen": 5.323790550231934,
      "logits/rejected": 5.755880832672119,
      "logps/chosen": -12.35572338104248,
      "logps/rejected": -24.77527618408203,
      "loss": 0.2085,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3975515365600586,
      "rewards/margins": 2.341681957244873,
      "rewards/rejected": -1.9441306591033936,
      "step": 301
    },
    {
      "epoch": 5.11864406779661,
      "grad_norm": 13.003919286175359,
      "learning_rate": 4.6387170744806813e-07,
      "logits/chosen": 4.886084079742432,
      "logits/rejected": 6.434139728546143,
      "logps/chosen": -17.88004493713379,
      "logps/rejected": -33.52429962158203,
      "loss": 0.2086,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4249918758869171,
      "rewards/margins": 3.037893772125244,
      "rewards/rejected": -2.6129019260406494,
      "step": 302
    },
    {
      "epoch": 5.135593220338983,
      "grad_norm": 13.709462601856833,
      "learning_rate": 4.634878169862557e-07,
      "logits/chosen": 6.236474514007568,
      "logits/rejected": 6.070339202880859,
      "logps/chosen": -18.325437545776367,
      "logps/rejected": -27.6246337890625,
      "loss": 0.2171,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09506276994943619,
      "rewards/margins": 1.819183588027954,
      "rewards/rejected": -1.7241206169128418,
      "step": 303
    },
    {
      "epoch": 5.1525423728813555,
      "grad_norm": 13.530622153939468,
      "learning_rate": 4.6310205832391065e-07,
      "logits/chosen": 4.541669845581055,
      "logits/rejected": 5.271422863006592,
      "logps/chosen": -18.641277313232422,
      "logps/rejected": -29.24665069580078,
      "loss": 0.1922,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12702220678329468,
      "rewards/margins": 3.098491668701172,
      "rewards/rejected": -2.9714694023132324,
      "step": 304
    },
    {
      "epoch": 5.169491525423728,
      "grad_norm": 16.78342955862146,
      "learning_rate": 4.6271443483675027e-07,
      "logits/chosen": 5.560043811798096,
      "logits/rejected": 6.474727630615234,
      "logps/chosen": -15.253793716430664,
      "logps/rejected": -22.487646102905273,
      "loss": 0.2217,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13450895249843597,
      "rewards/margins": 1.9386663436889648,
      "rewards/rejected": -1.8041574954986572,
      "step": 305
    },
    {
      "epoch": 5.186440677966102,
      "grad_norm": 13.925990324124848,
      "learning_rate": 4.6232494991681087e-07,
      "logits/chosen": 4.304049015045166,
      "logits/rejected": 5.471685886383057,
      "logps/chosen": -16.470518112182617,
      "logps/rejected": -29.901363372802734,
      "loss": 0.1896,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.024986281991004944,
      "rewards/margins": 2.9503417015075684,
      "rewards/rejected": -2.9253554344177246,
      "step": 306
    },
    {
      "epoch": 5.203389830508475,
      "grad_norm": 12.898539468992048,
      "learning_rate": 4.6193360697241766e-07,
      "logits/chosen": 6.212376594543457,
      "logits/rejected": 6.92727518081665,
      "logps/chosen": -19.12429428100586,
      "logps/rejected": -32.73984146118164,
      "loss": 0.1965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17453734576702118,
      "rewards/margins": 3.106781482696533,
      "rewards/rejected": -2.932243585586548,
      "step": 307
    },
    {
      "epoch": 5.220338983050848,
      "grad_norm": 14.083485689677397,
      "learning_rate": 4.615404094281554e-07,
      "logits/chosen": 5.328009605407715,
      "logits/rejected": 6.5234055519104,
      "logps/chosen": -16.867034912109375,
      "logps/rejected": -28.303224563598633,
      "loss": 0.1882,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20804297924041748,
      "rewards/margins": 2.509683847427368,
      "rewards/rejected": -2.301640748977661,
      "step": 308
    },
    {
      "epoch": 5.237288135593221,
      "grad_norm": 13.457404135592146,
      "learning_rate": 4.611453607248381e-07,
      "logits/chosen": 4.071807384490967,
      "logits/rejected": 5.283750534057617,
      "logps/chosen": -17.891639709472656,
      "logps/rejected": -26.934532165527344,
      "loss": 0.1876,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4134616255760193,
      "rewards/margins": 3.161255121231079,
      "rewards/rejected": -2.747793674468994,
      "step": 309
    },
    {
      "epoch": 5.254237288135593,
      "grad_norm": 14.350604886951919,
      "learning_rate": 4.607484643194788e-07,
      "logits/chosen": 5.258482456207275,
      "logits/rejected": 5.556510925292969,
      "logps/chosen": -15.743051528930664,
      "logps/rejected": -24.031320571899414,
      "loss": 0.2009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.37243759632110596,
      "rewards/margins": 2.6536245346069336,
      "rewards/rejected": -2.281187057495117,
      "step": 310
    },
    {
      "epoch": 5.271186440677966,
      "grad_norm": 14.91827512482227,
      "learning_rate": 4.6034972368525957e-07,
      "logits/chosen": 5.340587139129639,
      "logits/rejected": 6.861990451812744,
      "logps/chosen": -15.216270446777344,
      "logps/rejected": -30.178709030151367,
      "loss": 0.2054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19152489304542542,
      "rewards/margins": 2.885601282119751,
      "rewards/rejected": -3.0771260261535645,
      "step": 311
    },
    {
      "epoch": 5.288135593220339,
      "grad_norm": 15.236909598130472,
      "learning_rate": 4.599491423115014e-07,
      "logits/chosen": 5.011383056640625,
      "logits/rejected": 5.633141040802002,
      "logps/chosen": -16.395797729492188,
      "logps/rejected": -29.717737197875977,
      "loss": 0.206,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19525401294231415,
      "rewards/margins": 2.0301461219787598,
      "rewards/rejected": -1.8348920345306396,
      "step": 312
    },
    {
      "epoch": 5.305084745762712,
      "grad_norm": 13.545653395969534,
      "learning_rate": 4.595467237036329e-07,
      "logits/chosen": 5.080275535583496,
      "logits/rejected": 5.903102397918701,
      "logps/chosen": -14.001216888427734,
      "logps/rejected": -21.97933006286621,
      "loss": 0.2029,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3123638927936554,
      "rewards/margins": 1.9239617586135864,
      "rewards/rejected": -1.6115977764129639,
      "step": 313
    },
    {
      "epoch": 5.322033898305085,
      "grad_norm": 13.661349245141958,
      "learning_rate": 4.591424713831602e-07,
      "logits/chosen": 4.491490840911865,
      "logits/rejected": 5.605785369873047,
      "logps/chosen": -14.745203018188477,
      "logps/rejected": -32.56706619262695,
      "loss": 0.2112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3098144829273224,
      "rewards/margins": 3.300020456314087,
      "rewards/rejected": -2.990206003189087,
      "step": 314
    },
    {
      "epoch": 5.338983050847458,
      "grad_norm": 15.004385831738075,
      "learning_rate": 4.587363888876361e-07,
      "logits/chosen": 6.41065788269043,
      "logits/rejected": 7.018966197967529,
      "logps/chosen": -13.388933181762695,
      "logps/rejected": -28.454553604125977,
      "loss": 0.2148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11646291613578796,
      "rewards/margins": 2.6084635257720947,
      "rewards/rejected": -2.4920005798339844,
      "step": 315
    },
    {
      "epoch": 5.3559322033898304,
      "grad_norm": 16.490268828216397,
      "learning_rate": 4.583284797706287e-07,
      "logits/chosen": 5.329119682312012,
      "logits/rejected": 5.552703380584717,
      "logps/chosen": -12.61679744720459,
      "logps/rejected": -22.200180053710938,
      "loss": 0.2401,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10427120327949524,
      "rewards/margins": 1.3166574239730835,
      "rewards/rejected": -1.212386131286621,
      "step": 316
    },
    {
      "epoch": 5.372881355932203,
      "grad_norm": 13.010137117952876,
      "learning_rate": 4.5791874760169093e-07,
      "logits/chosen": 3.9311819076538086,
      "logits/rejected": 4.017928123474121,
      "logps/chosen": -14.682848930358887,
      "logps/rejected": -20.934898376464844,
      "loss": 0.1723,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.12462379038333893,
      "rewards/margins": 2.234661817550659,
      "rewards/rejected": -2.1100382804870605,
      "step": 317
    },
    {
      "epoch": 5.389830508474576,
      "grad_norm": 13.709606261421065,
      "learning_rate": 4.575071959663288e-07,
      "logits/chosen": 5.797114372253418,
      "logits/rejected": 6.789894104003906,
      "logps/chosen": -19.55613899230957,
      "logps/rejected": -34.477935791015625,
      "loss": 0.1851,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.051314182579517365,
      "rewards/margins": 2.7669546604156494,
      "rewards/rejected": -2.7156405448913574,
      "step": 318
    },
    {
      "epoch": 5.406779661016949,
      "grad_norm": 14.989659075325534,
      "learning_rate": 4.570938284659702e-07,
      "logits/chosen": 7.181881904602051,
      "logits/rejected": 7.72728967666626,
      "logps/chosen": -16.837162017822266,
      "logps/rejected": -26.756256103515625,
      "loss": 0.2261,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11718781292438507,
      "rewards/margins": 2.574601888656616,
      "rewards/rejected": -2.457413911819458,
      "step": 319
    },
    {
      "epoch": 5.423728813559322,
      "grad_norm": 14.352574380790168,
      "learning_rate": 4.566786487179334e-07,
      "logits/chosen": 4.944997310638428,
      "logits/rejected": 6.036535739898682,
      "logps/chosen": -17.46659278869629,
      "logps/rejected": -25.725868225097656,
      "loss": 0.2242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4766841530799866,
      "rewards/margins": 3.0206315517425537,
      "rewards/rejected": -2.543947458267212,
      "step": 320
    },
    {
      "epoch": 5.440677966101695,
      "grad_norm": 12.37098062441416,
      "learning_rate": 4.5626166035539535e-07,
      "logits/chosen": 7.607375144958496,
      "logits/rejected": 9.128290176391602,
      "logps/chosen": -14.674689292907715,
      "logps/rejected": -27.710792541503906,
      "loss": 0.1668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13436803221702576,
      "rewards/margins": 2.9709839820861816,
      "rewards/rejected": -2.836616039276123,
      "step": 321
    },
    {
      "epoch": 5.4576271186440675,
      "grad_norm": 13.91317628066266,
      "learning_rate": 4.5584286702736007e-07,
      "logits/chosen": 4.578854084014893,
      "logits/rejected": 5.086130142211914,
      "logps/chosen": -14.649238586425781,
      "logps/rejected": -25.154186248779297,
      "loss": 0.1746,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16391971707344055,
      "rewards/margins": 1.9267592430114746,
      "rewards/rejected": -1.762839436531067,
      "step": 322
    },
    {
      "epoch": 5.47457627118644,
      "grad_norm": 15.211982042512723,
      "learning_rate": 4.5542227239862654e-07,
      "logits/chosen": 3.5927846431732178,
      "logits/rejected": 4.380569934844971,
      "logps/chosen": -17.435205459594727,
      "logps/rejected": -32.917701721191406,
      "loss": 0.2114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1431255340576172,
      "rewards/margins": 3.1278703212738037,
      "rewards/rejected": -3.270995616912842,
      "step": 323
    },
    {
      "epoch": 5.491525423728813,
      "grad_norm": 14.815140645438243,
      "learning_rate": 4.5499988014975635e-07,
      "logits/chosen": 4.980439186096191,
      "logits/rejected": 5.031620979309082,
      "logps/chosen": -21.051170349121094,
      "logps/rejected": -29.4047794342041,
      "loss": 0.2156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2153717428445816,
      "rewards/margins": 2.1704635620117188,
      "rewards/rejected": -1.9550918340682983,
      "step": 324
    },
    {
      "epoch": 5.508474576271187,
      "grad_norm": 20.500615522038547,
      "learning_rate": 4.545756939770422e-07,
      "logits/chosen": 6.855619430541992,
      "logits/rejected": 7.975409030914307,
      "logps/chosen": -12.433855056762695,
      "logps/rejected": -29.087566375732422,
      "loss": 0.1834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16718144714832306,
      "rewards/margins": 3.911283016204834,
      "rewards/rejected": -3.7441017627716064,
      "step": 325
    },
    {
      "epoch": 5.52542372881356,
      "grad_norm": 14.24067627416158,
      "learning_rate": 4.54149717592475e-07,
      "logits/chosen": 6.717032432556152,
      "logits/rejected": 7.574321269989014,
      "logps/chosen": -17.490558624267578,
      "logps/rejected": -24.359642028808594,
      "loss": 0.2031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06353382766246796,
      "rewards/margins": 2.110564708709717,
      "rewards/rejected": -2.174098491668701,
      "step": 326
    },
    {
      "epoch": 5.5423728813559325,
      "grad_norm": 13.71169586115688,
      "learning_rate": 4.537219547237114e-07,
      "logits/chosen": 5.765188694000244,
      "logits/rejected": 6.628974437713623,
      "logps/chosen": -14.898784637451172,
      "logps/rejected": -38.453922271728516,
      "loss": 0.203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0718056857585907,
      "rewards/margins": 3.7458438873291016,
      "rewards/rejected": -3.6740381717681885,
      "step": 327
    },
    {
      "epoch": 5.559322033898305,
      "grad_norm": 13.228697365678313,
      "learning_rate": 4.5329240911404167e-07,
      "logits/chosen": 6.111119747161865,
      "logits/rejected": 6.550088882446289,
      "logps/chosen": -12.508288383483887,
      "logps/rejected": -21.71690559387207,
      "loss": 0.1939,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08406098186969757,
      "rewards/margins": 2.3071329593658447,
      "rewards/rejected": -2.223072052001953,
      "step": 328
    },
    {
      "epoch": 5.576271186440678,
      "grad_norm": 14.841949798286183,
      "learning_rate": 4.528610845223562e-07,
      "logits/chosen": 3.667128324508667,
      "logits/rejected": 4.614826679229736,
      "logps/chosen": -17.83123779296875,
      "logps/rejected": -38.682918548583984,
      "loss": 0.189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46043580770492554,
      "rewards/margins": 3.5695717334747314,
      "rewards/rejected": -3.109135866165161,
      "step": 329
    },
    {
      "epoch": 5.593220338983051,
      "grad_norm": 13.699477662259353,
      "learning_rate": 4.5242798472311306e-07,
      "logits/chosen": 4.154301166534424,
      "logits/rejected": 4.492623329162598,
      "logps/chosen": -14.746369361877441,
      "logps/rejected": -20.418010711669922,
      "loss": 0.1711,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09611117839813232,
      "rewards/margins": 1.9738531112670898,
      "rewards/rejected": -1.877742052078247,
      "step": 330
    },
    {
      "epoch": 5.610169491525424,
      "grad_norm": 12.374676658865045,
      "learning_rate": 4.519931135063051e-07,
      "logits/chosen": 4.899678707122803,
      "logits/rejected": 5.104933738708496,
      "logps/chosen": -16.690101623535156,
      "logps/rejected": -30.977108001708984,
      "loss": 0.17,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09213536977767944,
      "rewards/margins": 3.308014392852783,
      "rewards/rejected": -3.215879201889038,
      "step": 331
    },
    {
      "epoch": 5.627118644067797,
      "grad_norm": 15.061400244189306,
      "learning_rate": 4.515564746774265e-07,
      "logits/chosen": 3.064007520675659,
      "logits/rejected": 3.6515920162200928,
      "logps/chosen": -15.63426399230957,
      "logps/rejected": -24.327058792114258,
      "loss": 0.2141,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3263694643974304,
      "rewards/margins": 2.3982205390930176,
      "rewards/rejected": -2.0718512535095215,
      "step": 332
    },
    {
      "epoch": 5.6440677966101696,
      "grad_norm": 13.261396058716384,
      "learning_rate": 4.5111807205743945e-07,
      "logits/chosen": 3.667691946029663,
      "logits/rejected": 5.319759845733643,
      "logps/chosen": -18.902265548706055,
      "logps/rejected": -36.717247009277344,
      "loss": 0.1695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5261211395263672,
      "rewards/margins": 3.4888546466827393,
      "rewards/rejected": -2.962733745574951,
      "step": 333
    },
    {
      "epoch": 5.661016949152542,
      "grad_norm": 14.645385511954315,
      "learning_rate": 4.5067790948274085e-07,
      "logits/chosen": 4.4554548263549805,
      "logits/rejected": 5.304936408996582,
      "logps/chosen": -15.04642391204834,
      "logps/rejected": -23.59757423400879,
      "loss": 0.191,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16090461611747742,
      "rewards/margins": 2.001107692718506,
      "rewards/rejected": -1.840202808380127,
      "step": 334
    },
    {
      "epoch": 5.677966101694915,
      "grad_norm": 13.666596346564294,
      "learning_rate": 4.5023599080512896e-07,
      "logits/chosen": 5.841002941131592,
      "logits/rejected": 6.072424411773682,
      "logps/chosen": -19.874303817749023,
      "logps/rejected": -26.235692977905273,
      "loss": 0.1684,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2315959334373474,
      "rewards/margins": 2.4640674591064453,
      "rewards/rejected": -2.2324717044830322,
      "step": 335
    },
    {
      "epoch": 5.694915254237288,
      "grad_norm": 13.787686175151595,
      "learning_rate": 4.4979231989176905e-07,
      "logits/chosen": 4.080766201019287,
      "logits/rejected": 4.8679890632629395,
      "logps/chosen": -13.054632186889648,
      "logps/rejected": -22.956096649169922,
      "loss": 0.1858,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0869242399930954,
      "rewards/margins": 2.300351858139038,
      "rewards/rejected": -2.2134275436401367,
      "step": 336
    },
    {
      "epoch": 5.711864406779661,
      "grad_norm": 13.551160822081176,
      "learning_rate": 4.493469006251601e-07,
      "logits/chosen": 5.657144069671631,
      "logits/rejected": 7.342559814453125,
      "logps/chosen": -17.066558837890625,
      "logps/rejected": -30.203340530395508,
      "loss": 0.1861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.165096715092659,
      "rewards/margins": 3.3928604125976562,
      "rewards/rejected": -3.2277636528015137,
      "step": 337
    },
    {
      "epoch": 5.728813559322034,
      "grad_norm": 14.419160323493225,
      "learning_rate": 4.488997369031008e-07,
      "logits/chosen": 4.785531997680664,
      "logits/rejected": 4.794522762298584,
      "logps/chosen": -14.324603080749512,
      "logps/rejected": -24.154769897460938,
      "loss": 0.1699,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2423357367515564,
      "rewards/margins": 2.1883811950683594,
      "rewards/rejected": -1.9460456371307373,
      "step": 338
    },
    {
      "epoch": 5.745762711864407,
      "grad_norm": 11.664671797317196,
      "learning_rate": 4.4845083263865514e-07,
      "logits/chosen": 2.3409574031829834,
      "logits/rejected": 3.2811951637268066,
      "logps/chosen": -17.960046768188477,
      "logps/rejected": -22.232858657836914,
      "loss": 0.1891,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36390525102615356,
      "rewards/margins": 2.5179195404052734,
      "rewards/rejected": -2.1540141105651855,
      "step": 339
    },
    {
      "epoch": 5.762711864406779,
      "grad_norm": 13.733632789469274,
      "learning_rate": 4.4800019176011847e-07,
      "logits/chosen": 4.645487308502197,
      "logits/rejected": 4.405810832977295,
      "logps/chosen": -14.33531379699707,
      "logps/rejected": -26.283172607421875,
      "loss": 0.217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1105184406042099,
      "rewards/margins": 2.3442327976226807,
      "rewards/rejected": -2.2337143421173096,
      "step": 340
    },
    {
      "epoch": 5.779661016949152,
      "grad_norm": 13.029185639792166,
      "learning_rate": 4.4754781821098286e-07,
      "logits/chosen": 6.101343631744385,
      "logits/rejected": 5.980262756347656,
      "logps/chosen": -19.913057327270508,
      "logps/rejected": -28.51534652709961,
      "loss": 0.167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.290585994720459,
      "rewards/margins": 3.1626698970794678,
      "rewards/rejected": -2.872083902359009,
      "step": 341
    },
    {
      "epoch": 5.796610169491525,
      "grad_norm": 13.259351062753181,
      "learning_rate": 4.470937159499028e-07,
      "logits/chosen": 6.815540313720703,
      "logits/rejected": 7.2496442794799805,
      "logps/chosen": -12.720376968383789,
      "logps/rejected": -24.852739334106445,
      "loss": 0.1981,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05745570361614227,
      "rewards/margins": 2.748012065887451,
      "rewards/rejected": -2.690556287765503,
      "step": 342
    },
    {
      "epoch": 5.813559322033898,
      "grad_norm": 14.211807222272416,
      "learning_rate": 4.4663788895066065e-07,
      "logits/chosen": 5.461134433746338,
      "logits/rejected": 5.95833158493042,
      "logps/chosen": -16.62125015258789,
      "logps/rejected": -25.2918701171875,
      "loss": 0.1881,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31963789463043213,
      "rewards/margins": 2.098417282104492,
      "rewards/rejected": -1.7787795066833496,
      "step": 343
    },
    {
      "epoch": 5.830508474576272,
      "grad_norm": 13.633790061419106,
      "learning_rate": 4.4618034120213135e-07,
      "logits/chosen": 5.231446266174316,
      "logits/rejected": 6.182756423950195,
      "logps/chosen": -16.792110443115234,
      "logps/rejected": -36.05128479003906,
      "loss": 0.1739,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2307071089744568,
      "rewards/margins": 3.797922134399414,
      "rewards/rejected": -3.5672149658203125,
      "step": 344
    },
    {
      "epoch": 5.847457627118644,
      "grad_norm": 12.438805612920406,
      "learning_rate": 4.4572107670824806e-07,
      "logits/chosen": 2.8178811073303223,
      "logits/rejected": 3.8991308212280273,
      "logps/chosen": -13.256142616271973,
      "logps/rejected": -28.83277702331543,
      "loss": 0.1554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3214130103588104,
      "rewards/margins": 3.2701797485351562,
      "rewards/rejected": -2.9487667083740234,
      "step": 345
    },
    {
      "epoch": 5.864406779661017,
      "grad_norm": 13.708565551444401,
      "learning_rate": 4.45260099487967e-07,
      "logits/chosen": 3.6462607383728027,
      "logits/rejected": 3.852987289428711,
      "logps/chosen": -24.624631881713867,
      "logps/rejected": -24.646089553833008,
      "loss": 0.209,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16818492114543915,
      "rewards/margins": 2.566938638687134,
      "rewards/rejected": -2.3987538814544678,
      "step": 346
    },
    {
      "epoch": 5.88135593220339,
      "grad_norm": 13.340203940885758,
      "learning_rate": 4.4479741357523204e-07,
      "logits/chosen": 6.4154052734375,
      "logits/rejected": 7.19559907913208,
      "logps/chosen": -16.736295700073242,
      "logps/rejected": -28.568880081176758,
      "loss": 0.174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3518092632293701,
      "rewards/margins": 2.901329517364502,
      "rewards/rejected": -2.549520254135132,
      "step": 347
    },
    {
      "epoch": 5.898305084745763,
      "grad_norm": 14.529901903868208,
      "learning_rate": 4.4433302301893983e-07,
      "logits/chosen": 3.5100386142730713,
      "logits/rejected": 3.799931049346924,
      "logps/chosen": -13.322935104370117,
      "logps/rejected": -29.25261116027832,
      "loss": 0.1959,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.017160028219223022,
      "rewards/margins": 2.818272829055786,
      "rewards/rejected": -2.835433006286621,
      "step": 348
    },
    {
      "epoch": 5.915254237288136,
      "grad_norm": 14.146444228683375,
      "learning_rate": 4.438669318829037e-07,
      "logits/chosen": 0.44511693716049194,
      "logits/rejected": 1.87983238697052,
      "logps/chosen": -17.3454532623291,
      "logps/rejected": -27.43291473388672,
      "loss": 0.1855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7081802487373352,
      "rewards/margins": 2.4732515811920166,
      "rewards/rejected": -1.7650713920593262,
      "step": 349
    },
    {
      "epoch": 5.932203389830509,
      "grad_norm": 13.385150325169393,
      "learning_rate": 4.433991442458188e-07,
      "logits/chosen": 2.5039730072021484,
      "logits/rejected": 4.11275577545166,
      "logps/chosen": -23.479930877685547,
      "logps/rejected": -25.535865783691406,
      "loss": 0.1886,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.12501907348632812,
      "rewards/margins": 2.16085147857666,
      "rewards/rejected": -2.035832405090332,
      "step": 350
    },
    {
      "epoch": 5.9491525423728815,
      "grad_norm": 13.909362766221472,
      "learning_rate": 4.4292966420122613e-07,
      "logits/chosen": 7.141190528869629,
      "logits/rejected": 7.579991340637207,
      "logps/chosen": -15.560004234313965,
      "logps/rejected": -25.857866287231445,
      "loss": 0.1725,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.055767402052879333,
      "rewards/margins": 2.5317444801330566,
      "rewards/rejected": -2.4759769439697266,
      "step": 351
    },
    {
      "epoch": 5.966101694915254,
      "grad_norm": 14.588943173031627,
      "learning_rate": 4.4245849585747655e-07,
      "logits/chosen": 2.5870280265808105,
      "logits/rejected": 2.972602367401123,
      "logps/chosen": -17.201574325561523,
      "logps/rejected": -27.69642448425293,
      "loss": 0.2357,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09064903110265732,
      "rewards/margins": 2.292398691177368,
      "rewards/rejected": -2.383047580718994,
      "step": 352
    },
    {
      "epoch": 5.983050847457627,
      "grad_norm": 12.836870682632282,
      "learning_rate": 4.41985643337695e-07,
      "logits/chosen": 7.1439924240112305,
      "logits/rejected": 7.3029303550720215,
      "logps/chosen": -16.506511688232422,
      "logps/rejected": -31.438173294067383,
      "loss": 0.1732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15994346141815186,
      "rewards/margins": 3.0539016723632812,
      "rewards/rejected": -2.89395809173584,
      "step": 353
    },
    {
      "epoch": 6.0,
      "grad_norm": 13.14834477516658,
      "learning_rate": 4.415111107797445e-07,
      "logits/chosen": 5.239683628082275,
      "logits/rejected": 6.064949035644531,
      "logps/chosen": -11.112751960754395,
      "logps/rejected": -22.786832809448242,
      "loss": 0.168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2163887917995453,
      "rewards/margins": 3.0100321769714355,
      "rewards/rejected": -2.7936432361602783,
      "step": 354
    },
    {
      "epoch": 6.016949152542373,
      "grad_norm": 12.532478313650268,
      "learning_rate": 4.410349023361897e-07,
      "logits/chosen": 5.389923095703125,
      "logits/rejected": 5.660534381866455,
      "logps/chosen": -23.009815216064453,
      "logps/rejected": -30.078847885131836,
      "loss": 0.186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05637688934803009,
      "rewards/margins": 2.8357455730438232,
      "rewards/rejected": -2.892122507095337,
      "step": 355
    },
    {
      "epoch": 6.033898305084746,
      "grad_norm": 10.813866763874628,
      "learning_rate": 4.4055702217426085e-07,
      "logits/chosen": 3.7662200927734375,
      "logits/rejected": 4.253184795379639,
      "logps/chosen": -14.30909538269043,
      "logps/rejected": -25.599939346313477,
      "loss": 0.1599,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2995351552963257,
      "rewards/margins": 3.287034273147583,
      "rewards/rejected": -2.987499475479126,
      "step": 356
    },
    {
      "epoch": 6.0508474576271185,
      "grad_norm": 12.085541722736634,
      "learning_rate": 4.40077474475817e-07,
      "logits/chosen": 2.5573201179504395,
      "logits/rejected": 3.847440004348755,
      "logps/chosen": -18.95703887939453,
      "logps/rejected": -28.729902267456055,
      "loss": 0.1712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.37842535972595215,
      "rewards/margins": 2.1750941276550293,
      "rewards/rejected": -1.7966687679290771,
      "step": 357
    },
    {
      "epoch": 6.067796610169491,
      "grad_norm": 10.470231348613757,
      "learning_rate": 4.395962634373096e-07,
      "logits/chosen": 6.930731773376465,
      "logits/rejected": 7.621533393859863,
      "logps/chosen": -15.80143928527832,
      "logps/rejected": -27.89528465270996,
      "loss": 0.1435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10261240601539612,
      "rewards/margins": 2.8732903003692627,
      "rewards/rejected": -2.975903034210205,
      "step": 358
    },
    {
      "epoch": 6.084745762711864,
      "grad_norm": 10.943734781770678,
      "learning_rate": 4.3911339326974584e-07,
      "logits/chosen": 5.029911518096924,
      "logits/rejected": 5.953924179077148,
      "logps/chosen": -11.060075759887695,
      "logps/rejected": -31.347972869873047,
      "loss": 0.1391,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3969661295413971,
      "rewards/margins": 4.131930351257324,
      "rewards/rejected": -3.734964370727539,
      "step": 359
    },
    {
      "epoch": 6.101694915254237,
      "grad_norm": 11.465145430024815,
      "learning_rate": 4.386288681986516e-07,
      "logits/chosen": 5.777263164520264,
      "logits/rejected": 6.994723320007324,
      "logps/chosen": -19.753942489624023,
      "logps/rejected": -29.311805725097656,
      "loss": 0.1415,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.22933685779571533,
      "rewards/margins": 3.1403188705444336,
      "rewards/rejected": -3.3696556091308594,
      "step": 360
    },
    {
      "epoch": 6.11864406779661,
      "grad_norm": 11.328941727318716,
      "learning_rate": 4.3814269246403456e-07,
      "logits/chosen": 4.634091377258301,
      "logits/rejected": 5.644282817840576,
      "logps/chosen": -16.082172393798828,
      "logps/rejected": -26.386442184448242,
      "loss": 0.1354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08987744152545929,
      "rewards/margins": 2.8858983516693115,
      "rewards/rejected": -2.796020984649658,
      "step": 361
    },
    {
      "epoch": 6.135593220338983,
      "grad_norm": 11.969751512905047,
      "learning_rate": 4.3765487032034737e-07,
      "logits/chosen": 1.9608010053634644,
      "logits/rejected": 2.5253000259399414,
      "logps/chosen": -21.299962997436523,
      "logps/rejected": -33.22754669189453,
      "loss": 0.1546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23107457160949707,
      "rewards/margins": 3.7274532318115234,
      "rewards/rejected": -3.4963784217834473,
      "step": 362
    },
    {
      "epoch": 6.1525423728813555,
      "grad_norm": 11.229599928113995,
      "learning_rate": 4.371654060364498e-07,
      "logits/chosen": 3.036740779876709,
      "logits/rejected": 3.398423194885254,
      "logps/chosen": -14.125732421875,
      "logps/rejected": -20.322280883789062,
      "loss": 0.17,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22011126577854156,
      "rewards/margins": 2.515415668487549,
      "rewards/rejected": -2.295304536819458,
      "step": 363
    },
    {
      "epoch": 6.169491525423728,
      "grad_norm": 11.749415025290373,
      "learning_rate": 4.366743038955719e-07,
      "logits/chosen": 3.778263568878174,
      "logits/rejected": 4.308474540710449,
      "logps/chosen": -18.887042999267578,
      "logps/rejected": -26.429588317871094,
      "loss": 0.1574,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3668030798435211,
      "rewards/margins": 2.436537742614746,
      "rewards/rejected": -2.069734811782837,
      "step": 364
    },
    {
      "epoch": 6.186440677966102,
      "grad_norm": 11.80077626000249,
      "learning_rate": 4.361815681952765e-07,
      "logits/chosen": 1.918591856956482,
      "logits/rejected": 2.6716151237487793,
      "logps/chosen": -19.709033966064453,
      "logps/rejected": -21.16021156311035,
      "loss": 0.17,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25224724411964417,
      "rewards/margins": 2.316915273666382,
      "rewards/rejected": -2.0646679401397705,
      "step": 365
    },
    {
      "epoch": 6.203389830508475,
      "grad_norm": 11.838957746477757,
      "learning_rate": 4.3568720324742126e-07,
      "logits/chosen": 5.8752264976501465,
      "logits/rejected": 7.20805549621582,
      "logps/chosen": -17.11652183532715,
      "logps/rejected": -31.32016372680664,
      "loss": 0.1459,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32180485129356384,
      "rewards/margins": 2.999842405319214,
      "rewards/rejected": -2.678037643432617,
      "step": 366
    },
    {
      "epoch": 6.220338983050848,
      "grad_norm": 11.944090270848385,
      "learning_rate": 4.351912133781212e-07,
      "logits/chosen": 6.407975196838379,
      "logits/rejected": 6.661712169647217,
      "logps/chosen": -15.283435821533203,
      "logps/rejected": -19.59105682373047,
      "loss": 0.1742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.47592228651046753,
      "rewards/margins": 1.68387770652771,
      "rewards/rejected": -1.2079553604125977,
      "step": 367
    },
    {
      "epoch": 6.237288135593221,
      "grad_norm": 11.440576248397297,
      "learning_rate": 4.3469360292771096e-07,
      "logits/chosen": 4.260448455810547,
      "logits/rejected": 4.510221481323242,
      "logps/chosen": -14.869780540466309,
      "logps/rejected": -22.963233947753906,
      "loss": 0.1535,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.5458751320838928,
      "rewards/margins": 2.5354907512664795,
      "rewards/rejected": -1.989615559577942,
      "step": 368
    },
    {
      "epoch": 6.254237288135593,
      "grad_norm": 13.011393149825132,
      "learning_rate": 4.3419437625070634e-07,
      "logits/chosen": 4.716248035430908,
      "logits/rejected": 5.315852165222168,
      "logps/chosen": -14.17831802368164,
      "logps/rejected": -23.364871978759766,
      "loss": 0.1877,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.23576220870018005,
      "rewards/margins": 2.1676623821258545,
      "rewards/rejected": -1.9319000244140625,
      "step": 369
    },
    {
      "epoch": 6.271186440677966,
      "grad_norm": 12.22060362782235,
      "learning_rate": 4.336935377157668e-07,
      "logits/chosen": 1.9044804573059082,
      "logits/rejected": 2.334730386734009,
      "logps/chosen": -16.843324661254883,
      "logps/rejected": -28.972496032714844,
      "loss": 0.1899,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5382827520370483,
      "rewards/margins": 3.484276294708252,
      "rewards/rejected": -2.945993661880493,
      "step": 370
    },
    {
      "epoch": 6.288135593220339,
      "grad_norm": 11.110250794754856,
      "learning_rate": 4.3319109170565676e-07,
      "logits/chosen": 4.34942102432251,
      "logits/rejected": 5.465624809265137,
      "logps/chosen": -12.080187797546387,
      "logps/rejected": -29.275917053222656,
      "loss": 0.1556,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.535807192325592,
      "rewards/margins": 3.084014654159546,
      "rewards/rejected": -2.5482072830200195,
      "step": 371
    },
    {
      "epoch": 6.305084745762712,
      "grad_norm": 10.689860258387656,
      "learning_rate": 4.3268704261720745e-07,
      "logits/chosen": 2.993368625640869,
      "logits/rejected": 3.6167068481445312,
      "logps/chosen": -15.360583305358887,
      "logps/rejected": -26.135984420776367,
      "loss": 0.1471,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6923851370811462,
      "rewards/margins": 2.7075209617614746,
      "rewards/rejected": -2.0151357650756836,
      "step": 372
    },
    {
      "epoch": 6.322033898305085,
      "grad_norm": 11.813171861529911,
      "learning_rate": 4.321813948612785e-07,
      "logits/chosen": 5.465794563293457,
      "logits/rejected": 5.721805572509766,
      "logps/chosen": -17.526994705200195,
      "logps/rejected": -23.695585250854492,
      "loss": 0.1569,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14771461486816406,
      "rewards/margins": 2.829829692840576,
      "rewards/rejected": -2.9775443077087402,
      "step": 373
    },
    {
      "epoch": 6.338983050847458,
      "grad_norm": 10.966391063899888,
      "learning_rate": 4.31674152862719e-07,
      "logits/chosen": 2.381113290786743,
      "logits/rejected": 2.6802804470062256,
      "logps/chosen": -11.867440223693848,
      "logps/rejected": -24.956613540649414,
      "loss": 0.158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08301214873790741,
      "rewards/margins": 2.746257781982422,
      "rewards/rejected": -2.663245677947998,
      "step": 374
    },
    {
      "epoch": 6.3559322033898304,
      "grad_norm": 11.645900837905495,
      "learning_rate": 4.311653210603293e-07,
      "logits/chosen": 3.390183448791504,
      "logits/rejected": 5.339263439178467,
      "logps/chosen": -22.072269439697266,
      "logps/rejected": -30.078414916992188,
      "loss": 0.1575,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2785249948501587,
      "rewards/margins": 3.537834644317627,
      "rewards/rejected": -3.2593092918395996,
      "step": 375
    },
    {
      "epoch": 6.372881355932203,
      "grad_norm": 11.44172677194351,
      "learning_rate": 4.306549039068218e-07,
      "logits/chosen": 2.786032199859619,
      "logits/rejected": 3.0518574714660645,
      "logps/chosen": -17.41475486755371,
      "logps/rejected": -22.37212371826172,
      "loss": 0.14,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.38636505603790283,
      "rewards/margins": 2.6983985900878906,
      "rewards/rejected": -2.3120336532592773,
      "step": 376
    },
    {
      "epoch": 6.389830508474576,
      "grad_norm": 13.337553165557853,
      "learning_rate": 4.301429058687819e-07,
      "logits/chosen": 1.9814975261688232,
      "logits/rejected": 3.0942461490631104,
      "logps/chosen": -15.964947700500488,
      "logps/rejected": -35.37507247924805,
      "loss": 0.1701,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8118382692337036,
      "rewards/margins": 4.838287830352783,
      "rewards/rejected": -4.026450157165527,
      "step": 377
    },
    {
      "epoch": 6.406779661016949,
      "grad_norm": 9.255376884922308,
      "learning_rate": 4.296293314266294e-07,
      "logits/chosen": 2.894780158996582,
      "logits/rejected": 3.551926851272583,
      "logps/chosen": -13.228153228759766,
      "logps/rejected": -26.36261558532715,
      "loss": 0.1186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35001492500305176,
      "rewards/margins": 3.228752851486206,
      "rewards/rejected": -2.8787379264831543,
      "step": 378
    },
    {
      "epoch": 6.423728813559322,
      "grad_norm": 12.86108986778706,
      "learning_rate": 4.2911418507457876e-07,
      "logits/chosen": 1.326431155204773,
      "logits/rejected": 1.1022425889968872,
      "logps/chosen": -18.70794105529785,
      "logps/rejected": -25.21274185180664,
      "loss": 0.1771,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.20193463563919067,
      "rewards/margins": 2.162876844406128,
      "rewards/rejected": -1.960942268371582,
      "step": 379
    },
    {
      "epoch": 6.440677966101695,
      "grad_norm": 10.842665215954792,
      "learning_rate": 4.285974713206e-07,
      "logits/chosen": 4.0209760665893555,
      "logits/rejected": 5.607435703277588,
      "logps/chosen": -18.19493865966797,
      "logps/rejected": -29.27100372314453,
      "loss": 0.1646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25257328152656555,
      "rewards/margins": 3.3824639320373535,
      "rewards/rejected": -3.1298906803131104,
      "step": 380
    },
    {
      "epoch": 6.4576271186440675,
      "grad_norm": 11.725063198263442,
      "learning_rate": 4.280791946863794e-07,
      "logits/chosen": 1.2813572883605957,
      "logits/rejected": 2.023667097091675,
      "logps/chosen": -14.165773391723633,
      "logps/rejected": -25.905025482177734,
      "loss": 0.1548,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08021500706672668,
      "rewards/margins": 2.9218506813049316,
      "rewards/rejected": -2.8416357040405273,
      "step": 381
    },
    {
      "epoch": 6.47457627118644,
      "grad_norm": 11.46721147012392,
      "learning_rate": 4.275593597072795e-07,
      "logits/chosen": 1.4821670055389404,
      "logits/rejected": 1.7210092544555664,
      "logps/chosen": -18.427885055541992,
      "logps/rejected": -25.48371696472168,
      "loss": 0.1285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41084200143814087,
      "rewards/margins": 2.797346353530884,
      "rewards/rejected": -2.3865044116973877,
      "step": 382
    },
    {
      "epoch": 6.491525423728813,
      "grad_norm": 10.51651979591079,
      "learning_rate": 4.270379709323001e-07,
      "logits/chosen": 4.080705642700195,
      "logits/rejected": 4.421995162963867,
      "logps/chosen": -17.705881118774414,
      "logps/rejected": -31.33135986328125,
      "loss": 0.1394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04687368869781494,
      "rewards/margins": 3.1977744102478027,
      "rewards/rejected": -3.2446484565734863,
      "step": 383
    },
    {
      "epoch": 6.508474576271187,
      "grad_norm": 11.273261378268472,
      "learning_rate": 4.265150329240376e-07,
      "logits/chosen": 3.1031620502471924,
      "logits/rejected": 4.636520862579346,
      "logps/chosen": -14.841632843017578,
      "logps/rejected": -26.25077247619629,
      "loss": 0.1285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06430919468402863,
      "rewards/margins": 2.5861589908599854,
      "rewards/rejected": -2.5218496322631836,
      "step": 384
    },
    {
      "epoch": 6.52542372881356,
      "grad_norm": 11.992095342201655,
      "learning_rate": 4.259905502586457e-07,
      "logits/chosen": 2.4647104740142822,
      "logits/rejected": 3.4929392337799072,
      "logps/chosen": -15.373156547546387,
      "logps/rejected": -26.317827224731445,
      "loss": 0.1608,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12471124529838562,
      "rewards/margins": 2.660918951034546,
      "rewards/rejected": -2.536207675933838,
      "step": 385
    },
    {
      "epoch": 6.5423728813559325,
      "grad_norm": 12.945212906753929,
      "learning_rate": 4.254645275257953e-07,
      "logits/chosen": 3.468475580215454,
      "logits/rejected": 4.383484840393066,
      "logps/chosen": -13.625848770141602,
      "logps/rejected": -28.03760528564453,
      "loss": 0.1689,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24450263381004333,
      "rewards/margins": 2.8928561210632324,
      "rewards/rejected": -2.6483535766601562,
      "step": 386
    },
    {
      "epoch": 6.559322033898305,
      "grad_norm": 11.089672955876997,
      "learning_rate": 4.24936969328634e-07,
      "logits/chosen": 4.420220375061035,
      "logits/rejected": 5.376060485839844,
      "logps/chosen": -11.676502227783203,
      "logps/rejected": -24.155860900878906,
      "loss": 0.152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08071908354759216,
      "rewards/margins": 2.936311721801758,
      "rewards/rejected": -2.855592727661133,
      "step": 387
    },
    {
      "epoch": 6.576271186440678,
      "grad_norm": 13.695617001861233,
      "learning_rate": 4.244078802837462e-07,
      "logits/chosen": 5.1619062423706055,
      "logits/rejected": 5.240504264831543,
      "logps/chosen": -18.182863235473633,
      "logps/rejected": -21.219486236572266,
      "loss": 0.1946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22650668025016785,
      "rewards/margins": 2.3760101795196533,
      "rewards/rejected": -2.149503469467163,
      "step": 388
    },
    {
      "epoch": 6.593220338983051,
      "grad_norm": 11.788406533092438,
      "learning_rate": 4.238772650211123e-07,
      "logits/chosen": 1.1291437149047852,
      "logits/rejected": 2.072202682495117,
      "logps/chosen": -14.250567436218262,
      "logps/rejected": -30.102697372436523,
      "loss": 0.1543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.043179064989089966,
      "rewards/margins": 3.3207321166992188,
      "rewards/rejected": -3.277553081512451,
      "step": 389
    },
    {
      "epoch": 6.610169491525424,
      "grad_norm": 11.274977736495742,
      "learning_rate": 4.233451281840685e-07,
      "logits/chosen": 3.7870407104492188,
      "logits/rejected": 3.9119009971618652,
      "logps/chosen": -15.645469665527344,
      "logps/rejected": -24.250320434570312,
      "loss": 0.1671,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.23693856596946716,
      "rewards/margins": 1.8370524644851685,
      "rewards/rejected": -1.6001137495040894,
      "step": 390
    },
    {
      "epoch": 6.627118644067797,
      "grad_norm": 11.026889873931545,
      "learning_rate": 4.2281147442926636e-07,
      "logits/chosen": 2.935783624649048,
      "logits/rejected": 3.035520553588867,
      "logps/chosen": -12.330340385437012,
      "logps/rejected": -23.562206268310547,
      "loss": 0.1625,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.31259769201278687,
      "rewards/margins": 2.575361490249634,
      "rewards/rejected": -2.262763500213623,
      "step": 391
    },
    {
      "epoch": 6.6440677966101696,
      "grad_norm": 11.464351312206663,
      "learning_rate": 4.222763084266313e-07,
      "logits/chosen": 2.7037510871887207,
      "logits/rejected": 4.515506267547607,
      "logps/chosen": -14.069086074829102,
      "logps/rejected": -28.03109359741211,
      "loss": 0.1318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20625919103622437,
      "rewards/margins": 3.8207006454467773,
      "rewards/rejected": -3.6144416332244873,
      "step": 392
    },
    {
      "epoch": 6.661016949152542,
      "grad_norm": 9.640842179436438,
      "learning_rate": 4.217396348593224e-07,
      "logits/chosen": 3.874035120010376,
      "logits/rejected": 4.602170944213867,
      "logps/chosen": -22.469409942626953,
      "logps/rejected": -33.009944915771484,
      "loss": 0.1544,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.03429122269153595,
      "rewards/margins": 3.2556371688842773,
      "rewards/rejected": -3.2899281978607178,
      "step": 393
    },
    {
      "epoch": 6.677966101694915,
      "grad_norm": 13.652483617561197,
      "learning_rate": 4.2120145842369137e-07,
      "logits/chosen": 3.716986656188965,
      "logits/rejected": 4.291690826416016,
      "logps/chosen": -14.710618019104004,
      "logps/rejected": -27.50786590576172,
      "loss": 0.1801,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3418799936771393,
      "rewards/margins": 3.645103931427002,
      "rewards/rejected": -3.3032238483428955,
      "step": 394
    },
    {
      "epoch": 6.694915254237288,
      "grad_norm": 11.48146565873232,
      "learning_rate": 4.206617838292411e-07,
      "logits/chosen": 4.914379119873047,
      "logits/rejected": 5.654470443725586,
      "logps/chosen": -14.613869667053223,
      "logps/rejected": -29.499710083007812,
      "loss": 0.1565,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.09016422927379608,
      "rewards/margins": 3.4852559566497803,
      "rewards/rejected": -3.575420379638672,
      "step": 395
    },
    {
      "epoch": 6.711864406779661,
      "grad_norm": 9.939140025420576,
      "learning_rate": 4.201206157985846e-07,
      "logits/chosen": 5.169272422790527,
      "logits/rejected": 6.066037654876709,
      "logps/chosen": -13.552163124084473,
      "logps/rejected": -25.132427215576172,
      "loss": 0.124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26199135184288025,
      "rewards/margins": 3.1676011085510254,
      "rewards/rejected": -2.905609607696533,
      "step": 396
    },
    {
      "epoch": 6.728813559322034,
      "grad_norm": 11.987189290505833,
      "learning_rate": 4.1957795906740403e-07,
      "logits/chosen": 2.286560535430908,
      "logits/rejected": 2.5394349098205566,
      "logps/chosen": -12.481584548950195,
      "logps/rejected": -23.126205444335938,
      "loss": 0.1557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4645286798477173,
      "rewards/margins": 2.306704044342041,
      "rewards/rejected": -1.8421754837036133,
      "step": 397
    },
    {
      "epoch": 6.745762711864407,
      "grad_norm": 10.168711244583603,
      "learning_rate": 4.1903381838440853e-07,
      "logits/chosen": 4.190589427947998,
      "logits/rejected": 4.447041034698486,
      "logps/chosen": -17.075122833251953,
      "logps/rejected": -26.705982208251953,
      "loss": 0.1375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1718921661376953,
      "rewards/margins": 2.3209805488586426,
      "rewards/rejected": -2.1490883827209473,
      "step": 398
    },
    {
      "epoch": 6.762711864406779,
      "grad_norm": 10.139455694407227,
      "learning_rate": 4.1848819851129345e-07,
      "logits/chosen": 2.6103100776672363,
      "logits/rejected": 2.7593235969543457,
      "logps/chosen": -23.044191360473633,
      "logps/rejected": -33.22798156738281,
      "loss": 0.1222,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31734511256217957,
      "rewards/margins": 3.8872694969177246,
      "rewards/rejected": -3.5699243545532227,
      "step": 399
    },
    {
      "epoch": 6.779661016949152,
      "grad_norm": 11.787533074792071,
      "learning_rate": 4.179411042226982e-07,
      "logits/chosen": 3.197756052017212,
      "logits/rejected": 3.3959622383117676,
      "logps/chosen": -21.1182918548584,
      "logps/rejected": -29.089521408081055,
      "loss": 0.1524,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14221636950969696,
      "rewards/margins": 3.5914371013641357,
      "rewards/rejected": -3.7336530685424805,
      "step": 400
    },
    {
      "epoch": 6.796610169491525,
      "grad_norm": 10.509355548093078,
      "learning_rate": 4.173925403061644e-07,
      "logits/chosen": 0.36680668592453003,
      "logits/rejected": 1.1943421363830566,
      "logps/chosen": -21.096643447875977,
      "logps/rejected": -44.89948272705078,
      "loss": 0.1227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27233120799064636,
      "rewards/margins": 4.326108455657959,
      "rewards/rejected": -4.05377721786499,
      "step": 401
    },
    {
      "epoch": 6.813559322033898,
      "grad_norm": 11.105525805087115,
      "learning_rate": 4.1684251156209437e-07,
      "logits/chosen": 3.423051118850708,
      "logits/rejected": 4.709883689880371,
      "logps/chosen": -12.655998229980469,
      "logps/rejected": -32.92218780517578,
      "loss": 0.1419,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6730118989944458,
      "rewards/margins": 4.144976615905762,
      "rewards/rejected": -3.4719643592834473,
      "step": 402
    },
    {
      "epoch": 6.830508474576272,
      "grad_norm": 11.881691689692572,
      "learning_rate": 4.16291022803709e-07,
      "logits/chosen": 3.5171265602111816,
      "logits/rejected": 3.0639753341674805,
      "logps/chosen": -18.158029556274414,
      "logps/rejected": -22.001617431640625,
      "loss": 0.1534,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4230116307735443,
      "rewards/margins": 2.6759896278381348,
      "rewards/rejected": -2.2529778480529785,
      "step": 403
    },
    {
      "epoch": 6.847457627118644,
      "grad_norm": 11.334720471938402,
      "learning_rate": 4.1573807885700523e-07,
      "logits/chosen": 3.2304704189300537,
      "logits/rejected": 3.686030387878418,
      "logps/chosen": -17.79078483581543,
      "logps/rejected": -36.99302291870117,
      "loss": 0.1354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3478419780731201,
      "rewards/margins": 4.222134113311768,
      "rewards/rejected": -3.8742923736572266,
      "step": 404
    },
    {
      "epoch": 6.864406779661017,
      "grad_norm": 10.228794064023427,
      "learning_rate": 4.151836845607144e-07,
      "logits/chosen": 2.565765142440796,
      "logits/rejected": 3.0117526054382324,
      "logps/chosen": -19.339305877685547,
      "logps/rejected": -25.283498764038086,
      "loss": 0.1395,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.0219439268112183,
      "rewards/margins": 3.060976266860962,
      "rewards/rejected": -2.0390326976776123,
      "step": 405
    },
    {
      "epoch": 6.88135593220339,
      "grad_norm": 12.09741072980745,
      "learning_rate": 4.146278447662597e-07,
      "logits/chosen": 6.464285850524902,
      "logits/rejected": 6.6425957679748535,
      "logps/chosen": -12.595333099365234,
      "logps/rejected": -25.08333396911621,
      "loss": 0.169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24255235493183136,
      "rewards/margins": 2.9764745235443115,
      "rewards/rejected": -2.733922004699707,
      "step": 406
    },
    {
      "epoch": 6.898305084745763,
      "grad_norm": 9.900890386099787,
      "learning_rate": 4.1407056433771324e-07,
      "logits/chosen": 5.483569145202637,
      "logits/rejected": 6.747907638549805,
      "logps/chosen": -16.0140438079834,
      "logps/rejected": -32.348541259765625,
      "loss": 0.1148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2642427384853363,
      "rewards/margins": 3.627537727355957,
      "rewards/rejected": -3.891780376434326,
      "step": 407
    },
    {
      "epoch": 6.915254237288136,
      "grad_norm": 18.674013367884058,
      "learning_rate": 4.1351184815175456e-07,
      "logits/chosen": 2.835052251815796,
      "logits/rejected": 4.391816139221191,
      "logps/chosen": -20.78997230529785,
      "logps/rejected": -29.96456527709961,
      "loss": 0.1488,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.27503877878189087,
      "rewards/margins": 2.9371590614318848,
      "rewards/rejected": -2.6621201038360596,
      "step": 408
    },
    {
      "epoch": 6.932203389830509,
      "grad_norm": 11.088887852350569,
      "learning_rate": 4.1295170109762677e-07,
      "logits/chosen": 2.4660263061523438,
      "logits/rejected": 2.9183759689331055,
      "logps/chosen": -17.321752548217773,
      "logps/rejected": -28.743206024169922,
      "loss": 0.1468,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0913095474243164,
      "rewards/margins": 3.131824016571045,
      "rewards/rejected": -3.0405147075653076,
      "step": 409
    },
    {
      "epoch": 6.9491525423728815,
      "grad_norm": 10.620335803144801,
      "learning_rate": 4.1239012807709444e-07,
      "logits/chosen": 0.7266221046447754,
      "logits/rejected": 1.808227777481079,
      "logps/chosen": -16.597368240356445,
      "logps/rejected": -35.07196044921875,
      "loss": 0.1351,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12688055634498596,
      "rewards/margins": 3.9789581298828125,
      "rewards/rejected": -4.105838775634766,
      "step": 410
    },
    {
      "epoch": 6.966101694915254,
      "grad_norm": 10.627874755495654,
      "learning_rate": 4.1182713400440074e-07,
      "logits/chosen": 3.519763708114624,
      "logits/rejected": 4.384012699127197,
      "logps/chosen": -21.77133560180664,
      "logps/rejected": -29.07388687133789,
      "loss": 0.1378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16306172311306,
      "rewards/margins": 3.332913875579834,
      "rewards/rejected": -3.1698520183563232,
      "step": 411
    },
    {
      "epoch": 6.983050847457627,
      "grad_norm": 10.90365370777316,
      "learning_rate": 4.112627238062238e-07,
      "logits/chosen": 3.6548125743865967,
      "logits/rejected": 4.417596340179443,
      "logps/chosen": -13.329391479492188,
      "logps/rejected": -23.57076644897461,
      "loss": 0.1444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.530329704284668,
      "rewards/margins": 2.5617852210998535,
      "rewards/rejected": -2.0314555168151855,
      "step": 412
    },
    {
      "epoch": 7.0,
      "grad_norm": 13.2953949469969,
      "learning_rate": 4.106969024216348e-07,
      "logits/chosen": 3.9974312782287598,
      "logits/rejected": 4.03011417388916,
      "logps/chosen": -15.118673324584961,
      "logps/rejected": -29.512248992919922,
      "loss": 0.1894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01425081491470337,
      "rewards/margins": 3.2692325115203857,
      "rewards/rejected": -3.2834835052490234,
      "step": 413
    },
    {
      "epoch": 7.016949152542373,
      "grad_norm": 9.87165410115901,
      "learning_rate": 4.101296748020533e-07,
      "logits/chosen": -0.07437923550605774,
      "logits/rejected": 0.5111943483352661,
      "logps/chosen": -14.69372844696045,
      "logps/rejected": -25.89691162109375,
      "loss": 0.1389,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21298867464065552,
      "rewards/margins": 3.466539144515991,
      "rewards/rejected": -3.2535505294799805,
      "step": 414
    },
    {
      "epoch": 7.033898305084746,
      "grad_norm": 10.23895017926183,
      "learning_rate": 4.09561045911205e-07,
      "logits/chosen": 5.7406206130981445,
      "logits/rejected": 6.087039947509766,
      "logps/chosen": -16.156099319458008,
      "logps/rejected": -25.66880226135254,
      "loss": 0.0953,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3853185176849365,
      "rewards/margins": 3.1551337242126465,
      "rewards/rejected": -2.769815444946289,
      "step": 415
    },
    {
      "epoch": 7.0508474576271185,
      "grad_norm": 8.915034579880647,
      "learning_rate": 4.0899102072507773e-07,
      "logits/chosen": 3.7955563068389893,
      "logits/rejected": 4.054969310760498,
      "logps/chosen": -14.184603691101074,
      "logps/rejected": -23.060760498046875,
      "loss": 0.1179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09845928102731705,
      "rewards/margins": 2.538086414337158,
      "rewards/rejected": -2.439626932144165,
      "step": 416
    },
    {
      "epoch": 7.067796610169491,
      "grad_norm": 9.239050917639268,
      "learning_rate": 4.084196042318783e-07,
      "logits/chosen": 0.5394778847694397,
      "logits/rejected": 0.6815662384033203,
      "logps/chosen": -18.747802734375,
      "logps/rejected": -30.199262619018555,
      "loss": 0.1287,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.28906530141830444,
      "rewards/margins": 2.6135823726654053,
      "rewards/rejected": -2.324517011642456,
      "step": 417
    },
    {
      "epoch": 7.084745762711864,
      "grad_norm": 9.872275462051524,
      "learning_rate": 4.0784680143198837e-07,
      "logits/chosen": 4.96520471572876,
      "logits/rejected": 6.231711387634277,
      "logps/chosen": -13.00781536102295,
      "logps/rejected": -29.61225128173828,
      "loss": 0.1289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14047710597515106,
      "rewards/margins": 4.104274749755859,
      "rewards/rejected": -3.9637980461120605,
      "step": 418
    },
    {
      "epoch": 7.101694915254237,
      "grad_norm": 8.491591923304918,
      "learning_rate": 4.0727261733792124e-07,
      "logits/chosen": 4.542697429656982,
      "logits/rejected": 4.926889419555664,
      "logps/chosen": -15.30163288116455,
      "logps/rejected": -26.00771713256836,
      "loss": 0.1146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32629522681236267,
      "rewards/margins": 2.9046356678009033,
      "rewards/rejected": -2.5783402919769287,
      "step": 419
    },
    {
      "epoch": 7.11864406779661,
      "grad_norm": 14.655705370533742,
      "learning_rate": 4.0669705697427754e-07,
      "logits/chosen": 0.6694058179855347,
      "logits/rejected": 1.186065912246704,
      "logps/chosen": -21.470455169677734,
      "logps/rejected": -32.79707717895508,
      "loss": 0.1201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5823428630828857,
      "rewards/margins": 3.7213852405548096,
      "rewards/rejected": -3.1390419006347656,
      "step": 420
    },
    {
      "epoch": 7.135593220338983,
      "grad_norm": 9.811884566086711,
      "learning_rate": 4.061201253777015e-07,
      "logits/chosen": 3.503796100616455,
      "logits/rejected": 3.606149435043335,
      "logps/chosen": -18.951021194458008,
      "logps/rejected": -23.3331241607666,
      "loss": 0.1272,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27215754985809326,
      "rewards/margins": 2.7002599239349365,
      "rewards/rejected": -2.428102493286133,
      "step": 421
    },
    {
      "epoch": 7.1525423728813555,
      "grad_norm": 8.9341598495611,
      "learning_rate": 4.0554182759683675e-07,
      "logits/chosen": 2.680142641067505,
      "logits/rejected": 3.6626601219177246,
      "logps/chosen": -10.236148834228516,
      "logps/rejected": -23.045013427734375,
      "loss": 0.1144,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.27169984579086304,
      "rewards/margins": 2.928940773010254,
      "rewards/rejected": -2.657240867614746,
      "step": 422
    },
    {
      "epoch": 7.169491525423728,
      "grad_norm": 11.341699653332553,
      "learning_rate": 4.049621686922823e-07,
      "logits/chosen": 4.442047119140625,
      "logits/rejected": 4.583933353424072,
      "logps/chosen": -20.456544876098633,
      "logps/rejected": -30.686235427856445,
      "loss": 0.1428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.31697165966033936,
      "rewards/margins": 2.6978225708007812,
      "rewards/rejected": -3.01479434967041,
      "step": 423
    },
    {
      "epoch": 7.186440677966102,
      "grad_norm": 9.639665090084435,
      "learning_rate": 4.0438115373654795e-07,
      "logits/chosen": 1.4850196838378906,
      "logits/rejected": 2.471409559249878,
      "logps/chosen": -19.277435302734375,
      "logps/rejected": -31.83528709411621,
      "loss": 0.1367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31030380725860596,
      "rewards/margins": 3.9561548233032227,
      "rewards/rejected": -3.6458511352539062,
      "step": 424
    },
    {
      "epoch": 7.203389830508475,
      "grad_norm": 10.000078800507277,
      "learning_rate": 4.0379878781401046e-07,
      "logits/chosen": 0.7568904757499695,
      "logits/rejected": 1.3338820934295654,
      "logps/chosen": -15.746091842651367,
      "logps/rejected": -29.87813949584961,
      "loss": 0.1318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3971250057220459,
      "rewards/margins": 3.3505163192749023,
      "rewards/rejected": -2.9533913135528564,
      "step": 425
    },
    {
      "epoch": 7.220338983050848,
      "grad_norm": 10.381904195353888,
      "learning_rate": 4.0321507602086836e-07,
      "logits/chosen": -0.01644599437713623,
      "logits/rejected": 1.1252076625823975,
      "logps/chosen": -17.216590881347656,
      "logps/rejected": -30.554401397705078,
      "loss": 0.1252,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3007734417915344,
      "rewards/margins": 3.6961429119110107,
      "rewards/rejected": -3.395369529724121,
      "step": 426
    },
    {
      "epoch": 7.237288135593221,
      "grad_norm": 9.150425635235424,
      "learning_rate": 4.026300234650979e-07,
      "logits/chosen": 2.1568071842193604,
      "logits/rejected": 2.3625097274780273,
      "logps/chosen": -19.713909149169922,
      "logps/rejected": -32.234867095947266,
      "loss": 0.1069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5371749401092529,
      "rewards/margins": 3.1868789196014404,
      "rewards/rejected": -2.6497037410736084,
      "step": 427
    },
    {
      "epoch": 7.254237288135593,
      "grad_norm": 8.714918171505428,
      "learning_rate": 4.020436352664079e-07,
      "logits/chosen": 1.7266515493392944,
      "logits/rejected": 2.308364152908325,
      "logps/chosen": -15.875692367553711,
      "logps/rejected": -25.839502334594727,
      "loss": 0.1141,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39192503690719604,
      "rewards/margins": 3.4913225173950195,
      "rewards/rejected": -3.099397659301758,
      "step": 428
    },
    {
      "epoch": 7.271186440677966,
      "grad_norm": 9.08065333589319,
      "learning_rate": 4.014559165561956e-07,
      "logits/chosen": 3.0672965049743652,
      "logits/rejected": 4.741214752197266,
      "logps/chosen": -16.432716369628906,
      "logps/rejected": -32.22150421142578,
      "loss": 0.1133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.271750807762146,
      "rewards/margins": 5.287177562713623,
      "rewards/rejected": -5.0154266357421875,
      "step": 429
    },
    {
      "epoch": 7.288135593220339,
      "grad_norm": 8.930294809336747,
      "learning_rate": 4.0086687247750095e-07,
      "logits/chosen": 3.319852113723755,
      "logits/rejected": 3.4241392612457275,
      "logps/chosen": -14.741172790527344,
      "logps/rejected": -23.185720443725586,
      "loss": 0.1161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5052129030227661,
      "rewards/margins": 2.2517757415771484,
      "rewards/rejected": -1.7465627193450928,
      "step": 430
    },
    {
      "epoch": 7.305084745762712,
      "grad_norm": 9.14004965474861,
      "learning_rate": 4.0027650818496226e-07,
      "logits/chosen": 4.515099048614502,
      "logits/rejected": 5.159193515777588,
      "logps/chosen": -15.581079483032227,
      "logps/rejected": -35.971229553222656,
      "loss": 0.1288,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.013070344924926758,
      "rewards/margins": 4.25930643081665,
      "rewards/rejected": -4.272377014160156,
      "step": 431
    },
    {
      "epoch": 7.322033898305085,
      "grad_norm": 9.813549653689446,
      "learning_rate": 3.996848288447707e-07,
      "logits/chosen": 2.0078516006469727,
      "logits/rejected": 2.6934330463409424,
      "logps/chosen": -12.148004531860352,
      "logps/rejected": -26.57069206237793,
      "loss": 0.1269,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11509425938129425,
      "rewards/margins": 3.352147102355957,
      "rewards/rejected": -3.2370529174804688,
      "step": 432
    },
    {
      "epoch": 7.338983050847458,
      "grad_norm": 8.984701012628799,
      "learning_rate": 3.9909183963462536e-07,
      "logits/chosen": 0.2953256368637085,
      "logits/rejected": 2.0255162715911865,
      "logps/chosen": -22.085187911987305,
      "logps/rejected": -33.014617919921875,
      "loss": 0.1274,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4210081696510315,
      "rewards/margins": 3.541736364364624,
      "rewards/rejected": -3.120728015899658,
      "step": 433
    },
    {
      "epoch": 7.3559322033898304,
      "grad_norm": 10.288893478868536,
      "learning_rate": 3.984975457436876e-07,
      "logits/chosen": 3.767810821533203,
      "logits/rejected": 4.347950458526611,
      "logps/chosen": -13.075026512145996,
      "logps/rejected": -26.276962280273438,
      "loss": 0.1332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17277248203754425,
      "rewards/margins": 3.197721481323242,
      "rewards/rejected": -3.024949073791504,
      "step": 434
    },
    {
      "epoch": 7.372881355932203,
      "grad_norm": 9.51515469393944,
      "learning_rate": 3.979019523725361e-07,
      "logits/chosen": 4.625918388366699,
      "logits/rejected": 4.527889728546143,
      "logps/chosen": -18.1661376953125,
      "logps/rejected": -20.289915084838867,
      "loss": 0.1262,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22275793552398682,
      "rewards/margins": 2.77077054977417,
      "rewards/rejected": -2.5480129718780518,
      "step": 435
    },
    {
      "epoch": 7.389830508474576,
      "grad_norm": 10.055043599285973,
      "learning_rate": 3.973050647331209e-07,
      "logits/chosen": 3.001349925994873,
      "logits/rejected": 3.237732410430908,
      "logps/chosen": -19.620223999023438,
      "logps/rejected": -32.204917907714844,
      "loss": 0.1159,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4234805703163147,
      "rewards/margins": 3.593975782394409,
      "rewards/rejected": -3.17049503326416,
      "step": 436
    },
    {
      "epoch": 7.406779661016949,
      "grad_norm": 9.794022274217578,
      "learning_rate": 3.967068880487181e-07,
      "logits/chosen": 2.001206159591675,
      "logits/rejected": 2.360400915145874,
      "logps/chosen": -16.585494995117188,
      "logps/rejected": -31.013713836669922,
      "loss": 0.1444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5124250054359436,
      "rewards/margins": 4.170086860656738,
      "rewards/rejected": -3.6576623916625977,
      "step": 437
    },
    {
      "epoch": 7.423728813559322,
      "grad_norm": 11.01890506925563,
      "learning_rate": 3.9610742755388406e-07,
      "logits/chosen": 4.098840713500977,
      "logits/rejected": 4.901972770690918,
      "logps/chosen": -14.838088989257812,
      "logps/rejected": -20.27056884765625,
      "loss": 0.1509,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4280650019645691,
      "rewards/margins": 2.6378211975097656,
      "rewards/rejected": -2.209756374359131,
      "step": 438
    },
    {
      "epoch": 7.440677966101695,
      "grad_norm": 9.451372970532205,
      "learning_rate": 3.955066884944094e-07,
      "logits/chosen": 0.9893157482147217,
      "logits/rejected": 1.3610438108444214,
      "logps/chosen": -20.96686363220215,
      "logps/rejected": -29.387542724609375,
      "loss": 0.1093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38434547185897827,
      "rewards/margins": 3.0744457244873047,
      "rewards/rejected": -2.6900997161865234,
      "step": 439
    },
    {
      "epoch": 7.4576271186440675,
      "grad_norm": 8.812937276501128,
      "learning_rate": 3.949046761272735e-07,
      "logits/chosen": 4.211581230163574,
      "logits/rejected": 4.303144454956055,
      "logps/chosen": -11.633298873901367,
      "logps/rejected": -22.921463012695312,
      "loss": 0.1111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6645989418029785,
      "rewards/margins": 2.8330225944519043,
      "rewards/rejected": -2.168423891067505,
      "step": 440
    },
    {
      "epoch": 7.47457627118644,
      "grad_norm": 8.39612269762287,
      "learning_rate": 3.9430139572059815e-07,
      "logits/chosen": 3.9839651584625244,
      "logits/rejected": 4.943660736083984,
      "logps/chosen": -20.955799102783203,
      "logps/rejected": -38.92570495605469,
      "loss": 0.1154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20799720287322998,
      "rewards/margins": 4.353493690490723,
      "rewards/rejected": -4.145496845245361,
      "step": 441
    },
    {
      "epoch": 7.491525423728813,
      "grad_norm": 9.062673269123717,
      "learning_rate": 3.9369685255360173e-07,
      "logits/chosen": 2.831712007522583,
      "logits/rejected": 3.3848414421081543,
      "logps/chosen": -17.006778717041016,
      "logps/rejected": -27.789405822753906,
      "loss": 0.0997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08365049958229065,
      "rewards/margins": 3.540269374847412,
      "rewards/rejected": -3.4566190242767334,
      "step": 442
    },
    {
      "epoch": 7.508474576271187,
      "grad_norm": 10.944641060023693,
      "learning_rate": 3.9309105191655247e-07,
      "logits/chosen": 1.4619724750518799,
      "logits/rejected": 1.7838622331619263,
      "logps/chosen": -14.890405654907227,
      "logps/rejected": -29.264156341552734,
      "loss": 0.1429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27657827734947205,
      "rewards/margins": 3.4900012016296387,
      "rewards/rejected": -3.2134225368499756,
      "step": 443
    },
    {
      "epoch": 7.52542372881356,
      "grad_norm": 8.665383852874081,
      "learning_rate": 3.924839991107229e-07,
      "logits/chosen": 2.904837131500244,
      "logits/rejected": 3.712893486022949,
      "logps/chosen": -20.52821922302246,
      "logps/rejected": -42.29707717895508,
      "loss": 0.1018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07331550121307373,
      "rewards/margins": 4.935427665710449,
      "rewards/rejected": -4.862112045288086,
      "step": 444
    },
    {
      "epoch": 7.5423728813559325,
      "grad_norm": 9.29215580691363,
      "learning_rate": 3.918756994483429e-07,
      "logits/chosen": 5.155490398406982,
      "logits/rejected": 6.437995910644531,
      "logps/chosen": -12.761405944824219,
      "logps/rejected": -30.25531005859375,
      "loss": 0.1124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3791295289993286,
      "rewards/margins": 3.675171136856079,
      "rewards/rejected": -3.296041488647461,
      "step": 445
    },
    {
      "epoch": 7.559322033898305,
      "grad_norm": 8.788116788401233,
      "learning_rate": 3.912661582525536e-07,
      "logits/chosen": 2.623152256011963,
      "logits/rejected": 3.0128703117370605,
      "logps/chosen": -18.31006622314453,
      "logps/rejected": -27.341602325439453,
      "loss": 0.126,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6639620065689087,
      "rewards/margins": 2.841630697250366,
      "rewards/rejected": -2.177668571472168,
      "step": 446
    },
    {
      "epoch": 7.576271186440678,
      "grad_norm": 8.59243714374398,
      "learning_rate": 3.906553808573604e-07,
      "logits/chosen": 2.9111697673797607,
      "logits/rejected": 3.34836745262146,
      "logps/chosen": -16.26513671875,
      "logps/rejected": -26.296367645263672,
      "loss": 0.1282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3604896068572998,
      "rewards/margins": 3.5548095703125,
      "rewards/rejected": -3.1943202018737793,
      "step": 447
    },
    {
      "epoch": 7.593220338983051,
      "grad_norm": 9.388091105308293,
      "learning_rate": 3.9004337260758644e-07,
      "logits/chosen": 3.1576616764068604,
      "logits/rejected": 3.7552833557128906,
      "logps/chosen": -15.407849311828613,
      "logps/rejected": -30.661590576171875,
      "loss": 0.1299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30992385745048523,
      "rewards/margins": 3.318376064300537,
      "rewards/rejected": -3.0084524154663086,
      "step": 448
    },
    {
      "epoch": 7.610169491525424,
      "grad_norm": 11.069966787411271,
      "learning_rate": 3.894301388588264e-07,
      "logits/chosen": 1.9091204404830933,
      "logits/rejected": 2.0430729389190674,
      "logps/chosen": -19.71815299987793,
      "logps/rejected": -27.437362670898438,
      "loss": 0.1598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05926653742790222,
      "rewards/margins": 3.001842498779297,
      "rewards/rejected": -2.942575693130493,
      "step": 449
    },
    {
      "epoch": 7.627118644067797,
      "grad_norm": 8.54855711232238,
      "learning_rate": 3.888156849773985e-07,
      "logits/chosen": 3.887667179107666,
      "logits/rejected": 5.632800102233887,
      "logps/chosen": -16.554094314575195,
      "logps/rejected": -27.82715606689453,
      "loss": 0.1129,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0867987871170044,
      "rewards/margins": 3.4089887142181396,
      "rewards/rejected": -3.4957876205444336,
      "step": 450
    },
    {
      "epoch": 7.6440677966101696,
      "grad_norm": 9.519539340294198,
      "learning_rate": 3.882000163402983e-07,
      "logits/chosen": 5.304140567779541,
      "logits/rejected": 5.604523658752441,
      "logps/chosen": -20.477975845336914,
      "logps/rejected": -31.88489532470703,
      "loss": 0.126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03972455859184265,
      "rewards/margins": 3.0314011573791504,
      "rewards/rejected": -3.0711255073547363,
      "step": 451
    },
    {
      "epoch": 7.661016949152542,
      "grad_norm": 10.369425267842848,
      "learning_rate": 3.8758313833515186e-07,
      "logits/chosen": 3.0293540954589844,
      "logits/rejected": 3.7426505088806152,
      "logps/chosen": -16.383888244628906,
      "logps/rejected": -28.657669067382812,
      "loss": 0.1347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2297368347644806,
      "rewards/margins": 4.051611423492432,
      "rewards/rejected": -4.28134822845459,
      "step": 452
    },
    {
      "epoch": 7.677966101694915,
      "grad_norm": 10.511612229614789,
      "learning_rate": 3.86965056360168e-07,
      "logits/chosen": 1.9586790800094604,
      "logits/rejected": 2.216707229614258,
      "logps/chosen": -14.328598022460938,
      "logps/rejected": -25.2967586517334,
      "loss": 0.1355,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4405815005302429,
      "rewards/margins": 2.9190452098846436,
      "rewards/rejected": -2.478464126586914,
      "step": 453
    },
    {
      "epoch": 7.694915254237288,
      "grad_norm": 10.275712619690644,
      "learning_rate": 3.8634577582409115e-07,
      "logits/chosen": 3.4585013389587402,
      "logits/rejected": 3.590578317642212,
      "logps/chosen": -8.868934631347656,
      "logps/rejected": -29.173030853271484,
      "loss": 0.1482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42794957756996155,
      "rewards/margins": 3.5148518085479736,
      "rewards/rejected": -3.086902141571045,
      "step": 454
    },
    {
      "epoch": 7.711864406779661,
      "grad_norm": 8.535758374035677,
      "learning_rate": 3.857253021461545e-07,
      "logits/chosen": 1.1870063543319702,
      "logits/rejected": 1.9431824684143066,
      "logps/chosen": -17.088212966918945,
      "logps/rejected": -25.219560623168945,
      "loss": 0.1255,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2807660698890686,
      "rewards/margins": 2.8102126121520996,
      "rewards/rejected": -2.529446601867676,
      "step": 455
    },
    {
      "epoch": 7.728813559322034,
      "grad_norm": 8.239820038925641,
      "learning_rate": 3.8510364075603185e-07,
      "logits/chosen": 2.6802639961242676,
      "logits/rejected": 3.867419958114624,
      "logps/chosen": -15.612676620483398,
      "logps/rejected": -36.14496612548828,
      "loss": 0.0925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16128285229206085,
      "rewards/margins": 5.361781120300293,
      "rewards/rejected": -5.523064613342285,
      "step": 456
    },
    {
      "epoch": 7.745762711864407,
      "grad_norm": 9.780405360243819,
      "learning_rate": 3.84480797093791e-07,
      "logits/chosen": 2.6172919273376465,
      "logits/rejected": 3.19209623336792,
      "logps/chosen": -11.52426528930664,
      "logps/rejected": -22.12085723876953,
      "loss": 0.1213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30407530069351196,
      "rewards/margins": 3.035153865814209,
      "rewards/rejected": -2.731078624725342,
      "step": 457
    },
    {
      "epoch": 7.762711864406779,
      "grad_norm": 8.55870058153661,
      "learning_rate": 3.8385677660984514e-07,
      "logits/chosen": 4.2267279624938965,
      "logits/rejected": 5.226436614990234,
      "logps/chosen": -16.732118606567383,
      "logps/rejected": -35.05083465576172,
      "loss": 0.1035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3358514904975891,
      "rewards/margins": 4.643409252166748,
      "rewards/rejected": -4.307557582855225,
      "step": 458
    },
    {
      "epoch": 7.779661016949152,
      "grad_norm": 9.816530221791213,
      "learning_rate": 3.83231584764906e-07,
      "logits/chosen": -1.649599313735962,
      "logits/rejected": 0.7722344398498535,
      "logps/chosen": -19.57205581665039,
      "logps/rejected": -30.491806030273438,
      "loss": 0.1411,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12148743867874146,
      "rewards/margins": 3.641042470932007,
      "rewards/rejected": -3.51955509185791,
      "step": 459
    },
    {
      "epoch": 7.796610169491525,
      "grad_norm": 9.580671233027974,
      "learning_rate": 3.826052270299356e-07,
      "logits/chosen": 3.5561270713806152,
      "logits/rejected": 3.568795680999756,
      "logps/chosen": -17.304832458496094,
      "logps/rejected": -27.24372673034668,
      "loss": 0.123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1926884800195694,
      "rewards/margins": 3.580801486968994,
      "rewards/rejected": -3.388113260269165,
      "step": 460
    },
    {
      "epoch": 7.813559322033898,
      "grad_norm": 9.146891343129145,
      "learning_rate": 3.8197770888609846e-07,
      "logits/chosen": 2.516333818435669,
      "logits/rejected": 3.34853458404541,
      "logps/chosen": -15.37035083770752,
      "logps/rejected": -26.412580490112305,
      "loss": 0.1266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.251945436000824,
      "rewards/margins": 3.64552903175354,
      "rewards/rejected": -3.3935835361480713,
      "step": 461
    },
    {
      "epoch": 7.830508474576272,
      "grad_norm": 9.710956700917844,
      "learning_rate": 3.813490358247137e-07,
      "logits/chosen": 1.6899590492248535,
      "logits/rejected": 2.5557150840759277,
      "logps/chosen": -15.23322582244873,
      "logps/rejected": -33.32341003417969,
      "loss": 0.1115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.556183934211731,
      "rewards/margins": 3.5709474086761475,
      "rewards/rejected": -3.014763355255127,
      "step": 462
    },
    {
      "epoch": 7.847457627118644,
      "grad_norm": 8.690500202952354,
      "learning_rate": 3.807192133472069e-07,
      "logits/chosen": 4.0055131912231445,
      "logits/rejected": 4.724710941314697,
      "logps/chosen": -13.636266708374023,
      "logps/rejected": -28.918319702148438,
      "loss": 0.0946,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.31850284337997437,
      "rewards/margins": 3.9023942947387695,
      "rewards/rejected": -3.5838913917541504,
      "step": 463
    },
    {
      "epoch": 7.864406779661017,
      "grad_norm": 8.955582537191807,
      "learning_rate": 3.80088246965062e-07,
      "logits/chosen": 2.3977673053741455,
      "logits/rejected": 3.1791818141937256,
      "logps/chosen": -11.766952514648438,
      "logps/rejected": -30.629945755004883,
      "loss": 0.1064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18112745881080627,
      "rewards/margins": 4.072152614593506,
      "rewards/rejected": -3.8910253047943115,
      "step": 464
    },
    {
      "epoch": 7.88135593220339,
      "grad_norm": 9.28701234024736,
      "learning_rate": 3.794561421997734e-07,
      "logits/chosen": -0.8194286823272705,
      "logits/rejected": 0.3698238730430603,
      "logps/chosen": -16.36358642578125,
      "logps/rejected": -26.18773078918457,
      "loss": 0.1126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6522509455680847,
      "rewards/margins": 2.9714879989624023,
      "rewards/rejected": -2.319237232208252,
      "step": 465
    },
    {
      "epoch": 7.898305084745763,
      "grad_norm": 8.823571317960623,
      "learning_rate": 3.78822904582797e-07,
      "logits/chosen": 0.573715090751648,
      "logits/rejected": 2.2726151943206787,
      "logps/chosen": -15.998534202575684,
      "logps/rejected": -27.229896545410156,
      "loss": 0.1291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6307635307312012,
      "rewards/margins": 3.3138785362243652,
      "rewards/rejected": -2.683115243911743,
      "step": 466
    },
    {
      "epoch": 7.915254237288136,
      "grad_norm": 11.217955453701311,
      "learning_rate": 3.781885396555019e-07,
      "logits/chosen": 3.541079044342041,
      "logits/rejected": 4.791148662567139,
      "logps/chosen": -14.431351661682129,
      "logps/rejected": -32.49176788330078,
      "loss": 0.143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1487632840871811,
      "rewards/margins": 3.8755033016204834,
      "rewards/rejected": -3.7267403602600098,
      "step": 467
    },
    {
      "epoch": 7.932203389830509,
      "grad_norm": 8.59517424315105,
      "learning_rate": 3.775530529691227e-07,
      "logits/chosen": 3.7382278442382812,
      "logits/rejected": 4.195878028869629,
      "logps/chosen": -11.119057655334473,
      "logps/rejected": -25.03734588623047,
      "loss": 0.1092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38498881459236145,
      "rewards/margins": 3.7704718112945557,
      "rewards/rejected": -3.3854832649230957,
      "step": 468
    },
    {
      "epoch": 7.9491525423728815,
      "grad_norm": 8.901209729681504,
      "learning_rate": 3.7691645008470997e-07,
      "logits/chosen": -0.129108726978302,
      "logits/rejected": 0.6862486600875854,
      "logps/chosen": -14.666604995727539,
      "logps/rejected": -38.28477096557617,
      "loss": 0.126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20774927735328674,
      "rewards/margins": 4.353483200073242,
      "rewards/rejected": -4.145733833312988,
      "step": 469
    },
    {
      "epoch": 7.966101694915254,
      "grad_norm": 8.636425954622347,
      "learning_rate": 3.7627873657308206e-07,
      "logits/chosen": 1.7377265691757202,
      "logits/rejected": 2.822047710418701,
      "logps/chosen": -14.229039192199707,
      "logps/rejected": -29.386869430541992,
      "loss": 0.1252,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3835397958755493,
      "rewards/margins": 4.004900932312012,
      "rewards/rejected": -3.62136173248291,
      "step": 470
    },
    {
      "epoch": 7.983050847457627,
      "grad_norm": 8.838726675318133,
      "learning_rate": 3.7563991801477624e-07,
      "logits/chosen": -0.580298662185669,
      "logits/rejected": 0.00358683243393898,
      "logps/chosen": -20.286121368408203,
      "logps/rejected": -26.494842529296875,
      "loss": 0.1069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.030002586543560028,
      "rewards/margins": 3.3837900161743164,
      "rewards/rejected": -3.353787660598755,
      "step": 471
    },
    {
      "epoch": 8.0,
      "grad_norm": 10.328093966742186,
      "learning_rate": 3.75e-07,
      "logits/chosen": 0.8947811722755432,
      "logits/rejected": 2.392275094985962,
      "logps/chosen": -17.493534088134766,
      "logps/rejected": -23.69567108154297,
      "loss": 0.1471,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6442322731018066,
      "rewards/margins": 3.405353546142578,
      "rewards/rejected": -2.7611215114593506,
      "step": 472
    },
    {
      "epoch": 8.016949152542374,
      "grad_norm": 7.750172905554329,
      "learning_rate": 3.743589881285818e-07,
      "logits/chosen": 3.432040214538574,
      "logits/rejected": 3.4083125591278076,
      "logps/chosen": -19.61204719543457,
      "logps/rejected": -26.032976150512695,
      "loss": 0.1337,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05513594299554825,
      "rewards/margins": 2.830720901489258,
      "rewards/rejected": -2.8858566284179688,
      "step": 473
    },
    {
      "epoch": 8.033898305084746,
      "grad_norm": 9.092042783490344,
      "learning_rate": 3.737168880099223e-07,
      "logits/chosen": 4.506225109100342,
      "logits/rejected": 5.203121662139893,
      "logps/chosen": -20.774394989013672,
      "logps/rejected": -28.78338623046875,
      "loss": 0.1235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7455430626869202,
      "rewards/margins": 3.0526225566864014,
      "rewards/rejected": -2.307079792022705,
      "step": 474
    },
    {
      "epoch": 8.05084745762712,
      "grad_norm": 9.314515722517227,
      "learning_rate": 3.7307370526294553e-07,
      "logits/chosen": 2.7973568439483643,
      "logits/rejected": 3.653848648071289,
      "logps/chosen": -21.094676971435547,
      "logps/rejected": -27.839862823486328,
      "loss": 0.1412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3912617266178131,
      "rewards/margins": 3.751455307006836,
      "rewards/rejected": -3.3601937294006348,
      "step": 475
    },
    {
      "epoch": 8.067796610169491,
      "grad_norm": 8.64064476803114,
      "learning_rate": 3.724294455160491e-07,
      "logits/chosen": 2.164475440979004,
      "logits/rejected": 2.9439971446990967,
      "logps/chosen": -16.0778865814209,
      "logps/rejected": -32.343509674072266,
      "loss": 0.0957,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.5423865914344788,
      "rewards/margins": 3.9885687828063965,
      "rewards/rejected": -3.4461820125579834,
      "step": 476
    },
    {
      "epoch": 8.084745762711865,
      "grad_norm": 8.258600230037871,
      "learning_rate": 3.7178411440705556e-07,
      "logits/chosen": 3.7295310497283936,
      "logits/rejected": 4.5462646484375,
      "logps/chosen": -14.436502456665039,
      "logps/rejected": -28.71784019470215,
      "loss": 0.124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3619258999824524,
      "rewards/margins": 3.4388718605041504,
      "rewards/rejected": -3.076946496963501,
      "step": 477
    },
    {
      "epoch": 8.101694915254237,
      "grad_norm": 8.448006933926113,
      "learning_rate": 3.7113771758316255e-07,
      "logits/chosen": 4.252035617828369,
      "logits/rejected": 4.446700096130371,
      "logps/chosen": -19.428070068359375,
      "logps/rejected": -23.08380889892578,
      "loss": 0.122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7620154023170471,
      "rewards/margins": 2.585181951522827,
      "rewards/rejected": -1.8231666088104248,
      "step": 478
    },
    {
      "epoch": 8.11864406779661,
      "grad_norm": 6.910166322428136,
      "learning_rate": 3.704902607008938e-07,
      "logits/chosen": 0.2076493501663208,
      "logits/rejected": 1.2938125133514404,
      "logps/chosen": -21.162704467773438,
      "logps/rejected": -30.260038375854492,
      "loss": 0.0913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25260335206985474,
      "rewards/margins": 3.3553013801574707,
      "rewards/rejected": -3.1026980876922607,
      "step": 479
    },
    {
      "epoch": 8.135593220338983,
      "grad_norm": 8.235876606138394,
      "learning_rate": 3.698417494260494e-07,
      "logits/chosen": 1.5194463729858398,
      "logits/rejected": 3.1246232986450195,
      "logps/chosen": -19.262439727783203,
      "logps/rejected": -31.70151710510254,
      "loss": 0.0994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3763061761856079,
      "rewards/margins": 4.7519121170043945,
      "rewards/rejected": -4.375606536865234,
      "step": 480
    },
    {
      "epoch": 8.152542372881356,
      "grad_norm": 8.235302236969599,
      "learning_rate": 3.691921894336563e-07,
      "logits/chosen": 0.1965102255344391,
      "logits/rejected": 1.2409251928329468,
      "logps/chosen": -13.936543464660645,
      "logps/rejected": -26.925045013427734,
      "loss": 0.1128,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19482684135437012,
      "rewards/margins": 4.248325347900391,
      "rewards/rejected": -4.053498268127441,
      "step": 481
    },
    {
      "epoch": 8.169491525423728,
      "grad_norm": 7.368546080599964,
      "learning_rate": 3.685415864079185e-07,
      "logits/chosen": 2.3343710899353027,
      "logits/rejected": 3.2269225120544434,
      "logps/chosen": -19.352066040039062,
      "logps/rejected": -33.908836364746094,
      "loss": 0.0824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4641505777835846,
      "rewards/margins": 4.385346412658691,
      "rewards/rejected": -3.9211952686309814,
      "step": 482
    },
    {
      "epoch": 8.186440677966102,
      "grad_norm": 9.063756247108762,
      "learning_rate": 3.6788994604216764e-07,
      "logits/chosen": 0.6629161834716797,
      "logits/rejected": 1.4587095975875854,
      "logps/chosen": -11.309172630310059,
      "logps/rejected": -32.34671401977539,
      "loss": 0.1014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24906420707702637,
      "rewards/margins": 3.948303461074829,
      "rewards/rejected": -3.6992392539978027,
      "step": 483
    },
    {
      "epoch": 8.203389830508474,
      "grad_norm": 7.189037130415892,
      "learning_rate": 3.6723727403881275e-07,
      "logits/chosen": 1.5907362699508667,
      "logits/rejected": 3.4336838722229004,
      "logps/chosen": -18.35771942138672,
      "logps/rejected": -27.41173553466797,
      "loss": 0.0976,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.8187582492828369,
      "rewards/margins": 3.567868232727051,
      "rewards/rejected": -2.749110221862793,
      "step": 484
    },
    {
      "epoch": 8.220338983050848,
      "grad_norm": 7.784083778398003,
      "learning_rate": 3.665835761092908e-07,
      "logits/chosen": -1.079679012298584,
      "logits/rejected": -0.2996291220188141,
      "logps/chosen": -15.97018814086914,
      "logps/rejected": -25.22320556640625,
      "loss": 0.1025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6464424133300781,
      "rewards/margins": 3.0428719520568848,
      "rewards/rejected": -2.3964295387268066,
      "step": 485
    },
    {
      "epoch": 8.23728813559322,
      "grad_norm": 7.10522523168738,
      "learning_rate": 3.659288579740163e-07,
      "logits/chosen": 3.566469669342041,
      "logits/rejected": 4.723283767700195,
      "logps/chosen": -25.532649993896484,
      "logps/rejected": -30.565183639526367,
      "loss": 0.0953,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.48834341764450073,
      "rewards/margins": 3.6361196041107178,
      "rewards/rejected": -3.147775888442993,
      "step": 486
    },
    {
      "epoch": 8.254237288135593,
      "grad_norm": 6.861689922111202,
      "learning_rate": 3.6527312536233147e-07,
      "logits/chosen": 2.6697049140930176,
      "logits/rejected": 3.045163154602051,
      "logps/chosen": -15.023626327514648,
      "logps/rejected": -29.254968643188477,
      "loss": 0.0858,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3370720446109772,
      "rewards/margins": 3.555783987045288,
      "rewards/rejected": -3.2187116146087646,
      "step": 487
    },
    {
      "epoch": 8.271186440677965,
      "grad_norm": 7.755950966789172,
      "learning_rate": 3.646163840124561e-07,
      "logits/chosen": 1.4673826694488525,
      "logits/rejected": 2.106332540512085,
      "logps/chosen": -17.002397537231445,
      "logps/rejected": -26.75397300720215,
      "loss": 0.0894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4947773218154907,
      "rewards/margins": 3.349858522415161,
      "rewards/rejected": -2.855081081390381,
      "step": 488
    },
    {
      "epoch": 8.288135593220339,
      "grad_norm": 8.848971273791523,
      "learning_rate": 3.639586396714374e-07,
      "logits/chosen": 2.7336244583129883,
      "logits/rejected": 2.796771764755249,
      "logps/chosen": -12.618149757385254,
      "logps/rejected": -23.861251831054688,
      "loss": 0.1141,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30856549739837646,
      "rewards/margins": 3.3756675720214844,
      "rewards/rejected": -3.0671024322509766,
      "step": 489
    },
    {
      "epoch": 8.305084745762711,
      "grad_norm": 9.532540271403574,
      "learning_rate": 3.6329989809509933e-07,
      "logits/chosen": 0.5570180416107178,
      "logits/rejected": 1.2071716785430908,
      "logps/chosen": -15.186332702636719,
      "logps/rejected": -30.408462524414062,
      "loss": 0.1157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5822116136550903,
      "rewards/margins": 4.023227214813232,
      "rewards/rejected": -3.4410154819488525,
      "step": 490
    },
    {
      "epoch": 8.322033898305085,
      "grad_norm": 8.215897245877398,
      "learning_rate": 3.626401650479927e-07,
      "logits/chosen": 2.7207868099212646,
      "logits/rejected": 3.3667051792144775,
      "logps/chosen": -14.001188278198242,
      "logps/rejected": -25.76093292236328,
      "loss": 0.1102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2106812298297882,
      "rewards/margins": 3.618776798248291,
      "rewards/rejected": -3.408095359802246,
      "step": 491
    },
    {
      "epoch": 8.338983050847457,
      "grad_norm": 9.754620312298025,
      "learning_rate": 3.6197944630334465e-07,
      "logits/chosen": -0.6451621055603027,
      "logits/rejected": -0.036910831928253174,
      "logps/chosen": -15.894433975219727,
      "logps/rejected": -28.29058837890625,
      "loss": 0.1122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5367656946182251,
      "rewards/margins": 3.88037109375,
      "rewards/rejected": -3.3436052799224854,
      "step": 492
    },
    {
      "epoch": 8.35593220338983,
      "grad_norm": 8.705719115692244,
      "learning_rate": 3.6131774764300785e-07,
      "logits/chosen": 3.9698734283447266,
      "logits/rejected": 3.9669480323791504,
      "logps/chosen": -15.973098754882812,
      "logps/rejected": -21.522592544555664,
      "loss": 0.1106,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.44191816449165344,
      "rewards/margins": 2.4768028259277344,
      "rewards/rejected": -2.0348846912384033,
      "step": 493
    },
    {
      "epoch": 8.372881355932204,
      "grad_norm": 6.277591531069204,
      "learning_rate": 3.6065507485741e-07,
      "logits/chosen": 2.5741474628448486,
      "logits/rejected": 3.6729209423065186,
      "logps/chosen": -15.507135391235352,
      "logps/rejected": -27.956684112548828,
      "loss": 0.0731,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36563146114349365,
      "rewards/margins": 4.5596771240234375,
      "rewards/rejected": -4.194045066833496,
      "step": 494
    },
    {
      "epoch": 8.389830508474576,
      "grad_norm": 7.923605032689377,
      "learning_rate": 3.5999143374550334e-07,
      "logits/chosen": -0.3899872601032257,
      "logits/rejected": 0.7094336748123169,
      "logps/chosen": -20.5274715423584,
      "logps/rejected": -33.68539810180664,
      "loss": 0.1082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26818332076072693,
      "rewards/margins": 3.903979778289795,
      "rewards/rejected": -3.635796308517456,
      "step": 495
    },
    {
      "epoch": 8.40677966101695,
      "grad_norm": 8.438194296029398,
      "learning_rate": 3.593268301147139e-07,
      "logits/chosen": 3.5322861671447754,
      "logits/rejected": 4.039117336273193,
      "logps/chosen": -15.595931053161621,
      "logps/rejected": -25.944929122924805,
      "loss": 0.1163,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5988155603408813,
      "rewards/margins": 3.9967434406280518,
      "rewards/rejected": -3.397927761077881,
      "step": 496
    },
    {
      "epoch": 8.423728813559322,
      "grad_norm": 8.599338534257203,
      "learning_rate": 3.586612697808902e-07,
      "logits/chosen": 2.3289849758148193,
      "logits/rejected": 3.1125564575195312,
      "logps/chosen": -16.034414291381836,
      "logps/rejected": -26.47426986694336,
      "loss": 0.1103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3328191041946411,
      "rewards/margins": 3.7066524028778076,
      "rewards/rejected": -3.373833417892456,
      "step": 497
    },
    {
      "epoch": 8.440677966101696,
      "grad_norm": 9.964386548019352,
      "learning_rate": 3.579947585682532e-07,
      "logits/chosen": 0.27496790885925293,
      "logits/rejected": 1.4074618816375732,
      "logps/chosen": -16.960861206054688,
      "logps/rejected": -39.56070327758789,
      "loss": 0.0938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18327996134757996,
      "rewards/margins": 4.648859024047852,
      "rewards/rejected": -4.465579032897949,
      "step": 498
    },
    {
      "epoch": 8.457627118644067,
      "grad_norm": 7.602211724729448,
      "learning_rate": 3.573273023093446e-07,
      "logits/chosen": 3.052069664001465,
      "logits/rejected": 4.1386637687683105,
      "logps/chosen": -23.612491607666016,
      "logps/rejected": -38.5206298828125,
      "loss": 0.0975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14806516468524933,
      "rewards/margins": 4.848412990570068,
      "rewards/rejected": -4.700348377227783,
      "step": 499
    },
    {
      "epoch": 8.474576271186441,
      "grad_norm": 7.622477576386271,
      "learning_rate": 3.5665890684497605e-07,
      "logits/chosen": 2.440854072570801,
      "logits/rejected": 2.863269329071045,
      "logps/chosen": -16.440420150756836,
      "logps/rejected": -33.62314224243164,
      "loss": 0.0925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2879524827003479,
      "rewards/margins": 4.277005672454834,
      "rewards/rejected": -4.564958095550537,
      "step": 500
    },
    {
      "epoch": 8.491525423728813,
      "grad_norm": 8.484995585922134,
      "learning_rate": 3.559895780241781e-07,
      "logits/chosen": 2.012855291366577,
      "logits/rejected": 3.6795272827148438,
      "logps/chosen": -21.541399002075195,
      "logps/rejected": -24.667354583740234,
      "loss": 0.1068,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.7486720085144043,
      "rewards/margins": 3.1486990451812744,
      "rewards/rejected": -2.400026798248291,
      "step": 501
    },
    {
      "epoch": 8.508474576271187,
      "grad_norm": 7.646847524397191,
      "learning_rate": 3.553193217041489e-07,
      "logits/chosen": 1.8552759885787964,
      "logits/rejected": 2.502427577972412,
      "logps/chosen": -15.881732940673828,
      "logps/rejected": -27.009502410888672,
      "loss": 0.0913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28931599855422974,
      "rewards/margins": 3.4781432151794434,
      "rewards/rejected": -3.1888270378112793,
      "step": 502
    },
    {
      "epoch": 8.525423728813559,
      "grad_norm": 7.1749270448200635,
      "learning_rate": 3.546481437502032e-07,
      "logits/chosen": 0.3308050036430359,
      "logits/rejected": 1.2291865348815918,
      "logps/chosen": -16.362823486328125,
      "logps/rejected": -29.266414642333984,
      "loss": 0.1001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.471590518951416,
      "rewards/margins": 3.7012906074523926,
      "rewards/rejected": -3.2296998500823975,
      "step": 503
    },
    {
      "epoch": 8.542372881355933,
      "grad_norm": 6.72550755962128,
      "learning_rate": 3.539760500357206e-07,
      "logits/chosen": 1.2928094863891602,
      "logits/rejected": 2.6595168113708496,
      "logps/chosen": -19.021821975708008,
      "logps/rejected": -27.033843994140625,
      "loss": 0.0777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34017112851142883,
      "rewards/margins": 3.2771692276000977,
      "rewards/rejected": -2.936998128890991,
      "step": 504
    },
    {
      "epoch": 8.559322033898304,
      "grad_norm": 6.378108042023536,
      "learning_rate": 3.533030464420945e-07,
      "logits/chosen": 1.789313793182373,
      "logits/rejected": 2.7768216133117676,
      "logps/chosen": -19.944744110107422,
      "logps/rejected": -34.9632682800293,
      "loss": 0.0764,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15939801931381226,
      "rewards/margins": 4.254008769989014,
      "rewards/rejected": -4.413406848907471,
      "step": 505
    },
    {
      "epoch": 8.576271186440678,
      "grad_norm": 8.242800714593855,
      "learning_rate": 3.526291388586806e-07,
      "logits/chosen": 0.0323818176984787,
      "logits/rejected": 0.5169703364372253,
      "logps/chosen": -13.888110160827637,
      "logps/rejected": -29.98931884765625,
      "loss": 0.1098,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1510196030139923,
      "rewards/margins": 2.6823315620422363,
      "rewards/rejected": -2.531311511993408,
      "step": 506
    },
    {
      "epoch": 8.59322033898305,
      "grad_norm": 8.403972897386142,
      "learning_rate": 3.5195433318274515e-07,
      "logits/chosen": 4.257566452026367,
      "logits/rejected": 5.3338093757629395,
      "logps/chosen": -20.00336456298828,
      "logps/rejected": -34.83162307739258,
      "loss": 0.0927,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1355169117450714,
      "rewards/margins": 4.166006565093994,
      "rewards/rejected": -4.301523685455322,
      "step": 507
    },
    {
      "epoch": 8.610169491525424,
      "grad_norm": 7.373643813935829,
      "learning_rate": 3.5127863531941335e-07,
      "logits/chosen": 1.3813129663467407,
      "logits/rejected": 1.631213665008545,
      "logps/chosen": -17.48348617553711,
      "logps/rejected": -37.02750015258789,
      "loss": 0.0825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10136617720127106,
      "rewards/margins": 4.755850791931152,
      "rewards/rejected": -4.857217311859131,
      "step": 508
    },
    {
      "epoch": 8.627118644067796,
      "grad_norm": 8.315993274864201,
      "learning_rate": 3.5060205118161816e-07,
      "logits/chosen": 1.8312853574752808,
      "logits/rejected": 2.7615444660186768,
      "logps/chosen": -22.70886993408203,
      "logps/rejected": -27.375215530395508,
      "loss": 0.1118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6258946657180786,
      "rewards/margins": 4.108339786529541,
      "rewards/rejected": -3.4824447631835938,
      "step": 509
    },
    {
      "epoch": 8.64406779661017,
      "grad_norm": 8.304627193892093,
      "learning_rate": 3.49924586690048e-07,
      "logits/chosen": 0.5513447523117065,
      "logits/rejected": 1.1990902423858643,
      "logps/chosen": -18.463212966918945,
      "logps/rejected": -23.737966537475586,
      "loss": 0.1082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0850096940994263,
      "rewards/margins": 3.221740484237671,
      "rewards/rejected": -2.136730670928955,
      "step": 510
    },
    {
      "epoch": 8.661016949152543,
      "grad_norm": 7.118882711880802,
      "learning_rate": 3.4924624777309504e-07,
      "logits/chosen": 0.539596676826477,
      "logits/rejected": 1.4375852346420288,
      "logps/chosen": -15.951204299926758,
      "logps/rejected": -37.94011688232422,
      "loss": 0.0758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2643980383872986,
      "rewards/margins": 4.801435470581055,
      "rewards/rejected": -5.065833568572998,
      "step": 511
    },
    {
      "epoch": 8.677966101694915,
      "grad_norm": 7.605042914781135,
      "learning_rate": 3.4856704036680355e-07,
      "logits/chosen": 0.71272873878479,
      "logits/rejected": 1.4662222862243652,
      "logps/chosen": -14.760433197021484,
      "logps/rejected": -30.09404945373535,
      "loss": 0.0864,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23031166195869446,
      "rewards/margins": 3.1281630992889404,
      "rewards/rejected": -2.8978514671325684,
      "step": 512
    },
    {
      "epoch": 8.694915254237289,
      "grad_norm": 8.335339415371848,
      "learning_rate": 3.4788697041481786e-07,
      "logits/chosen": 0.4760729670524597,
      "logits/rejected": 2.0185904502868652,
      "logps/chosen": -14.30174446105957,
      "logps/rejected": -34.21358871459961,
      "loss": 0.105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6585089564323425,
      "rewards/margins": 4.3875041007995605,
      "rewards/rejected": -3.728994846343994,
      "step": 513
    },
    {
      "epoch": 8.711864406779661,
      "grad_norm": 7.834839249551774,
      "learning_rate": 3.472060438683302e-07,
      "logits/chosen": 0.7659852504730225,
      "logits/rejected": 2.2133705615997314,
      "logps/chosen": -21.14748191833496,
      "logps/rejected": -33.81575393676758,
      "loss": 0.1055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6409199833869934,
      "rewards/margins": 4.414564609527588,
      "rewards/rejected": -3.7736446857452393,
      "step": 514
    },
    {
      "epoch": 8.728813559322035,
      "grad_norm": 8.345623124734605,
      "learning_rate": 3.4652426668602863e-07,
      "logits/chosen": 1.0636900663375854,
      "logits/rejected": 2.09614634513855,
      "logps/chosen": -13.39885139465332,
      "logps/rejected": -24.176618576049805,
      "loss": 0.1031,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.15705546736717224,
      "rewards/margins": 3.6203300952911377,
      "rewards/rejected": -3.7773852348327637,
      "step": 515
    },
    {
      "epoch": 8.745762711864407,
      "grad_norm": 8.080741957241715,
      "learning_rate": 3.4584164483404535e-07,
      "logits/chosen": 1.1965031623840332,
      "logits/rejected": 2.299811363220215,
      "logps/chosen": -13.63241195678711,
      "logps/rejected": -23.092220306396484,
      "loss": 0.0903,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31253504753112793,
      "rewards/margins": 3.104691982269287,
      "rewards/rejected": -2.79215669631958,
      "step": 516
    },
    {
      "epoch": 8.76271186440678,
      "grad_norm": 8.47589990486935,
      "learning_rate": 3.4515818428590393e-07,
      "logits/chosen": 1.4136298894882202,
      "logits/rejected": 2.353076696395874,
      "logps/chosen": -16.381393432617188,
      "logps/rejected": -27.299104690551758,
      "loss": 0.091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3608657121658325,
      "rewards/margins": 3.4944076538085938,
      "rewards/rejected": -3.1335418224334717,
      "step": 517
    },
    {
      "epoch": 8.779661016949152,
      "grad_norm": 7.546298321983651,
      "learning_rate": 3.444738910224671e-07,
      "logits/chosen": 0.9574793577194214,
      "logits/rejected": 1.109427809715271,
      "logps/chosen": -16.541025161743164,
      "logps/rejected": -25.279096603393555,
      "loss": 0.106,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.5484856367111206,
      "rewards/margins": 2.9023871421813965,
      "rewards/rejected": -2.3539016246795654,
      "step": 518
    },
    {
      "epoch": 8.796610169491526,
      "grad_norm": 7.868010821435501,
      "learning_rate": 3.437887710318848e-07,
      "logits/chosen": 0.7616167068481445,
      "logits/rejected": 1.7057359218597412,
      "logps/chosen": -16.0111026763916,
      "logps/rejected": -28.748083114624023,
      "loss": 0.1059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7589426636695862,
      "rewards/margins": 3.7685017585754395,
      "rewards/rejected": -3.009559154510498,
      "step": 519
    },
    {
      "epoch": 8.813559322033898,
      "grad_norm": 7.77017102372031,
      "learning_rate": 3.4310283030954146e-07,
      "logits/chosen": -3.4271130561828613,
      "logits/rejected": -2.246941328048706,
      "logps/chosen": -20.427770614624023,
      "logps/rejected": -27.396621704101562,
      "loss": 0.1047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.33634325861930847,
      "rewards/margins": 4.049783706665039,
      "rewards/rejected": -3.71343994140625,
      "step": 520
    },
    {
      "epoch": 8.830508474576272,
      "grad_norm": 8.993381798716076,
      "learning_rate": 3.4241607485800363e-07,
      "logits/chosen": 4.1821794509887695,
      "logits/rejected": 5.369024753570557,
      "logps/chosen": -11.20211124420166,
      "logps/rejected": -31.801847457885742,
      "loss": 0.1192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26158803701400757,
      "rewards/margins": 4.944797039031982,
      "rewards/rejected": -4.68320894241333,
      "step": 521
    },
    {
      "epoch": 8.847457627118644,
      "grad_norm": 8.92274871440318,
      "learning_rate": 3.417285106869673e-07,
      "logits/chosen": 1.3400788307189941,
      "logits/rejected": 2.010988235473633,
      "logps/chosen": -19.691082000732422,
      "logps/rejected": -32.518096923828125,
      "loss": 0.099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15838858485221863,
      "rewards/margins": 4.223845481872559,
      "rewards/rejected": -4.065457344055176,
      "step": 522
    },
    {
      "epoch": 8.864406779661017,
      "grad_norm": 7.789159880434814,
      "learning_rate": 3.4104014381320555e-07,
      "logits/chosen": 3.6833863258361816,
      "logits/rejected": 3.613530158996582,
      "logps/chosen": -13.66424560546875,
      "logps/rejected": -29.675308227539062,
      "loss": 0.1182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25313207507133484,
      "rewards/margins": 4.033567905426025,
      "rewards/rejected": -3.780435562133789,
      "step": 523
    },
    {
      "epoch": 8.88135593220339,
      "grad_norm": 7.386491120135085,
      "learning_rate": 3.403509802605159e-07,
      "logits/chosen": 1.8282943964004517,
      "logits/rejected": 2.981198310852051,
      "logps/chosen": -12.948617935180664,
      "logps/rejected": -29.092636108398438,
      "loss": 0.085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14645610749721527,
      "rewards/margins": 4.348095893859863,
      "rewards/rejected": -4.201639652252197,
      "step": 524
    },
    {
      "epoch": 8.898305084745763,
      "grad_norm": 8.599775984807337,
      "learning_rate": 3.396610260596673e-07,
      "logits/chosen": 1.1022146940231323,
      "logits/rejected": 2.0041048526763916,
      "logps/chosen": -17.718761444091797,
      "logps/rejected": -31.86960220336914,
      "loss": 0.1063,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.21742774546146393,
      "rewards/margins": 3.964740753173828,
      "rewards/rejected": -3.7473130226135254,
      "step": 525
    },
    {
      "epoch": 8.915254237288135,
      "grad_norm": 9.74148232877656,
      "learning_rate": 3.389702872483477e-07,
      "logits/chosen": -1.2812941074371338,
      "logits/rejected": -0.24090474843978882,
      "logps/chosen": -15.892394065856934,
      "logps/rejected": -24.498292922973633,
      "loss": 0.1197,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4295375943183899,
      "rewards/margins": 2.9514575004577637,
      "rewards/rejected": -2.5219202041625977,
      "step": 526
    },
    {
      "epoch": 8.932203389830509,
      "grad_norm": 8.085275421779402,
      "learning_rate": 3.38278769871111e-07,
      "logits/chosen": 0.18231505155563354,
      "logits/rejected": 1.6734726428985596,
      "logps/chosen": -14.41200065612793,
      "logps/rejected": -24.13207244873047,
      "loss": 0.1144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.45978984236717224,
      "rewards/margins": 3.5683913230895996,
      "rewards/rejected": -3.1086015701293945,
      "step": 527
    },
    {
      "epoch": 8.94915254237288,
      "grad_norm": 7.581029960559348,
      "learning_rate": 3.375864799793242e-07,
      "logits/chosen": 0.2579716444015503,
      "logits/rejected": 0.8376175761222839,
      "logps/chosen": -15.547198295593262,
      "logps/rejected": -22.652027130126953,
      "loss": 0.0996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4138343930244446,
      "rewards/margins": 3.735337495803833,
      "rewards/rejected": -3.321503162384033,
      "step": 528
    },
    {
      "epoch": 8.966101694915254,
      "grad_norm": 8.98481843263997,
      "learning_rate": 3.368934236311143e-07,
      "logits/chosen": 0.5636337995529175,
      "logits/rejected": 0.6224699020385742,
      "logps/chosen": -19.514921188354492,
      "logps/rejected": -30.25006675720215,
      "loss": 0.1024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2242983728647232,
      "rewards/margins": 3.2491581439971924,
      "rewards/rejected": -3.02485990524292,
      "step": 529
    },
    {
      "epoch": 8.983050847457626,
      "grad_norm": 8.623348675400907,
      "learning_rate": 3.361996068913159e-07,
      "logits/chosen": -0.3652976155281067,
      "logits/rejected": 1.0697021484375,
      "logps/chosen": -16.017398834228516,
      "logps/rejected": -34.6381950378418,
      "loss": 0.1104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0027234703302383423,
      "rewards/margins": 4.782834529876709,
      "rewards/rejected": -4.785558223724365,
      "step": 530
    },
    {
      "epoch": 9.0,
      "grad_norm": 6.966863469240621,
      "learning_rate": 3.355050358314172e-07,
      "logits/chosen": 0.13183480501174927,
      "logits/rejected": 1.832362174987793,
      "logps/chosen": -15.880701065063477,
      "logps/rejected": -31.734975814819336,
      "loss": 0.0836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.018413469195365906,
      "rewards/margins": 4.415182113647461,
      "rewards/rejected": -4.433596134185791,
      "step": 531
    },
    {
      "epoch": 9.016949152542374,
      "grad_norm": 7.934422577021392,
      "learning_rate": 3.348097165295075e-07,
      "logits/chosen": 1.4493695497512817,
      "logits/rejected": 1.6005449295043945,
      "logps/chosen": -16.320154190063477,
      "logps/rejected": -30.15085792541504,
      "loss": 0.103,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.31645795702934265,
      "rewards/margins": 4.485091209411621,
      "rewards/rejected": -4.168633460998535,
      "step": 532
    },
    {
      "epoch": 9.033898305084746,
      "grad_norm": 7.861889123608613,
      "learning_rate": 3.341136550702241e-07,
      "logits/chosen": 3.3205459117889404,
      "logits/rejected": 3.6725425720214844,
      "logps/chosen": -18.245128631591797,
      "logps/rejected": -32.34790802001953,
      "loss": 0.0856,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05831918120384216,
      "rewards/margins": 4.102582931518555,
      "rewards/rejected": -4.044262886047363,
      "step": 533
    },
    {
      "epoch": 9.05084745762712,
      "grad_norm": 7.675292482188158,
      "learning_rate": 3.334168575446985e-07,
      "logits/chosen": -0.5090641379356384,
      "logits/rejected": 0.2972201108932495,
      "logps/chosen": -17.58763313293457,
      "logps/rejected": -30.431671142578125,
      "loss": 0.1073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13420388102531433,
      "rewards/margins": 3.9743454456329346,
      "rewards/rejected": -4.108549118041992,
      "step": 534
    },
    {
      "epoch": 9.067796610169491,
      "grad_norm": 6.42736068110364,
      "learning_rate": 3.327193300505035e-07,
      "logits/chosen": 0.4148287773132324,
      "logits/rejected": 0.887850821018219,
      "logps/chosen": -16.828723907470703,
      "logps/rejected": -36.98302459716797,
      "loss": 0.0818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18271449208259583,
      "rewards/margins": 4.25698709487915,
      "rewards/rejected": -4.074272155761719,
      "step": 535
    },
    {
      "epoch": 9.084745762711865,
      "grad_norm": 6.860146645240362,
      "learning_rate": 3.3202107869159967e-07,
      "logits/chosen": -1.478266716003418,
      "logits/rejected": -1.039851188659668,
      "logps/chosen": -20.26593017578125,
      "logps/rejected": -30.06897735595703,
      "loss": 0.0667,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8925938606262207,
      "rewards/margins": 4.3639235496521,
      "rewards/rejected": -3.4713294506073,
      "step": 536
    },
    {
      "epoch": 9.101694915254237,
      "grad_norm": 7.308444071525899,
      "learning_rate": 3.313221095782822e-07,
      "logits/chosen": 0.33444347977638245,
      "logits/rejected": 1.7593356370925903,
      "logps/chosen": -17.03250503540039,
      "logps/rejected": -31.375883102416992,
      "loss": 0.0858,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.5907961130142212,
      "rewards/margins": 4.174586772918701,
      "rewards/rejected": -3.5837903022766113,
      "step": 537
    },
    {
      "epoch": 9.11864406779661,
      "grad_norm": 7.017699161634787,
      "learning_rate": 3.306224288271272e-07,
      "logits/chosen": 4.382320404052734,
      "logits/rejected": 5.0999627113342285,
      "logps/chosen": -12.660648345947266,
      "logps/rejected": -29.118486404418945,
      "loss": 0.0845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3442675471305847,
      "rewards/margins": 3.5753989219665527,
      "rewards/rejected": -3.2311315536499023,
      "step": 538
    },
    {
      "epoch": 9.135593220338983,
      "grad_norm": 7.4522176589996585,
      "learning_rate": 3.2992204256093807e-07,
      "logits/chosen": -2.729804277420044,
      "logits/rejected": -2.560558319091797,
      "logps/chosen": -20.1125545501709,
      "logps/rejected": -32.09090042114258,
      "loss": 0.1015,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.01902557909488678,
      "rewards/margins": 3.7781434059143066,
      "rewards/rejected": -3.75911808013916,
      "step": 539
    },
    {
      "epoch": 9.152542372881356,
      "grad_norm": 7.283164358559854,
      "learning_rate": 3.2922095690869224e-07,
      "logits/chosen": 0.20417343080043793,
      "logits/rejected": 1.5824004411697388,
      "logps/chosen": -12.8890962600708,
      "logps/rejected": -31.73255729675293,
      "loss": 0.0836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18634510040283203,
      "rewards/margins": 5.074504375457764,
      "rewards/rejected": -4.88815975189209,
      "step": 540
    },
    {
      "epoch": 9.169491525423728,
      "grad_norm": 6.519601729714241,
      "learning_rate": 3.2851917800548725e-07,
      "logits/chosen": -0.5393965840339661,
      "logits/rejected": -0.480104923248291,
      "logps/chosen": -21.48811149597168,
      "logps/rejected": -34.56924819946289,
      "loss": 0.0886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3659580945968628,
      "rewards/margins": 4.822329044342041,
      "rewards/rejected": -4.456370830535889,
      "step": 541
    },
    {
      "epoch": 9.186440677966102,
      "grad_norm": 8.893575269669139,
      "learning_rate": 3.278167119924871e-07,
      "logits/chosen": 2.511479377746582,
      "logits/rejected": 3.0335445404052734,
      "logps/chosen": -21.226470947265625,
      "logps/rejected": -31.758705139160156,
      "loss": 0.1074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021360307931900024,
      "rewards/margins": 4.572617053985596,
      "rewards/rejected": -4.593977451324463,
      "step": 542
    },
    {
      "epoch": 9.203389830508474,
      "grad_norm": 7.151569424146644,
      "learning_rate": 3.2711356501686886e-07,
      "logits/chosen": 0.11793151497840881,
      "logits/rejected": 2.1041746139526367,
      "logps/chosen": -15.034038543701172,
      "logps/rejected": -31.883384704589844,
      "loss": 0.0931,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4007861316204071,
      "rewards/margins": 4.863787651062012,
      "rewards/rejected": -4.463001728057861,
      "step": 543
    },
    {
      "epoch": 9.220338983050848,
      "grad_norm": 5.900554985023887,
      "learning_rate": 3.2640974323176843e-07,
      "logits/chosen": -1.388903260231018,
      "logits/rejected": -0.5128521919250488,
      "logps/chosen": -11.40340518951416,
      "logps/rejected": -26.74514389038086,
      "loss": 0.0708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.631538987159729,
      "rewards/margins": 3.9903998374938965,
      "rewards/rejected": -3.358860969543457,
      "step": 544
    },
    {
      "epoch": 9.23728813559322,
      "grad_norm": 8.099563527529172,
      "learning_rate": 3.257052527962269e-07,
      "logits/chosen": -3.805596113204956,
      "logits/rejected": -2.9346771240234375,
      "logps/chosen": -15.774307250976562,
      "logps/rejected": -27.184717178344727,
      "loss": 0.096,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3437478840351105,
      "rewards/margins": 3.4009368419647217,
      "rewards/rejected": -3.0571885108947754,
      "step": 545
    },
    {
      "epoch": 9.254237288135593,
      "grad_norm": 8.00979281359722,
      "learning_rate": 3.250000998751365e-07,
      "logits/chosen": 0.29453498125076294,
      "logits/rejected": 1.2413694858551025,
      "logps/chosen": -17.441308975219727,
      "logps/rejected": -32.38224792480469,
      "loss": 0.0931,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14581690728664398,
      "rewards/margins": 3.907651662826538,
      "rewards/rejected": -4.053468227386475,
      "step": 546
    },
    {
      "epoch": 9.271186440677965,
      "grad_norm": 6.256499605525865,
      "learning_rate": 3.2429429063918694e-07,
      "logits/chosen": 1.1686536073684692,
      "logits/rejected": 1.2653439044952393,
      "logps/chosen": -14.776887893676758,
      "logps/rejected": -27.53533363342285,
      "loss": 0.0807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.022234037518501282,
      "rewards/margins": 3.4280288219451904,
      "rewards/rejected": -3.450263023376465,
      "step": 547
    },
    {
      "epoch": 9.288135593220339,
      "grad_norm": 7.36750818731658,
      "learning_rate": 3.235878312648112e-07,
      "logits/chosen": 0.6835775375366211,
      "logits/rejected": 1.4044272899627686,
      "logps/chosen": -11.718465805053711,
      "logps/rejected": -29.979129791259766,
      "loss": 0.1092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4759228229522705,
      "rewards/margins": 4.728598594665527,
      "rewards/rejected": -4.252676486968994,
      "step": 548
    },
    {
      "epoch": 9.305084745762711,
      "grad_norm": 7.267461085469627,
      "learning_rate": 3.2288072793413147e-07,
      "logits/chosen": 0.21600386500358582,
      "logits/rejected": 0.11410781741142273,
      "logps/chosen": -17.949951171875,
      "logps/rejected": -23.075180053710938,
      "loss": 0.0968,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7247830629348755,
      "rewards/margins": 3.1238279342651367,
      "rewards/rejected": -2.3990447521209717,
      "step": 549
    },
    {
      "epoch": 9.322033898305085,
      "grad_norm": 20.44947669682873,
      "learning_rate": 3.2217298683490525e-07,
      "logits/chosen": 1.887885332107544,
      "logits/rejected": 2.2117202281951904,
      "logps/chosen": -14.434246063232422,
      "logps/rejected": -26.846405029296875,
      "loss": 0.0903,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7232978940010071,
      "rewards/margins": 4.097745895385742,
      "rewards/rejected": -3.374448299407959,
      "step": 550
    },
    {
      "epoch": 9.338983050847457,
      "grad_norm": 6.959074038410189,
      "learning_rate": 3.214646141604709e-07,
      "logits/chosen": 0.24728596210479736,
      "logits/rejected": 1.4561734199523926,
      "logps/chosen": -26.268035888671875,
      "logps/rejected": -29.52174186706543,
      "loss": 0.0843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5730979442596436,
      "rewards/margins": 3.468724250793457,
      "rewards/rejected": -2.8956265449523926,
      "step": 551
    },
    {
      "epoch": 9.35593220338983,
      "grad_norm": 6.313077671551259,
      "learning_rate": 3.2075561610969347e-07,
      "logits/chosen": 1.3113412857055664,
      "logits/rejected": 2.5229415893554688,
      "logps/chosen": -19.23296546936035,
      "logps/rejected": -33.98227310180664,
      "loss": 0.0804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0007563233375549316,
      "rewards/margins": 4.5488176345825195,
      "rewards/rejected": -4.549574375152588,
      "step": 552
    },
    {
      "epoch": 9.372881355932204,
      "grad_norm": 6.765231684504796,
      "learning_rate": 3.200459988869111e-07,
      "logits/chosen": 2.283677101135254,
      "logits/rejected": 3.3818447589874268,
      "logps/chosen": -17.183874130249023,
      "logps/rejected": -27.654109954833984,
      "loss": 0.0714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10292646288871765,
      "rewards/margins": 3.9736781120300293,
      "rewards/rejected": -4.07660436630249,
      "step": 553
    },
    {
      "epoch": 9.389830508474576,
      "grad_norm": 7.653842264043208,
      "learning_rate": 3.193357687018797e-07,
      "logits/chosen": 3.6839277744293213,
      "logits/rejected": 3.4737772941589355,
      "logps/chosen": -15.96898078918457,
      "logps/rejected": -32.69147872924805,
      "loss": 0.0996,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.23338308930397034,
      "rewards/margins": 5.1718363761901855,
      "rewards/rejected": -4.938453674316406,
      "step": 554
    },
    {
      "epoch": 9.40677966101695,
      "grad_norm": 7.773525249469395,
      "learning_rate": 3.186249317697194e-07,
      "logits/chosen": 1.9703714847564697,
      "logits/rejected": 2.7468535900115967,
      "logps/chosen": -23.632442474365234,
      "logps/rejected": -31.607494354248047,
      "loss": 0.0793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16064058244228363,
      "rewards/margins": 4.493778228759766,
      "rewards/rejected": -4.333136558532715,
      "step": 555
    },
    {
      "epoch": 9.423728813559322,
      "grad_norm": 6.042060539510561,
      "learning_rate": 3.1791349431085965e-07,
      "logits/chosen": 1.4269167184829712,
      "logits/rejected": 2.471031427383423,
      "logps/chosen": -15.8950834274292,
      "logps/rejected": -30.275249481201172,
      "loss": 0.0718,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.443370521068573,
      "rewards/margins": 4.576178550720215,
      "rewards/rejected": -4.132808208465576,
      "step": 556
    },
    {
      "epoch": 9.440677966101696,
      "grad_norm": 7.04877899953803,
      "learning_rate": 3.1720146255098537e-07,
      "logits/chosen": -3.6721673011779785,
      "logits/rejected": -1.3306491374969482,
      "logps/chosen": -15.854928970336914,
      "logps/rejected": -34.22749328613281,
      "loss": 0.0755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24515631794929504,
      "rewards/margins": 5.318935394287109,
      "rewards/rejected": -5.073779106140137,
      "step": 557
    },
    {
      "epoch": 9.457627118644067,
      "grad_norm": 6.415901347555323,
      "learning_rate": 3.1648884272098177e-07,
      "logits/chosen": -1.1916613578796387,
      "logits/rejected": -0.4890459477901459,
      "logps/chosen": -12.988748550415039,
      "logps/rejected": -18.562076568603516,
      "loss": 0.0831,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3931862711906433,
      "rewards/margins": 2.553929328918457,
      "rewards/rejected": -2.160742998123169,
      "step": 558
    },
    {
      "epoch": 9.474576271186441,
      "grad_norm": 8.236180250931438,
      "learning_rate": 3.157756410568803e-07,
      "logits/chosen": -0.5213596820831299,
      "logits/rejected": 0.4198833107948303,
      "logps/chosen": -16.794715881347656,
      "logps/rejected": -24.685680389404297,
      "loss": 0.0992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.47782301902770996,
      "rewards/margins": 2.9782776832580566,
      "rewards/rejected": -2.5004544258117676,
      "step": 559
    },
    {
      "epoch": 9.491525423728813,
      "grad_norm": 6.682073953644928,
      "learning_rate": 3.150618637998041e-07,
      "logits/chosen": -1.3392764329910278,
      "logits/rejected": -0.44644010066986084,
      "logps/chosen": -14.473580360412598,
      "logps/rejected": -28.390796661376953,
      "loss": 0.1011,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.40848979353904724,
      "rewards/margins": 4.742112636566162,
      "rewards/rejected": -4.333622455596924,
      "step": 560
    },
    {
      "epoch": 9.508474576271187,
      "grad_norm": 7.9822120644527175,
      "learning_rate": 3.1434751719591305e-07,
      "logits/chosen": -2.3160512447357178,
      "logits/rejected": -2.051666736602783,
      "logps/chosen": -18.745946884155273,
      "logps/rejected": -34.98544692993164,
      "loss": 0.0725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22431953251361847,
      "rewards/margins": 3.5555598735809326,
      "rewards/rejected": -3.7798798084259033,
      "step": 561
    },
    {
      "epoch": 9.525423728813559,
      "grad_norm": 6.991538298987854,
      "learning_rate": 3.136326074963494e-07,
      "logits/chosen": 1.2757248878479004,
      "logits/rejected": 1.5806338787078857,
      "logps/chosen": -16.574125289916992,
      "logps/rejected": -25.393587112426758,
      "loss": 0.0897,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2125283181667328,
      "rewards/margins": 3.118790864944458,
      "rewards/rejected": -3.3313193321228027,
      "step": 562
    },
    {
      "epoch": 9.542372881355933,
      "grad_norm": 7.369679468767769,
      "learning_rate": 3.1291714095718294e-07,
      "logits/chosen": 2.265183448791504,
      "logits/rejected": 2.9056434631347656,
      "logps/chosen": -11.20726203918457,
      "logps/rejected": -30.12190818786621,
      "loss": 0.0899,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08223026990890503,
      "rewards/margins": 4.531458854675293,
      "rewards/rejected": -4.449228763580322,
      "step": 563
    },
    {
      "epoch": 9.559322033898304,
      "grad_norm": 6.569787571821808,
      "learning_rate": 3.122011238393562e-07,
      "logits/chosen": -0.13098454475402832,
      "logits/rejected": 0.32105502486228943,
      "logps/chosen": -11.81690788269043,
      "logps/rejected": -22.512004852294922,
      "loss": 0.0747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28879374265670776,
      "rewards/margins": 3.5874359607696533,
      "rewards/rejected": -3.298642158508301,
      "step": 564
    },
    {
      "epoch": 9.576271186440678,
      "grad_norm": 6.887546375786722,
      "learning_rate": 3.1148456240862993e-07,
      "logits/chosen": 1.671111822128296,
      "logits/rejected": 2.590162992477417,
      "logps/chosen": -18.62039566040039,
      "logps/rejected": -37.18769073486328,
      "loss": 0.0721,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12747378647327423,
      "rewards/margins": 5.317519187927246,
      "rewards/rejected": -5.444993019104004,
      "step": 565
    },
    {
      "epoch": 9.59322033898305,
      "grad_norm": 6.474331733578539,
      "learning_rate": 3.1076746293552785e-07,
      "logits/chosen": -0.6289358735084534,
      "logits/rejected": 0.30806756019592285,
      "logps/chosen": -13.156976699829102,
      "logps/rejected": -35.53199005126953,
      "loss": 0.078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07099157571792603,
      "rewards/margins": 5.9290876388549805,
      "rewards/rejected": -5.858095645904541,
      "step": 566
    },
    {
      "epoch": 9.610169491525424,
      "grad_norm": 6.754288182278723,
      "learning_rate": 3.1004983169528225e-07,
      "logits/chosen": 1.0062041282653809,
      "logits/rejected": 1.5405733585357666,
      "logps/chosen": -14.011225700378418,
      "logps/rejected": -28.679346084594727,
      "loss": 0.0821,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.006055355072021484,
      "rewards/margins": 4.468363285064697,
      "rewards/rejected": -4.462307929992676,
      "step": 567
    },
    {
      "epoch": 9.627118644067796,
      "grad_norm": 6.7447175824089785,
      "learning_rate": 3.0933167496777873e-07,
      "logits/chosen": 2.5769147872924805,
      "logits/rejected": 3.43656587600708,
      "logps/chosen": -14.334616661071777,
      "logps/rejected": -23.00216293334961,
      "loss": 0.0792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5674694776535034,
      "rewards/margins": 3.888597249984741,
      "rewards/rejected": -3.3211278915405273,
      "step": 568
    },
    {
      "epoch": 9.64406779661017,
      "grad_norm": 7.580248860572857,
      "learning_rate": 3.0861299903750115e-07,
      "logits/chosen": -2.0739083290100098,
      "logits/rejected": -1.222943902015686,
      "logps/chosen": -18.079673767089844,
      "logps/rejected": -33.84402847290039,
      "loss": 0.103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08615212887525558,
      "rewards/margins": 5.459482192993164,
      "rewards/rejected": -5.5456342697143555,
      "step": 569
    },
    {
      "epoch": 9.661016949152543,
      "grad_norm": 6.992408377021299,
      "learning_rate": 3.0789381019347724e-07,
      "logits/chosen": 1.04190993309021,
      "logits/rejected": 1.8616278171539307,
      "logps/chosen": -11.107009887695312,
      "logps/rejected": -24.896486282348633,
      "loss": 0.0826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5456734895706177,
      "rewards/margins": 3.182859182357788,
      "rewards/rejected": -2.63718581199646,
      "step": 570
    },
    {
      "epoch": 9.677966101694915,
      "grad_norm": 6.873279551179824,
      "learning_rate": 3.071741147292229e-07,
      "logits/chosen": 1.097784161567688,
      "logits/rejected": 1.5534999370574951,
      "logps/chosen": -19.162097930908203,
      "logps/rejected": -30.126440048217773,
      "loss": 0.0849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3364775478839874,
      "rewards/margins": 4.0454020500183105,
      "rewards/rejected": -3.7089245319366455,
      "step": 571
    },
    {
      "epoch": 9.694915254237289,
      "grad_norm": 7.3910530095138816,
      "learning_rate": 3.0645391894268734e-07,
      "logits/chosen": 0.8061460256576538,
      "logits/rejected": 0.8003032803535461,
      "logps/chosen": -18.292184829711914,
      "logps/rejected": -37.91918182373047,
      "loss": 0.0969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.048791974782943726,
      "rewards/margins": 4.811036586761475,
      "rewards/rejected": -4.859828472137451,
      "step": 572
    },
    {
      "epoch": 9.711864406779661,
      "grad_norm": 5.996192080077326,
      "learning_rate": 3.057332291361983e-07,
      "logits/chosen": -0.12891456484794617,
      "logits/rejected": 1.2491331100463867,
      "logps/chosen": -17.618732452392578,
      "logps/rejected": -30.310043334960938,
      "loss": 0.0722,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26622286438941956,
      "rewards/margins": 4.7754058837890625,
      "rewards/rejected": -4.509182929992676,
      "step": 573
    },
    {
      "epoch": 9.728813559322035,
      "grad_norm": 6.171874008126151,
      "learning_rate": 3.050120516164062e-07,
      "logits/chosen": -0.018170345574617386,
      "logits/rejected": 1.102428913116455,
      "logps/chosen": -17.116703033447266,
      "logps/rejected": -36.32782745361328,
      "loss": 0.0823,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4552394449710846,
      "rewards/margins": 5.033170700073242,
      "rewards/rejected": -4.577930927276611,
      "step": 574
    },
    {
      "epoch": 9.745762711864407,
      "grad_norm": 7.451739032357378,
      "learning_rate": 3.042903926942297e-07,
      "logits/chosen": -1.0211288928985596,
      "logits/rejected": 0.3760063350200653,
      "logps/chosen": -20.83000373840332,
      "logps/rejected": -33.21617889404297,
      "loss": 0.0854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09166756272315979,
      "rewards/margins": 4.8829121589660645,
      "rewards/rejected": -4.791244983673096,
      "step": 575
    },
    {
      "epoch": 9.76271186440678,
      "grad_norm": 6.577485413860199,
      "learning_rate": 3.0356825868480014e-07,
      "logits/chosen": 1.2451457977294922,
      "logits/rejected": 1.2695012092590332,
      "logps/chosen": -14.58353328704834,
      "logps/rejected": -26.606840133666992,
      "loss": 0.0827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3149658143520355,
      "rewards/margins": 3.5247819423675537,
      "rewards/rejected": -3.2098162174224854,
      "step": 576
    },
    {
      "epoch": 9.779661016949152,
      "grad_norm": 7.124665800971439,
      "learning_rate": 3.0284565590740607e-07,
      "logits/chosen": 0.004668239504098892,
      "logits/rejected": 1.3000279664993286,
      "logps/chosen": -15.21019172668457,
      "logps/rejected": -32.77819061279297,
      "loss": 0.0944,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3152053952217102,
      "rewards/margins": 4.309925079345703,
      "rewards/rejected": -3.9947195053100586,
      "step": 577
    },
    {
      "epoch": 9.796610169491526,
      "grad_norm": 7.258911609377368,
      "learning_rate": 3.021225906854383e-07,
      "logits/chosen": 0.9472925066947937,
      "logits/rejected": 0.8469686508178711,
      "logps/chosen": -16.569828033447266,
      "logps/rejected": -26.66617202758789,
      "loss": 0.0922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1706191599369049,
      "rewards/margins": 3.989084243774414,
      "rewards/rejected": -3.818464756011963,
      "step": 578
    },
    {
      "epoch": 9.813559322033898,
      "grad_norm": 6.194678399120452,
      "learning_rate": 3.013990693463344e-07,
      "logits/chosen": 2.516047954559326,
      "logits/rejected": 3.760178565979004,
      "logps/chosen": -19.39842987060547,
      "logps/rejected": -27.496429443359375,
      "loss": 0.0709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6658935546875,
      "rewards/margins": 3.9277522563934326,
      "rewards/rejected": -3.2618587017059326,
      "step": 579
    },
    {
      "epoch": 9.830508474576272,
      "grad_norm": 6.308757033182929,
      "learning_rate": 3.006750982215234e-07,
      "logits/chosen": -0.16934671998023987,
      "logits/rejected": 0.16713052988052368,
      "logps/chosen": -20.933340072631836,
      "logps/rejected": -27.842639923095703,
      "loss": 0.0902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.43451377749443054,
      "rewards/margins": 3.2033681869506836,
      "rewards/rejected": -2.7688541412353516,
      "step": 580
    },
    {
      "epoch": 9.847457627118644,
      "grad_norm": 7.185771510108942,
      "learning_rate": 2.9995068364637023e-07,
      "logits/chosen": 0.970811665058136,
      "logits/rejected": 1.682969331741333,
      "logps/chosen": -11.327841758728027,
      "logps/rejected": -30.293479919433594,
      "loss": 0.0889,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22116993367671967,
      "rewards/margins": 4.630741596221924,
      "rewards/rejected": -4.409571647644043,
      "step": 581
    },
    {
      "epoch": 9.864406779661017,
      "grad_norm": 5.920235267116203,
      "learning_rate": 2.9922583196012035e-07,
      "logits/chosen": 2.227673053741455,
      "logits/rejected": 3.2266247272491455,
      "logps/chosen": -14.300827980041504,
      "logps/rejected": -26.003128051757812,
      "loss": 0.0834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.379828542470932,
      "rewards/margins": 3.9268646240234375,
      "rewards/rejected": -3.5470361709594727,
      "step": 582
    },
    {
      "epoch": 9.88135593220339,
      "grad_norm": 8.120498709040012,
      "learning_rate": 2.985005495058446e-07,
      "logits/chosen": 2.3699660301208496,
      "logits/rejected": 3.4204599857330322,
      "logps/chosen": -13.776253700256348,
      "logps/rejected": -27.62920570373535,
      "loss": 0.0801,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11627595126628876,
      "rewards/margins": 4.1572794914245605,
      "rewards/rejected": -4.041003227233887,
      "step": 583
    },
    {
      "epoch": 9.898305084745763,
      "grad_norm": 6.7905769224670385,
      "learning_rate": 2.9777484263038303e-07,
      "logits/chosen": -0.20370978116989136,
      "logits/rejected": -0.03232604265213013,
      "logps/chosen": -19.46759605407715,
      "logps/rejected": -32.68392562866211,
      "loss": 0.0816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04020601511001587,
      "rewards/margins": 4.452649116516113,
      "rewards/rejected": -4.492855072021484,
      "step": 584
    },
    {
      "epoch": 9.915254237288135,
      "grad_norm": 8.057292474544546,
      "learning_rate": 2.9704871768429016e-07,
      "logits/chosen": 0.16171008348464966,
      "logits/rejected": 0.7341707348823547,
      "logps/chosen": -19.58940315246582,
      "logps/rejected": -27.96889877319336,
      "loss": 0.099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6214279532432556,
      "rewards/margins": 3.7912871837615967,
      "rewards/rejected": -3.1698591709136963,
      "step": 585
    },
    {
      "epoch": 9.932203389830509,
      "grad_norm": 7.937563375803782,
      "learning_rate": 2.9632218102177856e-07,
      "logits/chosen": 1.5007938146591187,
      "logits/rejected": 2.5846619606018066,
      "logps/chosen": -15.489872932434082,
      "logps/rejected": -25.30343246459961,
      "loss": 0.0993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.508277416229248,
      "rewards/margins": 3.338674306869507,
      "rewards/rejected": -2.830397129058838,
      "step": 586
    },
    {
      "epoch": 9.94915254237288,
      "grad_norm": 6.201025361723098,
      "learning_rate": 2.9559523900066393e-07,
      "logits/chosen": 3.0162744522094727,
      "logits/rejected": 4.763891696929932,
      "logps/chosen": -15.360055923461914,
      "logps/rejected": -23.94509506225586,
      "loss": 0.0815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10628265142440796,
      "rewards/margins": 3.4138388633728027,
      "rewards/rejected": -3.30755615234375,
      "step": 587
    },
    {
      "epoch": 9.966101694915254,
      "grad_norm": 6.718672190898394,
      "learning_rate": 2.948678979823092e-07,
      "logits/chosen": -1.6246399879455566,
      "logits/rejected": -1.125629186630249,
      "logps/chosen": -21.073156356811523,
      "logps/rejected": -27.51262092590332,
      "loss": 0.0851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7466506361961365,
      "rewards/margins": 4.355165004730225,
      "rewards/rejected": -3.6085143089294434,
      "step": 588
    },
    {
      "epoch": 9.983050847457626,
      "grad_norm": 6.881964686221444,
      "learning_rate": 2.941401643315686e-07,
      "logits/chosen": -0.25319910049438477,
      "logits/rejected": 0.6310909986495972,
      "logps/chosen": -13.684246063232422,
      "logps/rejected": -31.185253143310547,
      "loss": 0.0839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46263331174850464,
      "rewards/margins": 3.4962663650512695,
      "rewards/rejected": -3.033633232116699,
      "step": 589
    },
    {
      "epoch": 10.0,
      "grad_norm": 7.234238898566095,
      "learning_rate": 2.934120444167326e-07,
      "logits/chosen": -0.9664945602416992,
      "logits/rejected": 0.9399136304855347,
      "logps/chosen": -13.277244567871094,
      "logps/rejected": -25.155290603637695,
      "loss": 0.0974,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3529563844203949,
      "rewards/margins": 3.6515965461730957,
      "rewards/rejected": -3.298640251159668,
      "step": 590
    },
    {
      "epoch": 10.016949152542374,
      "grad_norm": 5.605599826609898,
      "learning_rate": 2.926835446094716e-07,
      "logits/chosen": -0.22446855902671814,
      "logits/rejected": 0.5719990730285645,
      "logps/chosen": -18.55261993408203,
      "logps/rejected": -34.263275146484375,
      "loss": 0.065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.035877808928489685,
      "rewards/margins": 4.546453952789307,
      "rewards/rejected": -4.582332134246826,
      "step": 591
    },
    {
      "epoch": 10.033898305084746,
      "grad_norm": 7.0637140819187545,
      "learning_rate": 2.919546712847804e-07,
      "logits/chosen": 1.9319177865982056,
      "logits/rejected": 4.413154125213623,
      "logps/chosen": -16.405902862548828,
      "logps/rejected": -38.4708251953125,
      "loss": 0.0868,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28276926279067993,
      "rewards/margins": 5.0292229652404785,
      "rewards/rejected": -4.746453762054443,
      "step": 592
    },
    {
      "epoch": 10.05084745762712,
      "grad_norm": 5.921559005856268,
      "learning_rate": 2.9122543082092246e-07,
      "logits/chosen": 0.27007830142974854,
      "logits/rejected": 0.5482916831970215,
      "logps/chosen": -19.874710083007812,
      "logps/rejected": -34.47208786010742,
      "loss": 0.0747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15760372579097748,
      "rewards/margins": 4.97886323928833,
      "rewards/rejected": -4.82125997543335,
      "step": 593
    },
    {
      "epoch": 10.067796610169491,
      "grad_norm": 5.603806953490683,
      "learning_rate": 2.9049582959937393e-07,
      "logits/chosen": -1.8163609504699707,
      "logits/rejected": -0.8079499006271362,
      "logps/chosen": -19.71383285522461,
      "logps/rejected": -27.676366806030273,
      "loss": 0.0728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5124025344848633,
      "rewards/margins": 3.4622397422790527,
      "rewards/rejected": -2.9498372077941895,
      "step": 594
    },
    {
      "epoch": 10.084745762711865,
      "grad_norm": 8.323328684692287,
      "learning_rate": 2.89765874004768e-07,
      "logits/chosen": 0.9996928572654724,
      "logits/rejected": 2.200232982635498,
      "logps/chosen": -14.065080642700195,
      "logps/rejected": -31.71418571472168,
      "loss": 0.0682,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40716665983200073,
      "rewards/margins": 4.4655961990356445,
      "rewards/rejected": -4.05842924118042,
      "step": 595
    },
    {
      "epoch": 10.101694915254237,
      "grad_norm": 6.075575616422893,
      "learning_rate": 2.890355704248388e-07,
      "logits/chosen": -0.06882792711257935,
      "logits/rejected": 0.32778534293174744,
      "logps/chosen": -14.473672866821289,
      "logps/rejected": -27.489944458007812,
      "loss": 0.0843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15015511214733124,
      "rewards/margins": 4.016153335571289,
      "rewards/rejected": -3.8659985065460205,
      "step": 596
    },
    {
      "epoch": 10.11864406779661,
      "grad_norm": 6.4149085949318385,
      "learning_rate": 2.8830492525036587e-07,
      "logits/chosen": -1.3330459594726562,
      "logits/rejected": -0.7347086668014526,
      "logps/chosen": -16.720176696777344,
      "logps/rejected": -30.86295509338379,
      "loss": 0.0719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8011523485183716,
      "rewards/margins": 4.588528633117676,
      "rewards/rejected": -3.7873764038085938,
      "step": 597
    },
    {
      "epoch": 10.135593220338983,
      "grad_norm": 6.036481300660005,
      "learning_rate": 2.875739448751176e-07,
      "logits/chosen": -0.32826659083366394,
      "logits/rejected": 0.4339306354522705,
      "logps/chosen": -14.774092674255371,
      "logps/rejected": -31.687725067138672,
      "loss": 0.0806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1728964000940323,
      "rewards/margins": 4.463289260864258,
      "rewards/rejected": -4.290392875671387,
      "step": 598
    },
    {
      "epoch": 10.152542372881356,
      "grad_norm": 5.535179077856142,
      "learning_rate": 2.8684263569579603e-07,
      "logits/chosen": -0.16325974464416504,
      "logits/rejected": 1.0072623491287231,
      "logps/chosen": -15.923517227172852,
      "logps/rejected": -30.345041275024414,
      "loss": 0.0735,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2372168004512787,
      "rewards/margins": 4.257624626159668,
      "rewards/rejected": -4.020407676696777,
      "step": 599
    },
    {
      "epoch": 10.169491525423728,
      "grad_norm": 5.163419851208816,
      "learning_rate": 2.8611100411198035e-07,
      "logits/chosen": 2.3917036056518555,
      "logits/rejected": 3.164809226989746,
      "logps/chosen": -13.81911849975586,
      "logps/rejected": -27.369752883911133,
      "loss": 0.0596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1035224199295044,
      "rewards/margins": 3.278686761856079,
      "rewards/rejected": -3.1751644611358643,
      "step": 600
    },
    {
      "epoch": 10.186440677966102,
      "grad_norm": 5.282168937091676,
      "learning_rate": 2.853790565260712e-07,
      "logits/chosen": 2.342893123626709,
      "logits/rejected": 3.2603580951690674,
      "logps/chosen": -10.237663269042969,
      "logps/rejected": -28.54045867919922,
      "loss": 0.0614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11436425149440765,
      "rewards/margins": 4.062482833862305,
      "rewards/rejected": -4.176846981048584,
      "step": 601
    },
    {
      "epoch": 10.203389830508474,
      "grad_norm": 6.559991217868432,
      "learning_rate": 2.846467993432342e-07,
      "logits/chosen": -0.0991126000881195,
      "logits/rejected": 1.038847804069519,
      "logps/chosen": -18.47357940673828,
      "logps/rejected": -32.717811584472656,
      "loss": 0.0744,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10679806023836136,
      "rewards/margins": 4.322455406188965,
      "rewards/rejected": -4.429253101348877,
      "step": 602
    },
    {
      "epoch": 10.220338983050848,
      "grad_norm": 6.029869722239407,
      "learning_rate": 2.8391423897134454e-07,
      "logits/chosen": 2.3927204608917236,
      "logits/rejected": 3.382364273071289,
      "logps/chosen": -22.96356964111328,
      "logps/rejected": -40.342674255371094,
      "loss": 0.0656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9394710063934326,
      "rewards/margins": 5.052562713623047,
      "rewards/rejected": -5.9920334815979,
      "step": 603
    },
    {
      "epoch": 10.23728813559322,
      "grad_norm": 6.053046265634852,
      "learning_rate": 2.8318138182093047e-07,
      "logits/chosen": 3.377546787261963,
      "logits/rejected": 3.5565762519836426,
      "logps/chosen": -14.09969711303711,
      "logps/rejected": -37.41474914550781,
      "loss": 0.0856,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06845837831497192,
      "rewards/margins": 5.450356483459473,
      "rewards/rejected": -5.518815040588379,
      "step": 604
    },
    {
      "epoch": 10.254237288135593,
      "grad_norm": 6.757171886215197,
      "learning_rate": 2.8244823430511725e-07,
      "logits/chosen": -0.2594801187515259,
      "logits/rejected": 0.3401494026184082,
      "logps/chosen": -20.410661697387695,
      "logps/rejected": -33.572357177734375,
      "loss": 0.0739,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09414413571357727,
      "rewards/margins": 4.429196357727051,
      "rewards/rejected": -4.335052013397217,
      "step": 605
    },
    {
      "epoch": 10.271186440677965,
      "grad_norm": 7.487518504831589,
      "learning_rate": 2.8171480283957117e-07,
      "logits/chosen": -1.103548288345337,
      "logits/rejected": -0.3884028196334839,
      "logps/chosen": -14.363391876220703,
      "logps/rejected": -26.286643981933594,
      "loss": 0.0859,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20181015133857727,
      "rewards/margins": 3.964519500732422,
      "rewards/rejected": -3.762709140777588,
      "step": 606
    },
    {
      "epoch": 10.288135593220339,
      "grad_norm": 9.61749318701916,
      "learning_rate": 2.8098109384244315e-07,
      "logits/chosen": -0.7662684917449951,
      "logits/rejected": 0.3284900188446045,
      "logps/chosen": -18.53818702697754,
      "logps/rejected": -30.907567977905273,
      "loss": 0.0675,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3579058051109314,
      "rewards/margins": 4.812725067138672,
      "rewards/rejected": -4.4548187255859375,
      "step": 607
    },
    {
      "epoch": 10.305084745762711,
      "grad_norm": 13.352854572328381,
      "learning_rate": 2.8024711373431297e-07,
      "logits/chosen": 3.6991214752197266,
      "logits/rejected": 5.248588562011719,
      "logps/chosen": -19.04616928100586,
      "logps/rejected": -34.87682342529297,
      "loss": 0.0856,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.015332631766796112,
      "rewards/margins": 4.818325519561768,
      "rewards/rejected": -4.833658695220947,
      "step": 608
    },
    {
      "epoch": 10.322033898305085,
      "grad_norm": 6.606910858411598,
      "learning_rate": 2.795128689381327e-07,
      "logits/chosen": -0.6707379817962646,
      "logits/rejected": 0.10683369636535645,
      "logps/chosen": -14.721006393432617,
      "logps/rejected": -32.36350631713867,
      "loss": 0.0832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08566325902938843,
      "rewards/margins": 4.577155113220215,
      "rewards/rejected": -4.491491794586182,
      "step": 609
    },
    {
      "epoch": 10.338983050847457,
      "grad_norm": 6.849176075211304,
      "learning_rate": 2.787783658791707e-07,
      "logits/chosen": 0.8173718452453613,
      "logits/rejected": 1.94963800907135,
      "logps/chosen": -17.403362274169922,
      "logps/rejected": -34.591819763183594,
      "loss": 0.0773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31257036328315735,
      "rewards/margins": 5.061819553375244,
      "rewards/rejected": -4.749249458312988,
      "step": 610
    },
    {
      "epoch": 10.35593220338983,
      "grad_norm": 5.6378028599700905,
      "learning_rate": 2.7804361098495547e-07,
      "logits/chosen": 1.038071632385254,
      "logits/rejected": 2.492664337158203,
      "logps/chosen": -21.735687255859375,
      "logps/rejected": -39.11198425292969,
      "loss": 0.0712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03703010082244873,
      "rewards/margins": 5.7275238037109375,
      "rewards/rejected": -5.764554023742676,
      "step": 611
    },
    {
      "epoch": 10.372881355932204,
      "grad_norm": 6.252348135448011,
      "learning_rate": 2.7730861068521913e-07,
      "logits/chosen": -0.26412123441696167,
      "logits/rejected": -0.7102423906326294,
      "logps/chosen": -15.097103118896484,
      "logps/rejected": -25.90133285522461,
      "loss": 0.0717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3093462884426117,
      "rewards/margins": 3.6670660972595215,
      "rewards/rejected": -3.357720136642456,
      "step": 612
    },
    {
      "epoch": 10.389830508474576,
      "grad_norm": 7.728236696916781,
      "learning_rate": 2.7657337141184134e-07,
      "logits/chosen": -1.9688349962234497,
      "logits/rejected": -0.6171210408210754,
      "logps/chosen": -16.57401466369629,
      "logps/rejected": -30.085830688476562,
      "loss": 0.0964,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.272685170173645,
      "rewards/margins": 4.039053440093994,
      "rewards/rejected": -3.7663679122924805,
      "step": 613
    },
    {
      "epoch": 10.40677966101695,
      "grad_norm": 5.485716187816843,
      "learning_rate": 2.75837899598793e-07,
      "logits/chosen": -3.0717618465423584,
      "logits/rejected": -2.5173027515411377,
      "logps/chosen": -11.400280952453613,
      "logps/rejected": -26.211803436279297,
      "loss": 0.0593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6044121384620667,
      "rewards/margins": 4.291702747344971,
      "rewards/rejected": -3.6872904300689697,
      "step": 614
    },
    {
      "epoch": 10.423728813559322,
      "grad_norm": 6.25330768538192,
      "learning_rate": 2.7510220168207996e-07,
      "logits/chosen": -0.6666077375411987,
      "logits/rejected": 0.4080359935760498,
      "logps/chosen": -15.857671737670898,
      "logps/rejected": -31.8955078125,
      "loss": 0.0748,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42522817850112915,
      "rewards/margins": 4.800365447998047,
      "rewards/rejected": -4.3751373291015625,
      "step": 615
    },
    {
      "epoch": 10.440677966101696,
      "grad_norm": 6.5295459646116125,
      "learning_rate": 2.743662840996866e-07,
      "logits/chosen": -0.040004000067710876,
      "logits/rejected": 1.337239146232605,
      "logps/chosen": -25.981477737426758,
      "logps/rejected": -37.080894470214844,
      "loss": 0.0922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7214669585227966,
      "rewards/margins": 3.7918219566345215,
      "rewards/rejected": -3.070355176925659,
      "step": 616
    },
    {
      "epoch": 10.457627118644067,
      "grad_norm": 6.723707358215044,
      "learning_rate": 2.736301532915196e-07,
      "logits/chosen": 2.539464235305786,
      "logits/rejected": 3.4116129875183105,
      "logps/chosen": -13.20813274383545,
      "logps/rejected": -23.4318904876709,
      "loss": 0.0881,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3384879231452942,
      "rewards/margins": 3.296609401702881,
      "rewards/rejected": -2.9581210613250732,
      "step": 617
    },
    {
      "epoch": 10.474576271186441,
      "grad_norm": 5.972258436867128,
      "learning_rate": 2.7289381569935167e-07,
      "logits/chosen": 1.7145967483520508,
      "logits/rejected": 2.2452683448791504,
      "logps/chosen": -20.402782440185547,
      "logps/rejected": -31.13981819152832,
      "loss": 0.0698,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.17722374200820923,
      "rewards/margins": 4.375895977020264,
      "rewards/rejected": -4.553119659423828,
      "step": 618
    },
    {
      "epoch": 10.491525423728813,
      "grad_norm": 5.552103251591103,
      "learning_rate": 2.7215727776676476e-07,
      "logits/chosen": -0.3053174316883087,
      "logits/rejected": -0.7472686767578125,
      "logps/chosen": -13.040075302124023,
      "logps/rejected": -28.802827835083008,
      "loss": 0.0647,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5394836068153381,
      "rewards/margins": 4.001735687255859,
      "rewards/rejected": -3.462251901626587,
      "step": 619
    },
    {
      "epoch": 10.508474576271187,
      "grad_norm": 6.23435993213956,
      "learning_rate": 2.714205459390942e-07,
      "logits/chosen": -2.3834853172302246,
      "logits/rejected": -0.9893413186073303,
      "logps/chosen": -20.2705020904541,
      "logps/rejected": -35.93256759643555,
      "loss": 0.0675,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2076442539691925,
      "rewards/margins": 4.942375659942627,
      "rewards/rejected": -4.734731197357178,
      "step": 620
    },
    {
      "epoch": 10.525423728813559,
      "grad_norm": 5.497320238058098,
      "learning_rate": 2.7068362666337213e-07,
      "logits/chosen": 0.960399866104126,
      "logits/rejected": 1.0540908575057983,
      "logps/chosen": -16.861644744873047,
      "logps/rejected": -31.652591705322266,
      "loss": 0.0587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.34831947088241577,
      "rewards/margins": 3.5641627311706543,
      "rewards/rejected": -3.912482261657715,
      "step": 621
    },
    {
      "epoch": 10.542372881355933,
      "grad_norm": 6.039868051539346,
      "learning_rate": 2.6994652638827075e-07,
      "logits/chosen": -1.5167878866195679,
      "logits/rejected": -0.33636558055877686,
      "logps/chosen": -15.524983406066895,
      "logps/rejected": -28.981571197509766,
      "loss": 0.0657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14473609626293182,
      "rewards/margins": 4.660205841064453,
      "rewards/rejected": -4.804942607879639,
      "step": 622
    },
    {
      "epoch": 10.559322033898304,
      "grad_norm": 7.569134750815346,
      "learning_rate": 2.6920925156404644e-07,
      "logits/chosen": -0.6254299283027649,
      "logits/rejected": 0.8782142996788025,
      "logps/chosen": -22.75749969482422,
      "logps/rejected": -31.623991012573242,
      "loss": 0.0881,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.046930328011512756,
      "rewards/margins": 3.9689981937408447,
      "rewards/rejected": -3.922067642211914,
      "step": 623
    },
    {
      "epoch": 10.576271186440678,
      "grad_norm": 6.555450045599343,
      "learning_rate": 2.684718086424828e-07,
      "logits/chosen": -1.0054882764816284,
      "logits/rejected": -1.0527081489562988,
      "logps/chosen": -13.448890686035156,
      "logps/rejected": -29.792495727539062,
      "loss": 0.0703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32683414220809937,
      "rewards/margins": 4.527410507202148,
      "rewards/rejected": -4.200576305389404,
      "step": 624
    },
    {
      "epoch": 10.59322033898305,
      "grad_norm": 6.4497992529556,
      "learning_rate": 2.677342040768346e-07,
      "logits/chosen": -0.724130392074585,
      "logits/rejected": -0.5873971581459045,
      "logps/chosen": -13.653425216674805,
      "logps/rejected": -21.41414451599121,
      "loss": 0.0984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3240708112716675,
      "rewards/margins": 2.993685007095337,
      "rewards/rejected": -2.669614315032959,
      "step": 625
    },
    {
      "epoch": 10.610169491525424,
      "grad_norm": 6.034047003671632,
      "learning_rate": 2.669964443217711e-07,
      "logits/chosen": 0.8687635064125061,
      "logits/rejected": 1.7205008268356323,
      "logps/chosen": -13.059608459472656,
      "logps/rejected": -24.26249122619629,
      "loss": 0.0703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6247247457504272,
      "rewards/margins": 4.4865922927856445,
      "rewards/rejected": -3.8618674278259277,
      "step": 626
    },
    {
      "epoch": 10.627118644067796,
      "grad_norm": 5.783511488843652,
      "learning_rate": 2.662585358333194e-07,
      "logits/chosen": 1.799331784248352,
      "logits/rejected": 2.220020294189453,
      "logps/chosen": -14.510954856872559,
      "logps/rejected": -28.626510620117188,
      "loss": 0.0632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4560384154319763,
      "rewards/margins": 4.656362533569336,
      "rewards/rejected": -4.200323581695557,
      "step": 627
    },
    {
      "epoch": 10.64406779661017,
      "grad_norm": 6.878208069873474,
      "learning_rate": 2.655204850688085e-07,
      "logits/chosen": -4.561980724334717,
      "logits/rejected": -4.086709022521973,
      "logps/chosen": -21.963542938232422,
      "logps/rejected": -29.26177978515625,
      "loss": 0.0985,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.047145962715148926,
      "rewards/margins": 3.6454405784606934,
      "rewards/rejected": -3.598294734954834,
      "step": 628
    },
    {
      "epoch": 10.661016949152543,
      "grad_norm": 6.441910708272441,
      "learning_rate": 2.6478229848681217e-07,
      "logits/chosen": -0.45938020944595337,
      "logits/rejected": 0.5661107897758484,
      "logps/chosen": -24.593807220458984,
      "logps/rejected": -43.87635040283203,
      "loss": 0.0777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4468405842781067,
      "rewards/margins": 4.661881446838379,
      "rewards/rejected": -4.215041160583496,
      "step": 629
    },
    {
      "epoch": 10.677966101694915,
      "grad_norm": 6.21379201456098,
      "learning_rate": 2.6404398254709283e-07,
      "logits/chosen": 0.44006413221359253,
      "logits/rejected": 2.213808298110962,
      "logps/chosen": -18.386600494384766,
      "logps/rejected": -24.819305419921875,
      "loss": 0.073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34933701157569885,
      "rewards/margins": 3.0405497550964355,
      "rewards/rejected": -2.6912126541137695,
      "step": 630
    },
    {
      "epoch": 10.694915254237289,
      "grad_norm": 6.025721748593164,
      "learning_rate": 2.633055437105446e-07,
      "logits/chosen": -2.20528507232666,
      "logits/rejected": -0.7411336898803711,
      "logps/chosen": -14.320496559143066,
      "logps/rejected": -29.599105834960938,
      "loss": 0.0787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7818676233291626,
      "rewards/margins": 4.8507161140441895,
      "rewards/rejected": -4.068848609924316,
      "step": 631
    },
    {
      "epoch": 10.711864406779661,
      "grad_norm": 5.45264581101388,
      "learning_rate": 2.6256698843913765e-07,
      "logits/chosen": -0.10968533158302307,
      "logits/rejected": 0.07270720601081848,
      "logps/chosen": -14.70391845703125,
      "logps/rejected": -33.96483612060547,
      "loss": 0.0603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03261980414390564,
      "rewards/margins": 5.4097900390625,
      "rewards/rejected": -5.377170562744141,
      "step": 632
    },
    {
      "epoch": 10.728813559322035,
      "grad_norm": 6.881559988949641,
      "learning_rate": 2.6182832319586045e-07,
      "logits/chosen": -1.6360795497894287,
      "logits/rejected": -0.09993427991867065,
      "logps/chosen": -25.044832229614258,
      "logps/rejected": -31.365337371826172,
      "loss": 0.0888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6498722434043884,
      "rewards/margins": 3.7184948921203613,
      "rewards/rejected": -3.068622589111328,
      "step": 633
    },
    {
      "epoch": 10.745762711864407,
      "grad_norm": 4.9186439647542315,
      "learning_rate": 2.6108955444466407e-07,
      "logits/chosen": -2.3466663360595703,
      "logits/rejected": -1.737764835357666,
      "logps/chosen": -14.39980697631836,
      "logps/rejected": -28.963144302368164,
      "loss": 0.0583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.045732468366622925,
      "rewards/margins": 4.375434875488281,
      "rewards/rejected": -4.329702377319336,
      "step": 634
    },
    {
      "epoch": 10.76271186440678,
      "grad_norm": 5.7723666132358575,
      "learning_rate": 2.6035068865040556e-07,
      "logits/chosen": 1.6776143312454224,
      "logits/rejected": 2.0671205520629883,
      "logps/chosen": -16.550514221191406,
      "logps/rejected": -36.80196762084961,
      "loss": 0.0597,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3467442989349365,
      "rewards/margins": 4.6361002922058105,
      "rewards/rejected": -4.289356231689453,
      "step": 635
    },
    {
      "epoch": 10.779661016949152,
      "grad_norm": 5.557597931065527,
      "learning_rate": 2.596117322787907e-07,
      "logits/chosen": -1.9914857149124146,
      "logits/rejected": -1.6738017797470093,
      "logps/chosen": -12.358434677124023,
      "logps/rejected": -27.78692626953125,
      "loss": 0.073,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19304338097572327,
      "rewards/margins": 3.466974973678589,
      "rewards/rejected": -3.2739317417144775,
      "step": 636
    },
    {
      "epoch": 10.796610169491526,
      "grad_norm": 6.364962907087287,
      "learning_rate": 2.588726917963183e-07,
      "logits/chosen": -0.4724288284778595,
      "logits/rejected": 0.24811546504497528,
      "logps/chosen": -20.698482513427734,
      "logps/rejected": -30.643646240234375,
      "loss": 0.0722,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10548533499240875,
      "rewards/margins": 3.7175071239471436,
      "rewards/rejected": -3.8229920864105225,
      "step": 637
    },
    {
      "epoch": 10.813559322033898,
      "grad_norm": 5.984849335536948,
      "learning_rate": 2.58133573670223e-07,
      "logits/chosen": -1.071981430053711,
      "logits/rejected": 0.18668809533119202,
      "logps/chosen": -19.217470169067383,
      "logps/rejected": -40.57974624633789,
      "loss": 0.0714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3740503191947937,
      "rewards/margins": 5.143554210662842,
      "rewards/rejected": -4.769504070281982,
      "step": 638
    },
    {
      "epoch": 10.830508474576272,
      "grad_norm": 5.392339021607484,
      "learning_rate": 2.5739438436841923e-07,
      "logits/chosen": -0.7297754287719727,
      "logits/rejected": -0.20466329157352448,
      "logps/chosen": -12.028447151184082,
      "logps/rejected": -24.535282135009766,
      "loss": 0.0688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4793805778026581,
      "rewards/margins": 4.4265217781066895,
      "rewards/rejected": -3.947141170501709,
      "step": 639
    },
    {
      "epoch": 10.847457627118644,
      "grad_norm": 5.987376643789218,
      "learning_rate": 2.566551303594437e-07,
      "logits/chosen": -2.031426429748535,
      "logits/rejected": -1.2302422523498535,
      "logps/chosen": -15.288844108581543,
      "logps/rejected": -25.842559814453125,
      "loss": 0.0801,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4474770426750183,
      "rewards/margins": 4.266576290130615,
      "rewards/rejected": -3.819099187850952,
      "step": 640
    },
    {
      "epoch": 10.864406779661017,
      "grad_norm": 6.364295591288036,
      "learning_rate": 2.559158181123998e-07,
      "logits/chosen": -2.1851398944854736,
      "logits/rejected": -0.4198753237724304,
      "logps/chosen": -17.15755844116211,
      "logps/rejected": -37.938419342041016,
      "loss": 0.0788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.057618916034698486,
      "rewards/margins": 5.671139717102051,
      "rewards/rejected": -5.613520622253418,
      "step": 641
    },
    {
      "epoch": 10.88135593220339,
      "grad_norm": 6.4165584079093385,
      "learning_rate": 2.5517645409690045e-07,
      "logits/chosen": -1.9795904159545898,
      "logits/rejected": 0.3546423316001892,
      "logps/chosen": -14.565407752990723,
      "logps/rejected": -32.81105422973633,
      "loss": 0.0759,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44077977538108826,
      "rewards/margins": 5.220073223114014,
      "rewards/rejected": -4.779293060302734,
      "step": 642
    },
    {
      "epoch": 10.898305084745763,
      "grad_norm": 5.958138967478258,
      "learning_rate": 2.544370447830115e-07,
      "logits/chosen": -1.0830657482147217,
      "logits/rejected": -0.3092998266220093,
      "logps/chosen": -11.562661170959473,
      "logps/rejected": -32.89630889892578,
      "loss": 0.0846,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.37156158685684204,
      "rewards/margins": 4.80405855178833,
      "rewards/rejected": -4.432497024536133,
      "step": 643
    },
    {
      "epoch": 10.915254237288135,
      "grad_norm": 6.0750333626885755,
      "learning_rate": 2.5369759664119533e-07,
      "logits/chosen": -1.6788640022277832,
      "logits/rejected": -1.4880775213241577,
      "logps/chosen": -10.794754028320312,
      "logps/rejected": -29.423439025878906,
      "loss": 0.0879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6279995441436768,
      "rewards/margins": 4.083406925201416,
      "rewards/rejected": -3.4554076194763184,
      "step": 644
    },
    {
      "epoch": 10.932203389830509,
      "grad_norm": 6.463093646854645,
      "learning_rate": 2.52958116142254e-07,
      "logits/chosen": 0.09284260869026184,
      "logits/rejected": 1.2297515869140625,
      "logps/chosen": -19.610464096069336,
      "logps/rejected": -35.90717315673828,
      "loss": 0.0848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3308804929256439,
      "rewards/margins": 5.242147445678711,
      "rewards/rejected": -4.911267280578613,
      "step": 645
    },
    {
      "epoch": 10.94915254237288,
      "grad_norm": 6.204542792920929,
      "learning_rate": 2.522186097572727e-07,
      "logits/chosen": -0.35911503434181213,
      "logits/rejected": -0.4865139126777649,
      "logps/chosen": -14.578887939453125,
      "logps/rejected": -30.42376708984375,
      "loss": 0.0755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18817180395126343,
      "rewards/margins": 3.996063232421875,
      "rewards/rejected": -3.807891368865967,
      "step": 646
    },
    {
      "epoch": 10.966101694915254,
      "grad_norm": 5.916857518131793,
      "learning_rate": 2.514790839575634e-07,
      "logits/chosen": -0.07091692090034485,
      "logits/rejected": 1.1772609949111938,
      "logps/chosen": -16.612977981567383,
      "logps/rejected": -38.160552978515625,
      "loss": 0.0659,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07712310552597046,
      "rewards/margins": 5.184111595153809,
      "rewards/rejected": -5.106988430023193,
      "step": 647
    },
    {
      "epoch": 10.983050847457626,
      "grad_norm": 6.407191345132673,
      "learning_rate": 2.507395452146074e-07,
      "logits/chosen": -0.2633797228336334,
      "logits/rejected": 0.21091461181640625,
      "logps/chosen": -17.18181037902832,
      "logps/rejected": -28.922496795654297,
      "loss": 0.074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.47414809465408325,
      "rewards/margins": 4.16361665725708,
      "rewards/rejected": -3.6894688606262207,
      "step": 648
    },
    {
      "epoch": 11.0,
      "grad_norm": 7.332363565320445,
      "learning_rate": 2.5e-07,
      "logits/chosen": 1.4961445331573486,
      "logits/rejected": 2.3838436603546143,
      "logps/chosen": -17.417078018188477,
      "logps/rejected": -31.51581573486328,
      "loss": 0.0915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.28374284505844116,
      "rewards/margins": 5.035284996032715,
      "rewards/rejected": -5.319027900695801,
      "step": 649
    },
    {
      "epoch": 11.016949152542374,
      "grad_norm": 6.710449322941788,
      "learning_rate": 2.4926045478539256e-07,
      "logits/chosen": -0.41290348768234253,
      "logits/rejected": -0.33926910161972046,
      "logps/chosen": -13.81264591217041,
      "logps/rejected": -32.02968215942383,
      "loss": 0.0819,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.03655707836151123,
      "rewards/margins": 4.345344066619873,
      "rewards/rejected": -4.381901741027832,
      "step": 650
    },
    {
      "epoch": 11.033898305084746,
      "grad_norm": 5.564446129176738,
      "learning_rate": 2.485209160424366e-07,
      "logits/chosen": -0.4851105213165283,
      "logits/rejected": 0.16047148406505585,
      "logps/chosen": -16.25713348388672,
      "logps/rejected": -25.432125091552734,
      "loss": 0.0668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.536626935005188,
      "rewards/margins": 3.623508930206299,
      "rewards/rejected": -3.0868821144104004,
      "step": 651
    },
    {
      "epoch": 11.05084745762712,
      "grad_norm": 5.5774736986307,
      "learning_rate": 2.477813902427272e-07,
      "logits/chosen": -0.30681484937667847,
      "logits/rejected": 0.5228248238563538,
      "logps/chosen": -19.037277221679688,
      "logps/rejected": -33.582115173339844,
      "loss": 0.0605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4117254316806793,
      "rewards/margins": 5.038257598876953,
      "rewards/rejected": -4.626532077789307,
      "step": 652
    },
    {
      "epoch": 11.067796610169491,
      "grad_norm": 5.723595464164498,
      "learning_rate": 2.47041883857746e-07,
      "logits/chosen": -2.9797439575195312,
      "logits/rejected": -0.5674165487289429,
      "logps/chosen": -18.158016204833984,
      "logps/rejected": -38.53350830078125,
      "loss": 0.0688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48181551694869995,
      "rewards/margins": 6.316951274871826,
      "rewards/rejected": -5.835135459899902,
      "step": 653
    },
    {
      "epoch": 11.084745762711865,
      "grad_norm": 5.650928750845028,
      "learning_rate": 2.463024033588046e-07,
      "logits/chosen": -2.9346420764923096,
      "logits/rejected": -1.6567249298095703,
      "logps/chosen": -18.69190216064453,
      "logps/rejected": -30.705974578857422,
      "loss": 0.0602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2089875191450119,
      "rewards/margins": 4.7780303955078125,
      "rewards/rejected": -4.569043159484863,
      "step": 654
    },
    {
      "epoch": 11.101694915254237,
      "grad_norm": 5.834575281966656,
      "learning_rate": 2.455629552169885e-07,
      "logits/chosen": -1.604995846748352,
      "logits/rejected": -1.15968656539917,
      "logps/chosen": -14.991609573364258,
      "logps/rejected": -36.816070556640625,
      "loss": 0.0704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32100000977516174,
      "rewards/margins": 4.978287220001221,
      "rewards/rejected": -4.657287120819092,
      "step": 655
    },
    {
      "epoch": 11.11864406779661,
      "grad_norm": 4.389904402387597,
      "learning_rate": 2.448235459030996e-07,
      "logits/chosen": -1.4382152557373047,
      "logits/rejected": -0.1963300108909607,
      "logps/chosen": -14.805438995361328,
      "logps/rejected": -25.915740966796875,
      "loss": 0.0466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7601976990699768,
      "rewards/margins": 3.805788516998291,
      "rewards/rejected": -3.045591115951538,
      "step": 656
    },
    {
      "epoch": 11.135593220338983,
      "grad_norm": 4.8932214139129435,
      "learning_rate": 2.4408418188760024e-07,
      "logits/chosen": 0.6055293083190918,
      "logits/rejected": 1.1443455219268799,
      "logps/chosen": -14.102373123168945,
      "logps/rejected": -31.51885223388672,
      "loss": 0.0644,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22653046250343323,
      "rewards/margins": 4.710800647735596,
      "rewards/rejected": -4.484270095825195,
      "step": 657
    },
    {
      "epoch": 11.152542372881356,
      "grad_norm": 5.086376596339275,
      "learning_rate": 2.433448696405563e-07,
      "logits/chosen": 0.898298442363739,
      "logits/rejected": 1.4417123794555664,
      "logps/chosen": -14.588051795959473,
      "logps/rejected": -32.29475784301758,
      "loss": 0.0569,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19099584221839905,
      "rewards/margins": 3.8837523460388184,
      "rewards/rejected": -4.074748516082764,
      "step": 658
    },
    {
      "epoch": 11.169491525423728,
      "grad_norm": 5.917003120913046,
      "learning_rate": 2.426056156315808e-07,
      "logits/chosen": -0.6991736888885498,
      "logits/rejected": 0.2075153887271881,
      "logps/chosen": -15.327507972717285,
      "logps/rejected": -32.986915588378906,
      "loss": 0.0761,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6373805999755859,
      "rewards/margins": 4.569518089294434,
      "rewards/rejected": -3.9321374893188477,
      "step": 659
    },
    {
      "epoch": 11.186440677966102,
      "grad_norm": 5.884139034140242,
      "learning_rate": 2.4186642632977697e-07,
      "logits/chosen": -1.9707210063934326,
      "logits/rejected": -2.106113910675049,
      "logps/chosen": -18.28716278076172,
      "logps/rejected": -47.34130096435547,
      "loss": 0.0578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2313433587551117,
      "rewards/margins": 5.2318034172058105,
      "rewards/rejected": -5.000459671020508,
      "step": 660
    },
    {
      "epoch": 11.203389830508474,
      "grad_norm": 5.643822454346463,
      "learning_rate": 2.4112730820368174e-07,
      "logits/chosen": -1.8582695722579956,
      "logits/rejected": -0.8738647699356079,
      "logps/chosen": -13.801697731018066,
      "logps/rejected": -25.913692474365234,
      "loss": 0.0664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.005020655691623688,
      "rewards/margins": 3.6902852058410645,
      "rewards/rejected": -3.6953060626983643,
      "step": 661
    },
    {
      "epoch": 11.220338983050848,
      "grad_norm": 4.680962274881366,
      "learning_rate": 2.403882677212093e-07,
      "logits/chosen": 0.6877023577690125,
      "logits/rejected": 2.0853915214538574,
      "logps/chosen": -18.771940231323242,
      "logps/rejected": -28.819293975830078,
      "loss": 0.0526,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35152608156204224,
      "rewards/margins": 4.176992893218994,
      "rewards/rejected": -3.825467109680176,
      "step": 662
    },
    {
      "epoch": 11.23728813559322,
      "grad_norm": 5.544585862782336,
      "learning_rate": 2.3964931134959447e-07,
      "logits/chosen": -1.1638025045394897,
      "logits/rejected": -1.0309693813323975,
      "logps/chosen": -18.670225143432617,
      "logps/rejected": -28.90134048461914,
      "loss": 0.0696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8741496801376343,
      "rewards/margins": 4.214159965515137,
      "rewards/rejected": -3.340010404586792,
      "step": 663
    },
    {
      "epoch": 11.254237288135593,
      "grad_norm": 5.503541176838127,
      "learning_rate": 2.3891044555533586e-07,
      "logits/chosen": 0.034407466650009155,
      "logits/rejected": 0.9282214641571045,
      "logps/chosen": -15.212516784667969,
      "logps/rejected": -28.66778564453125,
      "loss": 0.058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3786475956439972,
      "rewards/margins": 4.232975959777832,
      "rewards/rejected": -3.8543283939361572,
      "step": 664
    },
    {
      "epoch": 11.271186440677965,
      "grad_norm": 5.416764804202228,
      "learning_rate": 2.381716768041395e-07,
      "logits/chosen": 1.908860683441162,
      "logits/rejected": 2.344414234161377,
      "logps/chosen": -21.801124572753906,
      "logps/rejected": -48.7527961730957,
      "loss": 0.0611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.009497061371803284,
      "rewards/margins": 5.7271881103515625,
      "rewards/rejected": -5.717690467834473,
      "step": 665
    },
    {
      "epoch": 11.288135593220339,
      "grad_norm": 6.367781532097555,
      "learning_rate": 2.374330115608624e-07,
      "logits/chosen": -0.11974642425775528,
      "logits/rejected": 0.6266567707061768,
      "logps/chosen": -19.88003921508789,
      "logps/rejected": -35.740413665771484,
      "loss": 0.078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.530373215675354,
      "rewards/margins": 5.431632041931152,
      "rewards/rejected": -5.962004661560059,
      "step": 666
    },
    {
      "epoch": 11.305084745762711,
      "grad_norm": 5.626477352589517,
      "learning_rate": 2.3669445628945538e-07,
      "logits/chosen": 3.3318705558776855,
      "logits/rejected": 4.2224955558776855,
      "logps/chosen": -19.341773986816406,
      "logps/rejected": -34.1142463684082,
      "loss": 0.0662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06154298782348633,
      "rewards/margins": 4.670680046081543,
      "rewards/rejected": -4.732223033905029,
      "step": 667
    },
    {
      "epoch": 11.322033898305085,
      "grad_norm": 5.994475408383821,
      "learning_rate": 2.3595601745290725e-07,
      "logits/chosen": -0.7659940719604492,
      "logits/rejected": -0.017023414373397827,
      "logps/chosen": -13.725777626037598,
      "logps/rejected": -26.52062225341797,
      "loss": 0.0733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18746277689933777,
      "rewards/margins": 3.57584810256958,
      "rewards/rejected": -3.38838529586792,
      "step": 668
    },
    {
      "epoch": 11.338983050847457,
      "grad_norm": 6.014148031758321,
      "learning_rate": 2.3521770151318784e-07,
      "logits/chosen": -1.5654340982437134,
      "logits/rejected": -0.7088303565979004,
      "logps/chosen": -19.64613914489746,
      "logps/rejected": -35.96198272705078,
      "loss": 0.0746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24194762110710144,
      "rewards/margins": 5.484867572784424,
      "rewards/rejected": -5.242919921875,
      "step": 669
    },
    {
      "epoch": 11.35593220338983,
      "grad_norm": 5.904026735747155,
      "learning_rate": 2.344795149311915e-07,
      "logits/chosen": -1.6562185287475586,
      "logits/rejected": -0.89968341588974,
      "logps/chosen": -16.9832820892334,
      "logps/rejected": -32.758567810058594,
      "loss": 0.0666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13500508666038513,
      "rewards/margins": 4.955135822296143,
      "rewards/rejected": -4.820130825042725,
      "step": 670
    },
    {
      "epoch": 11.372881355932204,
      "grad_norm": 4.493982386752136,
      "learning_rate": 2.3374146416668062e-07,
      "logits/chosen": -1.7589337825775146,
      "logits/rejected": -1.0952833890914917,
      "logps/chosen": -18.65007209777832,
      "logps/rejected": -34.211727142333984,
      "loss": 0.0534,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2679644823074341,
      "rewards/margins": 5.072956562042236,
      "rewards/rejected": -4.804991722106934,
      "step": 671
    },
    {
      "epoch": 11.389830508474576,
      "grad_norm": 5.9376340717952,
      "learning_rate": 2.3300355567822893e-07,
      "logits/chosen": -1.799917221069336,
      "logits/rejected": -1.431991457939148,
      "logps/chosen": -12.619460105895996,
      "logps/rejected": -33.20170974731445,
      "loss": 0.0581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10192511230707169,
      "rewards/margins": 5.032021522521973,
      "rewards/rejected": -4.930096626281738,
      "step": 672
    },
    {
      "epoch": 11.40677966101695,
      "grad_norm": 5.493300214575323,
      "learning_rate": 2.3226579592316537e-07,
      "logits/chosen": -4.258989334106445,
      "logits/rejected": -3.360172748565674,
      "logps/chosen": -16.969221115112305,
      "logps/rejected": -30.5546932220459,
      "loss": 0.0621,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7313569188117981,
      "rewards/margins": 4.744319915771484,
      "rewards/rejected": -4.01296329498291,
      "step": 673
    },
    {
      "epoch": 11.423728813559322,
      "grad_norm": 5.344025354735871,
      "learning_rate": 2.315281913575172e-07,
      "logits/chosen": -0.1827794909477234,
      "logits/rejected": 0.7984082698822021,
      "logps/chosen": -16.188215255737305,
      "logps/rejected": -26.802419662475586,
      "loss": 0.0669,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4958565831184387,
      "rewards/margins": 4.04383659362793,
      "rewards/rejected": -3.5479798316955566,
      "step": 674
    },
    {
      "epoch": 11.440677966101696,
      "grad_norm": 5.640670191172759,
      "learning_rate": 2.3079074843595354e-07,
      "logits/chosen": -0.5284520983695984,
      "logits/rejected": 1.0243207216262817,
      "logps/chosen": -15.523112297058105,
      "logps/rejected": -33.79278564453125,
      "loss": 0.0789,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.5843504071235657,
      "rewards/margins": 4.946844577789307,
      "rewards/rejected": -4.362493515014648,
      "step": 675
    },
    {
      "epoch": 11.457627118644067,
      "grad_norm": 6.132077188997056,
      "learning_rate": 2.300534736117292e-07,
      "logits/chosen": -4.605233192443848,
      "logits/rejected": -3.059077739715576,
      "logps/chosen": -18.289066314697266,
      "logps/rejected": -29.978668212890625,
      "loss": 0.0719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5631596446037292,
      "rewards/margins": 4.844709873199463,
      "rewards/rejected": -4.281550407409668,
      "step": 676
    },
    {
      "epoch": 11.474576271186441,
      "grad_norm": 6.728620411744036,
      "learning_rate": 2.2931637333662785e-07,
      "logits/chosen": -2.1704790592193604,
      "logits/rejected": -1.2523642778396606,
      "logps/chosen": -17.027385711669922,
      "logps/rejected": -23.327802658081055,
      "loss": 0.0772,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.022564664483070374,
      "rewards/margins": 3.2541422843933105,
      "rewards/rejected": -3.276707172393799,
      "step": 677
    },
    {
      "epoch": 11.491525423728813,
      "grad_norm": 4.822714725679061,
      "learning_rate": 2.2857945406090578e-07,
      "logits/chosen": 1.1990220546722412,
      "logits/rejected": 1.711303472518921,
      "logps/chosen": -10.6589994430542,
      "logps/rejected": -30.195152282714844,
      "loss": 0.056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8227249383926392,
      "rewards/margins": 4.05718994140625,
      "rewards/rejected": -3.2344648838043213,
      "step": 678
    },
    {
      "epoch": 11.508474576271187,
      "grad_norm": 5.502535282470463,
      "learning_rate": 2.2784272223323527e-07,
      "logits/chosen": 0.7849825620651245,
      "logits/rejected": 0.8696894645690918,
      "logps/chosen": -12.942652702331543,
      "logps/rejected": -22.10407066345215,
      "loss": 0.0696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7114682197570801,
      "rewards/margins": 3.7902207374572754,
      "rewards/rejected": -3.078752279281616,
      "step": 679
    },
    {
      "epoch": 11.525423728813559,
      "grad_norm": 5.489019094233486,
      "learning_rate": 2.271061843006484e-07,
      "logits/chosen": 1.2034146785736084,
      "logits/rejected": 2.1347923278808594,
      "logps/chosen": -15.247549057006836,
      "logps/rejected": -34.84923553466797,
      "loss": 0.0726,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38953667879104614,
      "rewards/margins": 4.773509979248047,
      "rewards/rejected": -4.383973598480225,
      "step": 680
    },
    {
      "epoch": 11.542372881355933,
      "grad_norm": 5.059363361327104,
      "learning_rate": 2.263698467084804e-07,
      "logits/chosen": -2.265956163406372,
      "logits/rejected": -2.1130669116973877,
      "logps/chosen": -13.221713066101074,
      "logps/rejected": -44.21703338623047,
      "loss": 0.0585,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46408432722091675,
      "rewards/margins": 6.4323248863220215,
      "rewards/rejected": -5.968240261077881,
      "step": 681
    },
    {
      "epoch": 11.559322033898304,
      "grad_norm": 5.682481282090831,
      "learning_rate": 2.2563371590031338e-07,
      "logits/chosen": -0.5086088180541992,
      "logits/rejected": -0.18234482407569885,
      "logps/chosen": -13.774551391601562,
      "logps/rejected": -29.623085021972656,
      "loss": 0.0672,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3060076832771301,
      "rewards/margins": 4.601235866546631,
      "rewards/rejected": -4.295228004455566,
      "step": 682
    },
    {
      "epoch": 11.576271186440678,
      "grad_norm": 5.448076076241578,
      "learning_rate": 2.2489779831792004e-07,
      "logits/chosen": -1.4444375038146973,
      "logits/rejected": -1.0325533151626587,
      "logps/chosen": -17.940210342407227,
      "logps/rejected": -25.362424850463867,
      "loss": 0.0598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.061567530035972595,
      "rewards/margins": 3.1408650875091553,
      "rewards/rejected": -3.0792975425720215,
      "step": 683
    },
    {
      "epoch": 11.59322033898305,
      "grad_norm": 5.012773976084183,
      "learning_rate": 2.2416210040120701e-07,
      "logits/chosen": -2.4592440128326416,
      "logits/rejected": -0.24068701267242432,
      "logps/chosen": -17.526630401611328,
      "logps/rejected": -28.232179641723633,
      "loss": 0.0681,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.41350769996643066,
      "rewards/margins": 4.1663594245910645,
      "rewards/rejected": -3.752851724624634,
      "step": 684
    },
    {
      "epoch": 11.610169491525424,
      "grad_norm": 5.374681219785216,
      "learning_rate": 2.2342662858815867e-07,
      "logits/chosen": -1.426383137702942,
      "logits/rejected": -0.5009871125221252,
      "logps/chosen": -19.61524200439453,
      "logps/rejected": -36.145263671875,
      "loss": 0.0634,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7944812774658203,
      "rewards/margins": 5.898135662078857,
      "rewards/rejected": -5.103654861450195,
      "step": 685
    },
    {
      "epoch": 11.627118644067796,
      "grad_norm": 6.447716303096716,
      "learning_rate": 2.2269138931478082e-07,
      "logits/chosen": -1.0536390542984009,
      "logits/rejected": -0.29386839270591736,
      "logps/chosen": -17.011661529541016,
      "logps/rejected": -34.54016876220703,
      "loss": 0.0801,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05360276997089386,
      "rewards/margins": 4.743268013000488,
      "rewards/rejected": -4.796871185302734,
      "step": 686
    },
    {
      "epoch": 11.64406779661017,
      "grad_norm": 5.838531737146053,
      "learning_rate": 2.2195638901504448e-07,
      "logits/chosen": -3.675412654876709,
      "logits/rejected": -1.9526886940002441,
      "logps/chosen": -13.154958724975586,
      "logps/rejected": -25.237438201904297,
      "loss": 0.0613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6144644618034363,
      "rewards/margins": 4.4610209465026855,
      "rewards/rejected": -3.8465561866760254,
      "step": 687
    },
    {
      "epoch": 11.661016949152543,
      "grad_norm": 5.5640251268706615,
      "learning_rate": 2.2122163412082927e-07,
      "logits/chosen": -1.8933367729187012,
      "logits/rejected": -1.5240275859832764,
      "logps/chosen": -17.554466247558594,
      "logps/rejected": -29.81175422668457,
      "loss": 0.0639,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.03798585385084152,
      "rewards/margins": 3.6812267303466797,
      "rewards/rejected": -3.719212532043457,
      "step": 688
    },
    {
      "epoch": 11.677966101694915,
      "grad_norm": 5.298577590978091,
      "learning_rate": 2.2048713106186737e-07,
      "logits/chosen": 3.980241060256958,
      "logits/rejected": 4.915340900421143,
      "logps/chosen": -10.66274642944336,
      "logps/rejected": -35.31324768066406,
      "loss": 0.0504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28576070070266724,
      "rewards/margins": 5.346850395202637,
      "rewards/rejected": -5.061089515686035,
      "step": 689
    },
    {
      "epoch": 11.694915254237289,
      "grad_norm": 5.33692092549021,
      "learning_rate": 2.197528862656871e-07,
      "logits/chosen": 0.6891968846321106,
      "logits/rejected": 0.9424477815628052,
      "logps/chosen": -15.107596397399902,
      "logps/rejected": -28.289329528808594,
      "loss": 0.0695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3898676633834839,
      "rewards/margins": 4.073996067047119,
      "rewards/rejected": -3.6841282844543457,
      "step": 690
    },
    {
      "epoch": 11.711864406779661,
      "grad_norm": 6.204488409189156,
      "learning_rate": 2.190189061575569e-07,
      "logits/chosen": -2.2730422019958496,
      "logits/rejected": -2.2472658157348633,
      "logps/chosen": -13.751178741455078,
      "logps/rejected": -32.95414352416992,
      "loss": 0.0688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2908335030078888,
      "rewards/margins": 4.785499095916748,
      "rewards/rejected": -5.0763325691223145,
      "step": 691
    },
    {
      "epoch": 11.728813559322035,
      "grad_norm": 5.3301717255826375,
      "learning_rate": 2.1828519716042886e-07,
      "logits/chosen": -0.497864305973053,
      "logits/rejected": 0.37698572874069214,
      "logps/chosen": -16.339096069335938,
      "logps/rejected": -34.11829376220703,
      "loss": 0.0625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3000969886779785,
      "rewards/margins": 4.58716344833374,
      "rewards/rejected": -4.287066459655762,
      "step": 692
    },
    {
      "epoch": 11.745762711864407,
      "grad_norm": 5.19057795345044,
      "learning_rate": 2.1755176569488273e-07,
      "logits/chosen": 1.1150970458984375,
      "logits/rejected": 2.441777467727661,
      "logps/chosen": -16.924339294433594,
      "logps/rejected": -30.032011032104492,
      "loss": 0.0596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12007679045200348,
      "rewards/margins": 5.060428142547607,
      "rewards/rejected": -5.180505275726318,
      "step": 693
    },
    {
      "epoch": 11.76271186440678,
      "grad_norm": 6.742926871062795,
      "learning_rate": 2.168186181790695e-07,
      "logits/chosen": 0.5850726366043091,
      "logits/rejected": 1.4656791687011719,
      "logps/chosen": -17.600032806396484,
      "logps/rejected": -37.253028869628906,
      "loss": 0.0758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2762156128883362,
      "rewards/margins": 5.659514904022217,
      "rewards/rejected": -5.383298873901367,
      "step": 694
    },
    {
      "epoch": 11.779661016949152,
      "grad_norm": 6.11340248272783,
      "learning_rate": 2.1608576102865547e-07,
      "logits/chosen": -1.120666742324829,
      "logits/rejected": 0.017741888761520386,
      "logps/chosen": -15.677812576293945,
      "logps/rejected": -25.590530395507812,
      "loss": 0.0806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.49915313720703125,
      "rewards/margins": 3.54221248626709,
      "rewards/rejected": -3.0430593490600586,
      "step": 695
    },
    {
      "epoch": 11.796610169491526,
      "grad_norm": 4.952631946369327,
      "learning_rate": 2.1535320065676578e-07,
      "logits/chosen": -0.8462392687797546,
      "logits/rejected": 0.24551531672477722,
      "logps/chosen": -14.667747497558594,
      "logps/rejected": -32.78245544433594,
      "loss": 0.0647,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34924012422561646,
      "rewards/margins": 5.127192974090576,
      "rewards/rejected": -4.777953147888184,
      "step": 696
    },
    {
      "epoch": 11.813559322033898,
      "grad_norm": 5.705449211384567,
      "learning_rate": 2.1462094347392884e-07,
      "logits/chosen": -0.5555151700973511,
      "logits/rejected": 0.14884862303733826,
      "logps/chosen": -17.458023071289062,
      "logps/rejected": -35.274070739746094,
      "loss": 0.0719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2345159649848938,
      "rewards/margins": 5.021158218383789,
      "rewards/rejected": -4.786642551422119,
      "step": 697
    },
    {
      "epoch": 11.830508474576272,
      "grad_norm": 5.606218673543748,
      "learning_rate": 2.1388899588801963e-07,
      "logits/chosen": -1.732840657234192,
      "logits/rejected": -1.3636012077331543,
      "logps/chosen": -16.198732376098633,
      "logps/rejected": -29.274105072021484,
      "loss": 0.0581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05347772687673569,
      "rewards/margins": 4.097093105316162,
      "rewards/rejected": -4.150570392608643,
      "step": 698
    },
    {
      "epoch": 11.847457627118644,
      "grad_norm": 5.689400746949895,
      "learning_rate": 2.131573643042039e-07,
      "logits/chosen": -1.4225223064422607,
      "logits/rejected": -1.7895958423614502,
      "logps/chosen": -15.910755157470703,
      "logps/rejected": -34.615966796875,
      "loss": 0.0621,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4202052652835846,
      "rewards/margins": 5.09837007522583,
      "rewards/rejected": -4.678165435791016,
      "step": 699
    },
    {
      "epoch": 11.864406779661017,
      "grad_norm": 5.323397390244622,
      "learning_rate": 2.1242605512488245e-07,
      "logits/chosen": -1.2466119527816772,
      "logits/rejected": -0.6074275970458984,
      "logps/chosen": -15.015626907348633,
      "logps/rejected": -28.64008903503418,
      "loss": 0.0685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0258762836456299,
      "rewards/margins": 4.653193950653076,
      "rewards/rejected": -3.627316951751709,
      "step": 700
    },
    {
      "epoch": 11.88135593220339,
      "grad_norm": 5.664653229298019,
      "learning_rate": 2.116950747496342e-07,
      "logits/chosen": -2.161691188812256,
      "logits/rejected": -1.698690414428711,
      "logps/chosen": -18.23191261291504,
      "logps/rejected": -33.978004455566406,
      "loss": 0.0784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4678615927696228,
      "rewards/margins": 4.916198253631592,
      "rewards/rejected": -4.448336601257324,
      "step": 701
    },
    {
      "epoch": 11.898305084745763,
      "grad_norm": 5.550000502617084,
      "learning_rate": 2.1096442957516116e-07,
      "logits/chosen": -0.8441931009292603,
      "logits/rejected": -0.6889966726303101,
      "logps/chosen": -15.197530746459961,
      "logps/rejected": -31.47784423828125,
      "loss": 0.0777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.033519446849823,
      "rewards/margins": 4.367273807525635,
      "rewards/rejected": -4.400793552398682,
      "step": 702
    },
    {
      "epoch": 11.915254237288135,
      "grad_norm": 6.49263254472757,
      "learning_rate": 2.10234125995232e-07,
      "logits/chosen": -0.9990702867507935,
      "logits/rejected": -0.8798142671585083,
      "logps/chosen": -10.551839828491211,
      "logps/rejected": -26.121063232421875,
      "loss": 0.0993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38751670718193054,
      "rewards/margins": 4.126040458679199,
      "rewards/rejected": -3.7385239601135254,
      "step": 703
    },
    {
      "epoch": 11.932203389830509,
      "grad_norm": 5.247819487380474,
      "learning_rate": 2.0950417040062607e-07,
      "logits/chosen": 2.57755446434021,
      "logits/rejected": 2.9328744411468506,
      "logps/chosen": -12.72929573059082,
      "logps/rejected": -27.83072280883789,
      "loss": 0.0543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.030468828976154327,
      "rewards/margins": 4.585507392883301,
      "rewards/rejected": -4.5550384521484375,
      "step": 704
    },
    {
      "epoch": 11.94915254237288,
      "grad_norm": 5.151602852165204,
      "learning_rate": 2.0877456917907757e-07,
      "logits/chosen": -0.7559548616409302,
      "logits/rejected": -1.2953628301620483,
      "logps/chosen": -20.540603637695312,
      "logps/rejected": -31.206186294555664,
      "loss": 0.0546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11103224754333496,
      "rewards/margins": 4.5761895179748535,
      "rewards/rejected": -4.465157985687256,
      "step": 705
    },
    {
      "epoch": 11.966101694915254,
      "grad_norm": 5.688687348755467,
      "learning_rate": 2.0804532871521957e-07,
      "logits/chosen": -2.665072441101074,
      "logits/rejected": -1.002528429031372,
      "logps/chosen": -13.152605056762695,
      "logps/rejected": -29.87421226501465,
      "loss": 0.0696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5528115630149841,
      "rewards/margins": 4.340553283691406,
      "rewards/rejected": -3.7877418994903564,
      "step": 706
    },
    {
      "epoch": 11.983050847457626,
      "grad_norm": 6.496033619273407,
      "learning_rate": 2.0731645539052842e-07,
      "logits/chosen": -2.125807523727417,
      "logits/rejected": -1.361229658126831,
      "logps/chosen": -11.900805473327637,
      "logps/rejected": -33.30543899536133,
      "loss": 0.0694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30786752700805664,
      "rewards/margins": 5.710712909698486,
      "rewards/rejected": -5.40284538269043,
      "step": 707
    },
    {
      "epoch": 12.0,
      "grad_norm": 6.060562935251418,
      "learning_rate": 2.065879555832674e-07,
      "logits/chosen": -4.1413726806640625,
      "logits/rejected": -3.282559394836426,
      "logps/chosen": -16.134963989257812,
      "logps/rejected": -33.68052673339844,
      "loss": 0.0642,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.011490941047668457,
      "rewards/margins": 5.368725776672363,
      "rewards/rejected": -5.357234954833984,
      "step": 708
    },
    {
      "epoch": 12.016949152542374,
      "grad_norm": 4.532552644301971,
      "learning_rate": 2.0585983566843142e-07,
      "logits/chosen": -5.349707126617432,
      "logits/rejected": -4.256799697875977,
      "logps/chosen": -15.861433982849121,
      "logps/rejected": -31.989398956298828,
      "loss": 0.0484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5362915992736816,
      "rewards/margins": 5.06649923324585,
      "rewards/rejected": -4.530208110809326,
      "step": 709
    },
    {
      "epoch": 12.033898305084746,
      "grad_norm": 5.532947699353363,
      "learning_rate": 2.0513210201769083e-07,
      "logits/chosen": -1.8150622844696045,
      "logits/rejected": -0.2966909408569336,
      "logps/chosen": -14.874473571777344,
      "logps/rejected": -24.714536666870117,
      "loss": 0.0701,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8530625700950623,
      "rewards/margins": 4.482250213623047,
      "rewards/rejected": -3.62918758392334,
      "step": 710
    },
    {
      "epoch": 12.05084745762712,
      "grad_norm": 5.5718381236496475,
      "learning_rate": 2.0440476099933602e-07,
      "logits/chosen": -4.311933994293213,
      "logits/rejected": -2.9934637546539307,
      "logps/chosen": -18.516633987426758,
      "logps/rejected": -29.81332015991211,
      "loss": 0.0628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17470814287662506,
      "rewards/margins": 4.888881683349609,
      "rewards/rejected": -4.714173316955566,
      "step": 711
    },
    {
      "epoch": 12.067796610169491,
      "grad_norm": 6.360617007701148,
      "learning_rate": 2.0367781897822144e-07,
      "logits/chosen": 0.7523773908615112,
      "logits/rejected": 1.1841909885406494,
      "logps/chosen": -17.700777053833008,
      "logps/rejected": -29.529111862182617,
      "loss": 0.0862,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.30104872584342957,
      "rewards/margins": 5.281501770019531,
      "rewards/rejected": -5.582550525665283,
      "step": 712
    },
    {
      "epoch": 12.084745762711865,
      "grad_norm": 4.70334660438877,
      "learning_rate": 2.0295128231570984e-07,
      "logits/chosen": 0.837024986743927,
      "logits/rejected": 0.9363638162612915,
      "logps/chosen": -13.225208282470703,
      "logps/rejected": -35.414329528808594,
      "loss": 0.0491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.149062380194664,
      "rewards/margins": 5.915937900543213,
      "rewards/rejected": -5.766875743865967,
      "step": 713
    },
    {
      "epoch": 12.101694915254237,
      "grad_norm": 5.4006474930748105,
      "learning_rate": 2.0222515736961692e-07,
      "logits/chosen": -1.7546391487121582,
      "logits/rejected": 0.6276485919952393,
      "logps/chosen": -19.25953483581543,
      "logps/rejected": -44.87244415283203,
      "loss": 0.0544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08649277687072754,
      "rewards/margins": 7.5871148109436035,
      "rewards/rejected": -7.67360782623291,
      "step": 714
    },
    {
      "epoch": 12.11864406779661,
      "grad_norm": 5.368433493971488,
      "learning_rate": 2.0149945049415546e-07,
      "logits/chosen": -1.8162565231323242,
      "logits/rejected": -0.9864399433135986,
      "logps/chosen": -12.635334014892578,
      "logps/rejected": -26.762569427490234,
      "loss": 0.0592,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3255395293235779,
      "rewards/margins": 4.295178413391113,
      "rewards/rejected": -3.9696390628814697,
      "step": 715
    },
    {
      "epoch": 12.135593220338983,
      "grad_norm": 5.192183933114531,
      "learning_rate": 2.0077416803987963e-07,
      "logits/chosen": -0.8176920413970947,
      "logits/rejected": -0.31932979822158813,
      "logps/chosen": -18.625732421875,
      "logps/rejected": -31.451997756958008,
      "loss": 0.0567,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17950454354286194,
      "rewards/margins": 5.262685298919678,
      "rewards/rejected": -5.083180904388428,
      "step": 716
    },
    {
      "epoch": 12.152542372881356,
      "grad_norm": 5.093631383500938,
      "learning_rate": 2.0004931635362982e-07,
      "logits/chosen": -1.2001303434371948,
      "logits/rejected": -0.3195866346359253,
      "logps/chosen": -13.616204261779785,
      "logps/rejected": -24.842506408691406,
      "loss": 0.0522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5055986642837524,
      "rewards/margins": 4.0246782302856445,
      "rewards/rejected": -3.5190796852111816,
      "step": 717
    },
    {
      "epoch": 12.169491525423728,
      "grad_norm": 5.49085446169452,
      "learning_rate": 1.993249017784766e-07,
      "logits/chosen": 0.5020394325256348,
      "logits/rejected": 0.33351314067840576,
      "logps/chosen": -14.211220741271973,
      "logps/rejected": -35.0599365234375,
      "loss": 0.0755,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.25228646397590637,
      "rewards/margins": 5.2668375968933105,
      "rewards/rejected": -5.014551639556885,
      "step": 718
    },
    {
      "epoch": 12.186440677966102,
      "grad_norm": 5.629520362419779,
      "learning_rate": 1.9860093065366557e-07,
      "logits/chosen": -2.5751099586486816,
      "logits/rejected": -2.715381145477295,
      "logps/chosen": -15.755634307861328,
      "logps/rejected": -25.378055572509766,
      "loss": 0.0731,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10519762337207794,
      "rewards/margins": 3.9933674335479736,
      "rewards/rejected": -3.888169765472412,
      "step": 719
    },
    {
      "epoch": 12.203389830508474,
      "grad_norm": 5.952068955136311,
      "learning_rate": 1.9787740931456164e-07,
      "logits/chosen": -0.4816681444644928,
      "logits/rejected": 1.571818232536316,
      "logps/chosen": -11.133060455322266,
      "logps/rejected": -36.0597038269043,
      "loss": 0.0708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23930492997169495,
      "rewards/margins": 5.7355451583862305,
      "rewards/rejected": -5.496240615844727,
      "step": 720
    },
    {
      "epoch": 12.220338983050848,
      "grad_norm": 4.290693041598025,
      "learning_rate": 1.971543440925939e-07,
      "logits/chosen": -1.3183561563491821,
      "logits/rejected": 0.07553315162658691,
      "logps/chosen": -13.733359336853027,
      "logps/rejected": -31.047632217407227,
      "loss": 0.0355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36901021003723145,
      "rewards/margins": 5.332430839538574,
      "rewards/rejected": -4.963420867919922,
      "step": 721
    },
    {
      "epoch": 12.23728813559322,
      "grad_norm": 5.184170424363948,
      "learning_rate": 1.9643174131519984e-07,
      "logits/chosen": 3.3170368671417236,
      "logits/rejected": 3.977391004562378,
      "logps/chosen": -10.85775375366211,
      "logps/rejected": -29.93194580078125,
      "loss": 0.0583,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.11747324466705322,
      "rewards/margins": 4.625977993011475,
      "rewards/rejected": -4.508504867553711,
      "step": 722
    },
    {
      "epoch": 12.254237288135593,
      "grad_norm": 4.889541509223092,
      "learning_rate": 1.9570960730577032e-07,
      "logits/chosen": -0.7656147480010986,
      "logits/rejected": -0.24224507808685303,
      "logps/chosen": -19.524702072143555,
      "logps/rejected": -30.796566009521484,
      "loss": 0.0574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3538164794445038,
      "rewards/margins": 5.012910842895508,
      "rewards/rejected": -4.659094333648682,
      "step": 723
    },
    {
      "epoch": 12.271186440677965,
      "grad_norm": 5.10453945431393,
      "learning_rate": 1.949879483835939e-07,
      "logits/chosen": -2.638871431350708,
      "logits/rejected": -1.0773653984069824,
      "logps/chosen": -12.800580978393555,
      "logps/rejected": -27.307966232299805,
      "loss": 0.0616,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5115848779678345,
      "rewards/margins": 4.255484104156494,
      "rewards/rejected": -3.74389910697937,
      "step": 724
    },
    {
      "epoch": 12.288135593220339,
      "grad_norm": 5.719698536850818,
      "learning_rate": 1.9426677086380183e-07,
      "logits/chosen": -0.1832807958126068,
      "logits/rejected": 0.017368942499160767,
      "logps/chosen": -14.976875305175781,
      "logps/rejected": -29.548471450805664,
      "loss": 0.0829,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.375164270401001,
      "rewards/margins": 4.5754265785217285,
      "rewards/rejected": -4.200262546539307,
      "step": 725
    },
    {
      "epoch": 12.305084745762711,
      "grad_norm": 5.774954444673895,
      "learning_rate": 1.9354608105731267e-07,
      "logits/chosen": -0.24937498569488525,
      "logits/rejected": 0.15181505680084229,
      "logps/chosen": -16.608240127563477,
      "logps/rejected": -40.357643127441406,
      "loss": 0.0611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4569090008735657,
      "rewards/margins": 6.628655433654785,
      "rewards/rejected": -7.085563659667969,
      "step": 726
    },
    {
      "epoch": 12.322033898305085,
      "grad_norm": 4.581448773761563,
      "learning_rate": 1.9282588527077713e-07,
      "logits/chosen": 0.02391517162322998,
      "logits/rejected": 0.6816602349281311,
      "logps/chosen": -16.32445526123047,
      "logps/rejected": -29.69344711303711,
      "loss": 0.0596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46570780873298645,
      "rewards/margins": 4.278436183929443,
      "rewards/rejected": -3.8127284049987793,
      "step": 727
    },
    {
      "epoch": 12.338983050847457,
      "grad_norm": 5.603070097043823,
      "learning_rate": 1.9210618980652273e-07,
      "logits/chosen": 1.931321620941162,
      "logits/rejected": 2.3644118309020996,
      "logps/chosen": -13.214820861816406,
      "logps/rejected": -30.852802276611328,
      "loss": 0.0666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0568082332611084,
      "rewards/margins": 5.3618268966674805,
      "rewards/rejected": -5.41863489151001,
      "step": 728
    },
    {
      "epoch": 12.35593220338983,
      "grad_norm": 5.080980560752212,
      "learning_rate": 1.9138700096249883e-07,
      "logits/chosen": -3.121365785598755,
      "logits/rejected": -1.288278341293335,
      "logps/chosen": -21.657861709594727,
      "logps/rejected": -37.93989181518555,
      "loss": 0.0466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0911291092634201,
      "rewards/margins": 5.4343109130859375,
      "rewards/rejected": -5.525440692901611,
      "step": 729
    },
    {
      "epoch": 12.372881355932204,
      "grad_norm": 5.2142904258182385,
      "learning_rate": 1.9066832503222128e-07,
      "logits/chosen": -1.7135505676269531,
      "logits/rejected": 0.22507372498512268,
      "logps/chosen": -20.781967163085938,
      "logps/rejected": -38.08007049560547,
      "loss": 0.0596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6541242003440857,
      "rewards/margins": 5.690436840057373,
      "rewards/rejected": -6.344560623168945,
      "step": 730
    },
    {
      "epoch": 12.389830508474576,
      "grad_norm": 4.191891312611567,
      "learning_rate": 1.899501683047177e-07,
      "logits/chosen": -2.5515406131744385,
      "logits/rejected": -2.6740291118621826,
      "logps/chosen": -17.506187438964844,
      "logps/rejected": -37.260231018066406,
      "loss": 0.0486,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08348126709461212,
      "rewards/margins": 5.0080647468566895,
      "rewards/rejected": -5.091546058654785,
      "step": 731
    },
    {
      "epoch": 12.40677966101695,
      "grad_norm": 5.779154114222945,
      "learning_rate": 1.892325370644721e-07,
      "logits/chosen": -0.9004234671592712,
      "logits/rejected": -0.19273042678833008,
      "logps/chosen": -18.21908187866211,
      "logps/rejected": -27.488815307617188,
      "loss": 0.0786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6875766515731812,
      "rewards/margins": 4.210147380828857,
      "rewards/rejected": -3.5225706100463867,
      "step": 732
    },
    {
      "epoch": 12.423728813559322,
      "grad_norm": 4.8198265842340575,
      "learning_rate": 1.8851543759137007e-07,
      "logits/chosen": -1.2583699226379395,
      "logits/rejected": -0.1383885145187378,
      "logps/chosen": -11.82453727722168,
      "logps/rejected": -26.698476791381836,
      "loss": 0.0538,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7940881252288818,
      "rewards/margins": 5.455242156982422,
      "rewards/rejected": -4.661154270172119,
      "step": 733
    },
    {
      "epoch": 12.440677966101696,
      "grad_norm": 4.338898328943316,
      "learning_rate": 1.8779887616064382e-07,
      "logits/chosen": -3.5943920612335205,
      "logits/rejected": -2.3568880558013916,
      "logps/chosen": -15.277399063110352,
      "logps/rejected": -25.739362716674805,
      "loss": 0.0485,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31375235319137573,
      "rewards/margins": 4.360467433929443,
      "rewards/rejected": -4.046714782714844,
      "step": 734
    },
    {
      "epoch": 12.457627118644067,
      "grad_norm": 5.221531148515628,
      "learning_rate": 1.8708285904281712e-07,
      "logits/chosen": 0.39441272616386414,
      "logits/rejected": 0.6481245756149292,
      "logps/chosen": -13.238401412963867,
      "logps/rejected": -25.95081329345703,
      "loss": 0.0672,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.049318715929985046,
      "rewards/margins": 4.6796040534973145,
      "rewards/rejected": -4.630285263061523,
      "step": 735
    },
    {
      "epoch": 12.474576271186441,
      "grad_norm": 4.662194142659164,
      "learning_rate": 1.8636739250365056e-07,
      "logits/chosen": -0.6668483018875122,
      "logits/rejected": 0.672187089920044,
      "logps/chosen": -17.061891555786133,
      "logps/rejected": -25.8005428314209,
      "loss": 0.0554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8260502219200134,
      "rewards/margins": 4.933106422424316,
      "rewards/rejected": -4.107056617736816,
      "step": 736
    },
    {
      "epoch": 12.491525423728813,
      "grad_norm": 5.547045768800254,
      "learning_rate": 1.8565248280408698e-07,
      "logits/chosen": -1.3748953342437744,
      "logits/rejected": -0.43877971172332764,
      "logps/chosen": -12.486265182495117,
      "logps/rejected": -35.297569274902344,
      "loss": 0.0732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32557570934295654,
      "rewards/margins": 6.038592338562012,
      "rewards/rejected": -5.713016510009766,
      "step": 737
    },
    {
      "epoch": 12.508474576271187,
      "grad_norm": 4.938011936686151,
      "learning_rate": 1.8493813620019595e-07,
      "logits/chosen": 0.07011851668357849,
      "logits/rejected": 0.6697176694869995,
      "logps/chosen": -18.494783401489258,
      "logps/rejected": -41.817710876464844,
      "loss": 0.0592,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09022299945354462,
      "rewards/margins": 5.310074329376221,
      "rewards/rejected": -5.400297164916992,
      "step": 738
    },
    {
      "epoch": 12.525423728813559,
      "grad_norm": 4.975592954809822,
      "learning_rate": 1.8422435894311973e-07,
      "logits/chosen": -4.636005878448486,
      "logits/rejected": -3.7690577507019043,
      "logps/chosen": -13.333321571350098,
      "logps/rejected": -29.106115341186523,
      "loss": 0.0578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.269630491733551,
      "rewards/margins": 4.382090091705322,
      "rewards/rejected": -4.112459182739258,
      "step": 739
    },
    {
      "epoch": 12.542372881355933,
      "grad_norm": 4.701494943737591,
      "learning_rate": 1.8351115727901829e-07,
      "logits/chosen": -0.8022490739822388,
      "logits/rejected": -0.29640018939971924,
      "logps/chosen": -16.64236831665039,
      "logps/rejected": -34.56496810913086,
      "loss": 0.0594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09874613583087921,
      "rewards/margins": 5.410571098327637,
      "rewards/rejected": -5.311825752258301,
      "step": 740
    },
    {
      "epoch": 12.559322033898304,
      "grad_norm": 4.983518760341374,
      "learning_rate": 1.8279853744901464e-07,
      "logits/chosen": -1.764894723892212,
      "logits/rejected": -1.817033052444458,
      "logps/chosen": -15.258569717407227,
      "logps/rejected": -26.04188346862793,
      "loss": 0.0635,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4003324806690216,
      "rewards/margins": 3.7917897701263428,
      "rewards/rejected": -3.3914575576782227,
      "step": 741
    },
    {
      "epoch": 12.576271186440678,
      "grad_norm": 4.476989646138773,
      "learning_rate": 1.8208650568914033e-07,
      "logits/chosen": -1.363398790359497,
      "logits/rejected": -0.9498892426490784,
      "logps/chosen": -19.603914260864258,
      "logps/rejected": -30.36231231689453,
      "loss": 0.0408,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.011793076992034912,
      "rewards/margins": 3.9837088584899902,
      "rewards/rejected": -3.995501756668091,
      "step": 742
    },
    {
      "epoch": 12.59322033898305,
      "grad_norm": 5.053665090290864,
      "learning_rate": 1.8137506823028065e-07,
      "logits/chosen": -1.609299898147583,
      "logits/rejected": -0.7718572616577148,
      "logps/chosen": -22.111347198486328,
      "logps/rejected": -28.185571670532227,
      "loss": 0.0561,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35352978110313416,
      "rewards/margins": 3.5998425483703613,
      "rewards/rejected": -3.2463128566741943,
      "step": 743
    },
    {
      "epoch": 12.610169491525424,
      "grad_norm": 5.304619777865993,
      "learning_rate": 1.8066423129812026e-07,
      "logits/chosen": 1.7559425830841064,
      "logits/rejected": 2.9329328536987305,
      "logps/chosen": -20.055051803588867,
      "logps/rejected": -36.75667190551758,
      "loss": 0.0571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09106519818305969,
      "rewards/margins": 4.863574028015137,
      "rewards/rejected": -4.954638957977295,
      "step": 744
    },
    {
      "epoch": 12.627118644067796,
      "grad_norm": 4.667357132937044,
      "learning_rate": 1.7995400111308883e-07,
      "logits/chosen": -0.961107611656189,
      "logits/rejected": -0.7769833207130432,
      "logps/chosen": -18.276569366455078,
      "logps/rejected": -30.646440505981445,
      "loss": 0.0544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07002657651901245,
      "rewards/margins": 4.622382640838623,
      "rewards/rejected": -4.552356243133545,
      "step": 745
    },
    {
      "epoch": 12.64406779661017,
      "grad_norm": 5.408680811716402,
      "learning_rate": 1.7924438389030648e-07,
      "logits/chosen": -1.4892151355743408,
      "logits/rejected": -0.24569085240364075,
      "logps/chosen": -20.448904037475586,
      "logps/rejected": -33.296146392822266,
      "loss": 0.0661,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.29597920179367065,
      "rewards/margins": 5.750075340270996,
      "rewards/rejected": -5.454095840454102,
      "step": 746
    },
    {
      "epoch": 12.661016949152543,
      "grad_norm": 5.247658476313327,
      "learning_rate": 1.785353858395292e-07,
      "logits/chosen": -1.1832478046417236,
      "logits/rejected": -0.4320974051952362,
      "logps/chosen": -16.966569900512695,
      "logps/rejected": -30.618276596069336,
      "loss": 0.0561,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.308743953704834,
      "rewards/margins": 4.0481038093566895,
      "rewards/rejected": -3.7393596172332764,
      "step": 747
    },
    {
      "epoch": 12.677966101694915,
      "grad_norm": 5.386940866382028,
      "learning_rate": 1.7782701316509478e-07,
      "logits/chosen": -0.8887053728103638,
      "logits/rejected": 0.26204565167427063,
      "logps/chosen": -15.196284294128418,
      "logps/rejected": -32.22180938720703,
      "loss": 0.0569,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.023950517177581787,
      "rewards/margins": 5.039477348327637,
      "rewards/rejected": -5.01552677154541,
      "step": 748
    },
    {
      "epoch": 12.694915254237289,
      "grad_norm": 6.448249897658643,
      "learning_rate": 1.7711927206586853e-07,
      "logits/chosen": -3.0813937187194824,
      "logits/rejected": -1.0136470794677734,
      "logps/chosen": -15.665678024291992,
      "logps/rejected": -31.437288284301758,
      "loss": 0.0766,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6306146383285522,
      "rewards/margins": 4.768771171569824,
      "rewards/rejected": -4.138156414031982,
      "step": 749
    },
    {
      "epoch": 12.711864406779661,
      "grad_norm": 5.1415627551289855,
      "learning_rate": 1.7641216873518876e-07,
      "logits/chosen": -0.4980939030647278,
      "logits/rejected": 0.9896915555000305,
      "logps/chosen": -13.606678009033203,
      "logps/rejected": -36.296627044677734,
      "loss": 0.0588,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3556250333786011,
      "rewards/margins": 5.726956844329834,
      "rewards/rejected": -5.371331691741943,
      "step": 750
    },
    {
      "epoch": 12.728813559322035,
      "grad_norm": 5.538397960309667,
      "learning_rate": 1.7570570936081306e-07,
      "logits/chosen": -5.0588178634643555,
      "logits/rejected": -4.427032947540283,
      "logps/chosen": -15.996036529541016,
      "logps/rejected": -29.211898803710938,
      "loss": 0.0711,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.036909669637680054,
      "rewards/margins": 4.613151550292969,
      "rewards/rejected": -4.576241970062256,
      "step": 751
    },
    {
      "epoch": 12.745762711864407,
      "grad_norm": 5.401159987947702,
      "learning_rate": 1.7499990012486348e-07,
      "logits/chosen": 0.5816026329994202,
      "logits/rejected": 2.0243024826049805,
      "logps/chosen": -14.950392723083496,
      "logps/rejected": -44.65636444091797,
      "loss": 0.0744,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08444496989250183,
      "rewards/margins": 5.976288795471191,
      "rewards/rejected": -6.060733318328857,
      "step": 752
    },
    {
      "epoch": 12.76271186440678,
      "grad_norm": 5.642148679884942,
      "learning_rate": 1.7429474720377312e-07,
      "logits/chosen": -4.293545722961426,
      "logits/rejected": -3.6384382247924805,
      "logps/chosen": -15.824508666992188,
      "logps/rejected": -21.923446655273438,
      "loss": 0.0616,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6682969331741333,
      "rewards/margins": 3.7970216274261475,
      "rewards/rejected": -3.1287245750427246,
      "step": 753
    },
    {
      "epoch": 12.779661016949152,
      "grad_norm": 5.456868524343284,
      "learning_rate": 1.735902567682315e-07,
      "logits/chosen": -0.1643550992012024,
      "logits/rejected": 1.1555442810058594,
      "logps/chosen": -18.49430274963379,
      "logps/rejected": -28.05358123779297,
      "loss": 0.0698,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23009660840034485,
      "rewards/margins": 4.420815467834473,
      "rewards/rejected": -4.190719127655029,
      "step": 754
    },
    {
      "epoch": 12.796610169491526,
      "grad_norm": 5.5899581786249675,
      "learning_rate": 1.7288643498313104e-07,
      "logits/chosen": -1.605297565460205,
      "logits/rejected": -1.2698827981948853,
      "logps/chosen": -17.66697120666504,
      "logps/rejected": -28.075071334838867,
      "loss": 0.0649,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.554067850112915,
      "rewards/margins": 4.1530561447143555,
      "rewards/rejected": -3.5989880561828613,
      "step": 755
    },
    {
      "epoch": 12.813559322033898,
      "grad_norm": 4.97167654515329,
      "learning_rate": 1.7218328800751285e-07,
      "logits/chosen": -1.9998224973678589,
      "logits/rejected": -1.9531162977218628,
      "logps/chosen": -16.823482513427734,
      "logps/rejected": -38.705322265625,
      "loss": 0.0501,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05464259535074234,
      "rewards/margins": 5.106647491455078,
      "rewards/rejected": -5.052005290985107,
      "step": 756
    },
    {
      "epoch": 12.830508474576272,
      "grad_norm": 4.636697957998646,
      "learning_rate": 1.7148082199451286e-07,
      "logits/chosen": 2.3911752700805664,
      "logits/rejected": 2.8658528327941895,
      "logps/chosen": -14.898900985717773,
      "logps/rejected": -32.14864730834961,
      "loss": 0.0509,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25076889991760254,
      "rewards/margins": 5.665815830230713,
      "rewards/rejected": -5.4150471687316895,
      "step": 757
    },
    {
      "epoch": 12.847457627118644,
      "grad_norm": 4.43156755971984,
      "learning_rate": 1.7077904309130782e-07,
      "logits/chosen": 0.07385343313217163,
      "logits/rejected": 1.4616187810897827,
      "logps/chosen": -16.277408599853516,
      "logps/rejected": -33.832191467285156,
      "loss": 0.0383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.02317967265844345,
      "rewards/margins": 4.975264549255371,
      "rewards/rejected": -4.952084541320801,
      "step": 758
    },
    {
      "epoch": 12.864406779661017,
      "grad_norm": 5.1087161747778955,
      "learning_rate": 1.7007795743906194e-07,
      "logits/chosen": -0.01966071128845215,
      "logits/rejected": 1.2537782192230225,
      "logps/chosen": -12.349102973937988,
      "logps/rejected": -29.57040023803711,
      "loss": 0.0624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.719579815864563,
      "rewards/margins": 4.931670665740967,
      "rewards/rejected": -4.212090969085693,
      "step": 759
    },
    {
      "epoch": 12.88135593220339,
      "grad_norm": 4.575423088701401,
      "learning_rate": 1.6937757117287276e-07,
      "logits/chosen": -2.572610378265381,
      "logits/rejected": -2.4339561462402344,
      "logps/chosen": -20.87567138671875,
      "logps/rejected": -26.362003326416016,
      "loss": 0.0543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36575549840927124,
      "rewards/margins": 3.4433891773223877,
      "rewards/rejected": -3.0776336193084717,
      "step": 760
    },
    {
      "epoch": 12.898305084745763,
      "grad_norm": 4.792758308336707,
      "learning_rate": 1.6867789042171777e-07,
      "logits/chosen": 0.42236053943634033,
      "logits/rejected": 1.349460244178772,
      "logps/chosen": -18.220428466796875,
      "logps/rejected": -40.099430084228516,
      "loss": 0.0565,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5892751216888428,
      "rewards/margins": 6.123309135437012,
      "rewards/rejected": -6.712584495544434,
      "step": 761
    },
    {
      "epoch": 12.915254237288135,
      "grad_norm": 5.182633483984975,
      "learning_rate": 1.6797892130840036e-07,
      "logits/chosen": -4.92634916305542,
      "logits/rejected": -3.316908359527588,
      "logps/chosen": -22.279062271118164,
      "logps/rejected": -40.13741683959961,
      "loss": 0.0524,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5362528562545776,
      "rewards/margins": 5.827565670013428,
      "rewards/rejected": -5.291313171386719,
      "step": 762
    },
    {
      "epoch": 12.932203389830509,
      "grad_norm": 6.2174069067649596,
      "learning_rate": 1.6728066994949658e-07,
      "logits/chosen": 0.3422883450984955,
      "logits/rejected": 1.34884512424469,
      "logps/chosen": -16.529205322265625,
      "logps/rejected": -31.202659606933594,
      "loss": 0.0652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2604439854621887,
      "rewards/margins": 5.107015609741211,
      "rewards/rejected": -4.846571445465088,
      "step": 763
    },
    {
      "epoch": 12.94915254237288,
      "grad_norm": 5.015581327963073,
      "learning_rate": 1.6658314245530148e-07,
      "logits/chosen": -1.485102653503418,
      "logits/rejected": 0.03505659103393555,
      "logps/chosen": -15.350225448608398,
      "logps/rejected": -32.654762268066406,
      "loss": 0.0628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5115726590156555,
      "rewards/margins": 5.575878143310547,
      "rewards/rejected": -5.064305305480957,
      "step": 764
    },
    {
      "epoch": 12.966101694915254,
      "grad_norm": 5.203411876922801,
      "learning_rate": 1.6588634492977582e-07,
      "logits/chosen": -2.822071075439453,
      "logits/rejected": -1.4864444732666016,
      "logps/chosen": -20.292339324951172,
      "logps/rejected": -41.56735610961914,
      "loss": 0.0754,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.02777126431465149,
      "rewards/margins": 5.915192604064941,
      "rewards/rejected": -5.887421607971191,
      "step": 765
    },
    {
      "epoch": 12.983050847457626,
      "grad_norm": 5.575372006782511,
      "learning_rate": 1.651902834704924e-07,
      "logits/chosen": -0.6836428642272949,
      "logits/rejected": -0.2021288275718689,
      "logps/chosen": -15.80721664428711,
      "logps/rejected": -22.794828414916992,
      "loss": 0.0614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13072407245635986,
      "rewards/margins": 3.470419406890869,
      "rewards/rejected": -3.3396952152252197,
      "step": 766
    },
    {
      "epoch": 13.0,
      "grad_norm": 4.979310125788461,
      "learning_rate": 1.6449496416858282e-07,
      "logits/chosen": -1.8837170600891113,
      "logits/rejected": -0.8573411703109741,
      "logps/chosen": -16.546485900878906,
      "logps/rejected": -28.7147216796875,
      "loss": 0.0512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3212031126022339,
      "rewards/margins": 4.140195846557617,
      "rewards/rejected": -3.8189926147460938,
      "step": 767
    },
    {
      "epoch": 13.016949152542374,
      "grad_norm": 4.661905901633895,
      "learning_rate": 1.6380039310868414e-07,
      "logits/chosen": -0.571354329586029,
      "logits/rejected": -0.13909736275672913,
      "logps/chosen": -17.805564880371094,
      "logps/rejected": -35.160640716552734,
      "loss": 0.0502,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1881602704524994,
      "rewards/margins": 4.904372692108154,
      "rewards/rejected": -5.092532634735107,
      "step": 768
    },
    {
      "epoch": 13.033898305084746,
      "grad_norm": 5.250431718898731,
      "learning_rate": 1.631065763688857e-07,
      "logits/chosen": -1.4799143075942993,
      "logits/rejected": -0.8538658618927002,
      "logps/chosen": -12.869924545288086,
      "logps/rejected": -30.551315307617188,
      "loss": 0.0671,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15870481729507446,
      "rewards/margins": 4.908497333526611,
      "rewards/rejected": -4.749792575836182,
      "step": 769
    },
    {
      "epoch": 13.05084745762712,
      "grad_norm": 4.746347720305799,
      "learning_rate": 1.6241352002067588e-07,
      "logits/chosen": -0.8848968148231506,
      "logits/rejected": -0.33918485045433044,
      "logps/chosen": -17.034452438354492,
      "logps/rejected": -36.654781341552734,
      "loss": 0.0474,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5132811069488525,
      "rewards/margins": 6.099752426147461,
      "rewards/rejected": -6.613033771514893,
      "step": 770
    },
    {
      "epoch": 13.067796610169491,
      "grad_norm": 4.650926687696971,
      "learning_rate": 1.61721230128889e-07,
      "logits/chosen": 1.7074960470199585,
      "logits/rejected": 2.339669704437256,
      "logps/chosen": -14.039243698120117,
      "logps/rejected": -34.935001373291016,
      "loss": 0.0497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.006116881966590881,
      "rewards/margins": 5.922494411468506,
      "rewards/rejected": -5.928612232208252,
      "step": 771
    },
    {
      "epoch": 13.084745762711865,
      "grad_norm": 4.992257153089093,
      "learning_rate": 1.6102971275165227e-07,
      "logits/chosen": -4.5710248947143555,
      "logits/rejected": -3.5121634006500244,
      "logps/chosen": -19.965408325195312,
      "logps/rejected": -39.98967361450195,
      "loss": 0.0652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06310770660638809,
      "rewards/margins": 6.08799934387207,
      "rewards/rejected": -6.1511077880859375,
      "step": 772
    },
    {
      "epoch": 13.101694915254237,
      "grad_norm": 5.116376837204931,
      "learning_rate": 1.603389739403327e-07,
      "logits/chosen": -3.5352561473846436,
      "logits/rejected": -2.2301392555236816,
      "logps/chosen": -16.83971405029297,
      "logps/rejected": -26.139163970947266,
      "loss": 0.062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8963336944580078,
      "rewards/margins": 4.319397926330566,
      "rewards/rejected": -3.4230642318725586,
      "step": 773
    },
    {
      "epoch": 13.11864406779661,
      "grad_norm": 4.984010048813125,
      "learning_rate": 1.5964901973948408e-07,
      "logits/chosen": 0.6005043983459473,
      "logits/rejected": 1.7892330884933472,
      "logps/chosen": -13.829337120056152,
      "logps/rejected": -29.835859298706055,
      "loss": 0.0658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5679302215576172,
      "rewards/margins": 4.191529750823975,
      "rewards/rejected": -3.6235995292663574,
      "step": 774
    },
    {
      "epoch": 13.135593220338983,
      "grad_norm": 4.060001356412656,
      "learning_rate": 1.5895985618679445e-07,
      "logits/chosen": -1.580199956893921,
      "logits/rejected": -0.7634269595146179,
      "logps/chosen": -18.07457733154297,
      "logps/rejected": -39.27557373046875,
      "loss": 0.0372,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12154760956764221,
      "rewards/margins": 5.554102897644043,
      "rewards/rejected": -5.432555675506592,
      "step": 775
    },
    {
      "epoch": 13.152542372881356,
      "grad_norm": 4.8504946205719595,
      "learning_rate": 1.5827148931303275e-07,
      "logits/chosen": -0.6837934255599976,
      "logits/rejected": 0.3661801517009735,
      "logps/chosen": -15.66816520690918,
      "logps/rejected": -41.75244140625,
      "loss": 0.0449,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.35962551832199097,
      "rewards/margins": 6.23133659362793,
      "rewards/rejected": -6.5909624099731445,
      "step": 776
    },
    {
      "epoch": 13.169491525423728,
      "grad_norm": 5.519362182135464,
      "learning_rate": 1.5758392514199643e-07,
      "logits/chosen": -3.332242488861084,
      "logits/rejected": -3.9557909965515137,
      "logps/chosen": -11.690735816955566,
      "logps/rejected": -27.75088119506836,
      "loss": 0.0594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2635051906108856,
      "rewards/margins": 4.264172554016113,
      "rewards/rejected": -4.527677536010742,
      "step": 777
    },
    {
      "epoch": 13.186440677966102,
      "grad_norm": 4.935439407056802,
      "learning_rate": 1.5689716969045847e-07,
      "logits/chosen": -0.9040203094482422,
      "logits/rejected": 0.7142425179481506,
      "logps/chosen": -20.44605255126953,
      "logps/rejected": -41.98016357421875,
      "loss": 0.0506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10384351015090942,
      "rewards/margins": 6.846504211425781,
      "rewards/rejected": -6.742660045623779,
      "step": 778
    },
    {
      "epoch": 13.203389830508474,
      "grad_norm": 4.6398107320600674,
      "learning_rate": 1.5621122896811522e-07,
      "logits/chosen": -0.5301164388656616,
      "logits/rejected": 0.8865979909896851,
      "logps/chosen": -14.487466812133789,
      "logps/rejected": -32.661075592041016,
      "loss": 0.0653,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13690048456192017,
      "rewards/margins": 5.08042049407959,
      "rewards/rejected": -4.943520545959473,
      "step": 779
    },
    {
      "epoch": 13.220338983050848,
      "grad_norm": 5.035524688198735,
      "learning_rate": 1.555261089775329e-07,
      "logits/chosen": -1.5996699333190918,
      "logits/rejected": -0.5739651918411255,
      "logps/chosen": -14.261418342590332,
      "logps/rejected": -33.98247146606445,
      "loss": 0.0593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08373254537582397,
      "rewards/margins": 5.217193126678467,
      "rewards/rejected": -5.3009257316589355,
      "step": 780
    },
    {
      "epoch": 13.23728813559322,
      "grad_norm": 4.270205266332028,
      "learning_rate": 1.548418157140961e-07,
      "logits/chosen": -2.086413860321045,
      "logits/rejected": -2.292921781539917,
      "logps/chosen": -22.09109878540039,
      "logps/rejected": -38.62481689453125,
      "loss": 0.0479,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9677011370658875,
      "rewards/margins": 4.687134265899658,
      "rewards/rejected": -5.6548357009887695,
      "step": 781
    },
    {
      "epoch": 13.254237288135593,
      "grad_norm": 4.219215065631512,
      "learning_rate": 1.5415835516595463e-07,
      "logits/chosen": 0.14554214477539062,
      "logits/rejected": 0.22008490562438965,
      "logps/chosen": -16.21091651916504,
      "logps/rejected": -27.570281982421875,
      "loss": 0.0568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2888966202735901,
      "rewards/margins": 4.920593738555908,
      "rewards/rejected": -4.631697177886963,
      "step": 782
    },
    {
      "epoch": 13.271186440677965,
      "grad_norm": 4.339597407909707,
      "learning_rate": 1.5347573331397135e-07,
      "logits/chosen": -1.9881035089492798,
      "logits/rejected": -0.008711844682693481,
      "logps/chosen": -21.09736442565918,
      "logps/rejected": -39.05769729614258,
      "loss": 0.0426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46814027428627014,
      "rewards/margins": 7.1204423904418945,
      "rewards/rejected": -6.6523027420043945,
      "step": 783
    },
    {
      "epoch": 13.288135593220339,
      "grad_norm": 5.367134909878089,
      "learning_rate": 1.5279395613166985e-07,
      "logits/chosen": -2.6423563957214355,
      "logits/rejected": -1.4679323434829712,
      "logps/chosen": -17.295196533203125,
      "logps/rejected": -26.447715759277344,
      "loss": 0.0601,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2713179886341095,
      "rewards/margins": 4.201862812042236,
      "rewards/rejected": -3.9305450916290283,
      "step": 784
    },
    {
      "epoch": 13.305084745762711,
      "grad_norm": 4.300679654482064,
      "learning_rate": 1.5211302958518214e-07,
      "logits/chosen": -0.8169465065002441,
      "logits/rejected": 0.09549763798713684,
      "logps/chosen": -18.21856689453125,
      "logps/rejected": -30.12501335144043,
      "loss": 0.0496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32225629687309265,
      "rewards/margins": 5.08899450302124,
      "rewards/rejected": -4.766737461090088,
      "step": 785
    },
    {
      "epoch": 13.322033898305085,
      "grad_norm": 4.29162168254634,
      "learning_rate": 1.5143295963319642e-07,
      "logits/chosen": -0.8983233571052551,
      "logits/rejected": 0.23173213005065918,
      "logps/chosen": -15.133694648742676,
      "logps/rejected": -30.526933670043945,
      "loss": 0.0489,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10836651176214218,
      "rewards/margins": 4.605082988739014,
      "rewards/rejected": -4.496715545654297,
      "step": 786
    },
    {
      "epoch": 13.338983050847457,
      "grad_norm": 4.876409740979101,
      "learning_rate": 1.5075375222690496e-07,
      "logits/chosen": -0.9811916351318359,
      "logits/rejected": 0.19567179679870605,
      "logps/chosen": -19.57292366027832,
      "logps/rejected": -36.624717712402344,
      "loss": 0.0502,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12690672278404236,
      "rewards/margins": 6.475020408630371,
      "rewards/rejected": -6.601926803588867,
      "step": 787
    },
    {
      "epoch": 13.35593220338983,
      "grad_norm": 4.607150296948369,
      "learning_rate": 1.5007541330995198e-07,
      "logits/chosen": -2.696889638900757,
      "logits/rejected": -1.390625,
      "logps/chosen": -20.865976333618164,
      "logps/rejected": -32.45897674560547,
      "loss": 0.065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.054636940360069275,
      "rewards/margins": 4.085897922515869,
      "rewards/rejected": -4.031260967254639,
      "step": 788
    },
    {
      "epoch": 13.372881355932204,
      "grad_norm": 6.11812735648075,
      "learning_rate": 1.4939794881838176e-07,
      "logits/chosen": -1.5392122268676758,
      "logits/rejected": 0.14951567351818085,
      "logps/chosen": -20.76491928100586,
      "logps/rejected": -30.57718276977539,
      "loss": 0.0761,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19175273180007935,
      "rewards/margins": 4.187775611877441,
      "rewards/rejected": -3.9960227012634277,
      "step": 789
    },
    {
      "epoch": 13.389830508474576,
      "grad_norm": 3.8459575352861775,
      "learning_rate": 1.487213646805866e-07,
      "logits/chosen": -2.8805551528930664,
      "logits/rejected": -2.2380049228668213,
      "logps/chosen": -16.49655532836914,
      "logps/rejected": -20.970523834228516,
      "loss": 0.0409,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7969187498092651,
      "rewards/margins": 3.531719446182251,
      "rewards/rejected": -2.7348005771636963,
      "step": 790
    },
    {
      "epoch": 13.40677966101695,
      "grad_norm": 5.154988116162254,
      "learning_rate": 1.4804566681725496e-07,
      "logits/chosen": 0.7958973050117493,
      "logits/rejected": 1.2581777572631836,
      "logps/chosen": -13.75499439239502,
      "logps/rejected": -27.726245880126953,
      "loss": 0.0647,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08662920445203781,
      "rewards/margins": 4.289105415344238,
      "rewards/rejected": -4.202475547790527,
      "step": 791
    },
    {
      "epoch": 13.423728813559322,
      "grad_norm": 4.401669016604261,
      "learning_rate": 1.473708611413194e-07,
      "logits/chosen": -0.515765368938446,
      "logits/rejected": 0.03481185436248779,
      "logps/chosen": -16.60962677001953,
      "logps/rejected": -32.53062438964844,
      "loss": 0.0433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2858050763607025,
      "rewards/margins": 5.730351448059082,
      "rewards/rejected": -6.01615571975708,
      "step": 792
    },
    {
      "epoch": 13.440677966101696,
      "grad_norm": 4.443887778355534,
      "learning_rate": 1.4669695355790552e-07,
      "logits/chosen": -2.8968968391418457,
      "logits/rejected": -2.2223594188690186,
      "logps/chosen": -18.33234405517578,
      "logps/rejected": -34.28547668457031,
      "loss": 0.045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19874697923660278,
      "rewards/margins": 4.827791213989258,
      "rewards/rejected": -4.629044532775879,
      "step": 793
    },
    {
      "epoch": 13.457627118644067,
      "grad_norm": 3.971025721995942,
      "learning_rate": 1.4602394996427942e-07,
      "logits/chosen": -0.9764919281005859,
      "logits/rejected": -0.13439202308654785,
      "logps/chosen": -15.439664840698242,
      "logps/rejected": -26.79970359802246,
      "loss": 0.0461,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06810703873634338,
      "rewards/margins": 3.983307361602783,
      "rewards/rejected": -3.9151999950408936,
      "step": 794
    },
    {
      "epoch": 13.474576271186441,
      "grad_norm": 4.907220694727352,
      "learning_rate": 1.4535185624979687e-07,
      "logits/chosen": -2.4083213806152344,
      "logits/rejected": -1.2281450033187866,
      "logps/chosen": -19.956815719604492,
      "logps/rejected": -38.46086883544922,
      "loss": 0.0515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3511507511138916,
      "rewards/margins": 5.372152328491211,
      "rewards/rejected": -5.723303318023682,
      "step": 795
    },
    {
      "epoch": 13.491525423728813,
      "grad_norm": 7.315327297358435,
      "learning_rate": 1.4468067829585108e-07,
      "logits/chosen": -2.2506372928619385,
      "logits/rejected": -2.7237861156463623,
      "logps/chosen": -17.557580947875977,
      "logps/rejected": -35.88199234008789,
      "loss": 0.0562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5984256267547607,
      "rewards/margins": 5.224565505981445,
      "rewards/rejected": -4.6261396408081055,
      "step": 796
    },
    {
      "epoch": 13.508474576271187,
      "grad_norm": 4.920641143929202,
      "learning_rate": 1.4401042197582192e-07,
      "logits/chosen": 0.2772759795188904,
      "logits/rejected": 0.9791284203529358,
      "logps/chosen": -14.289785385131836,
      "logps/rejected": -34.84285354614258,
      "loss": 0.0479,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13411599397659302,
      "rewards/margins": 5.060387134552002,
      "rewards/rejected": -4.926271438598633,
      "step": 797
    },
    {
      "epoch": 13.525423728813559,
      "grad_norm": 4.521477877513176,
      "learning_rate": 1.4334109315502392e-07,
      "logits/chosen": -0.9948372840881348,
      "logits/rejected": -0.7503252625465393,
      "logps/chosen": -16.5922908782959,
      "logps/rejected": -34.337181091308594,
      "loss": 0.0493,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3663084805011749,
      "rewards/margins": 4.992905616760254,
      "rewards/rejected": -4.6265974044799805,
      "step": 798
    },
    {
      "epoch": 13.542372881355933,
      "grad_norm": 5.520637708481951,
      "learning_rate": 1.4267269769065537e-07,
      "logits/chosen": -2.080489158630371,
      "logits/rejected": -1.0827324390411377,
      "logps/chosen": -17.34023094177246,
      "logps/rejected": -28.902450561523438,
      "loss": 0.0583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2214941382408142,
      "rewards/margins": 5.067776203155518,
      "rewards/rejected": -4.846282482147217,
      "step": 799
    },
    {
      "epoch": 13.559322033898304,
      "grad_norm": 3.7597183239261796,
      "learning_rate": 1.4200524143174676e-07,
      "logits/chosen": -2.283979892730713,
      "logits/rejected": -0.6506827473640442,
      "logps/chosen": -15.541634559631348,
      "logps/rejected": -32.611759185791016,
      "loss": 0.0412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1795252561569214,
      "rewards/margins": 5.449156761169434,
      "rewards/rejected": -5.628682613372803,
      "step": 800
    },
    {
      "epoch": 13.576271186440678,
      "grad_norm": 4.103020829191366,
      "learning_rate": 1.4133873021910976e-07,
      "logits/chosen": 0.5742231011390686,
      "logits/rejected": 1.5821070671081543,
      "logps/chosen": -13.288924217224121,
      "logps/rejected": -29.91840171813965,
      "loss": 0.0568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6092392802238464,
      "rewards/margins": 4.6803364753723145,
      "rewards/rejected": -4.0710978507995605,
      "step": 801
    },
    {
      "epoch": 13.59322033898305,
      "grad_norm": 5.313735220398172,
      "learning_rate": 1.4067316988528616e-07,
      "logits/chosen": -2.240776538848877,
      "logits/rejected": -1.4768493175506592,
      "logps/chosen": -21.16518783569336,
      "logps/rejected": -31.8751220703125,
      "loss": 0.0686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8960158824920654,
      "rewards/margins": 5.112182140350342,
      "rewards/rejected": -4.2161664962768555,
      "step": 802
    },
    {
      "epoch": 13.610169491525424,
      "grad_norm": 5.028648319330279,
      "learning_rate": 1.4000856625449664e-07,
      "logits/chosen": -3.578434467315674,
      "logits/rejected": -1.6968674659729004,
      "logps/chosen": -21.81180763244629,
      "logps/rejected": -36.5727653503418,
      "loss": 0.0659,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.005623072385787964,
      "rewards/margins": 5.015188217163086,
      "rewards/rejected": -5.020811080932617,
      "step": 803
    },
    {
      "epoch": 13.627118644067796,
      "grad_norm": 4.40829356214919,
      "learning_rate": 1.3934492514259003e-07,
      "logits/chosen": -1.2967630624771118,
      "logits/rejected": -0.4483084976673126,
      "logps/chosen": -14.010137557983398,
      "logps/rejected": -31.958181381225586,
      "loss": 0.0531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6740959882736206,
      "rewards/margins": 5.422782897949219,
      "rewards/rejected": -4.74868631362915,
      "step": 804
    },
    {
      "epoch": 13.64406779661017,
      "grad_norm": 5.745264644229589,
      "learning_rate": 1.3868225235699216e-07,
      "logits/chosen": -1.2424854040145874,
      "logits/rejected": -0.6555378437042236,
      "logps/chosen": -18.450178146362305,
      "logps/rejected": -35.39335632324219,
      "loss": 0.0487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5278493165969849,
      "rewards/margins": 5.411442279815674,
      "rewards/rejected": -5.939291477203369,
      "step": 805
    },
    {
      "epoch": 13.661016949152543,
      "grad_norm": 5.36790675366442,
      "learning_rate": 1.3802055369665533e-07,
      "logits/chosen": -0.2212720513343811,
      "logits/rejected": 1.6100802421569824,
      "logps/chosen": -14.109825134277344,
      "logps/rejected": -33.012298583984375,
      "loss": 0.0659,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.709932804107666,
      "rewards/margins": 5.652429103851318,
      "rewards/rejected": -4.9424967765808105,
      "step": 806
    },
    {
      "epoch": 13.677966101694915,
      "grad_norm": 14.088164156627155,
      "learning_rate": 1.373598349520073e-07,
      "logits/chosen": -2.5952467918395996,
      "logits/rejected": -0.777185320854187,
      "logps/chosen": -11.400592803955078,
      "logps/rejected": -29.009681701660156,
      "loss": 0.0455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41961854696273804,
      "rewards/margins": 5.154366970062256,
      "rewards/rejected": -4.734748840332031,
      "step": 807
    },
    {
      "epoch": 13.694915254237289,
      "grad_norm": 5.194429705443838,
      "learning_rate": 1.3670010190490073e-07,
      "logits/chosen": -2.7721073627471924,
      "logits/rejected": -0.28102076053619385,
      "logps/chosen": -19.503450393676758,
      "logps/rejected": -38.648590087890625,
      "loss": 0.0502,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23439528048038483,
      "rewards/margins": 5.5849690437316895,
      "rewards/rejected": -5.350574016571045,
      "step": 808
    },
    {
      "epoch": 13.711864406779661,
      "grad_norm": 3.4681710195473396,
      "learning_rate": 1.3604136032856268e-07,
      "logits/chosen": -4.101337909698486,
      "logits/rejected": -3.245271682739258,
      "logps/chosen": -18.42070198059082,
      "logps/rejected": -41.46506881713867,
      "loss": 0.0319,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3121660649776459,
      "rewards/margins": 7.131579399108887,
      "rewards/rejected": -6.819413185119629,
      "step": 809
    },
    {
      "epoch": 13.728813559322035,
      "grad_norm": 5.1618133217577,
      "learning_rate": 1.3538361598754382e-07,
      "logits/chosen": -2.2615439891815186,
      "logits/rejected": -0.9000074863433838,
      "logps/chosen": -18.307828903198242,
      "logps/rejected": -36.45680618286133,
      "loss": 0.0615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.048638343811035156,
      "rewards/margins": 6.239575386047363,
      "rewards/rejected": -6.190937042236328,
      "step": 810
    },
    {
      "epoch": 13.745762711864407,
      "grad_norm": 5.144615388580858,
      "learning_rate": 1.3472687463766848e-07,
      "logits/chosen": 0.011030316352844238,
      "logits/rejected": 0.2687840461730957,
      "logps/chosen": -15.965656280517578,
      "logps/rejected": -29.35083770751953,
      "loss": 0.0606,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2761033773422241,
      "rewards/margins": 4.888636589050293,
      "rewards/rejected": -4.6125335693359375,
      "step": 811
    },
    {
      "epoch": 13.76271186440678,
      "grad_norm": 5.434824546786577,
      "learning_rate": 1.3407114202598368e-07,
      "logits/chosen": -2.206752061843872,
      "logits/rejected": -1.648052453994751,
      "logps/chosen": -12.60938835144043,
      "logps/rejected": -23.64356231689453,
      "loss": 0.0754,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21786317229270935,
      "rewards/margins": 3.707275390625,
      "rewards/rejected": -3.9251387119293213,
      "step": 812
    },
    {
      "epoch": 13.779661016949152,
      "grad_norm": 4.815034905940384,
      "learning_rate": 1.3341642389070926e-07,
      "logits/chosen": -0.7637806534767151,
      "logits/rejected": -1.1668951511383057,
      "logps/chosen": -16.947185516357422,
      "logps/rejected": -35.942691802978516,
      "loss": 0.0591,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3600042462348938,
      "rewards/margins": 5.317866325378418,
      "rewards/rejected": -4.95786190032959,
      "step": 813
    },
    {
      "epoch": 13.796610169491526,
      "grad_norm": 4.948954844647574,
      "learning_rate": 1.3276272596118728e-07,
      "logits/chosen": -2.535177707672119,
      "logits/rejected": -2.6263809204101562,
      "logps/chosen": -22.20641326904297,
      "logps/rejected": -35.22035598754883,
      "loss": 0.0614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07845325767993927,
      "rewards/margins": 5.658707618713379,
      "rewards/rejected": -5.580254554748535,
      "step": 814
    },
    {
      "epoch": 13.813559322033898,
      "grad_norm": 5.081803609403105,
      "learning_rate": 1.3211005395783244e-07,
      "logits/chosen": -3.0509417057037354,
      "logits/rejected": -1.9049749374389648,
      "logps/chosen": -17.269140243530273,
      "logps/rejected": -35.48225402832031,
      "loss": 0.0652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17116232216358185,
      "rewards/margins": 5.267909049987793,
      "rewards/rejected": -5.096746921539307,
      "step": 815
    },
    {
      "epoch": 13.830508474576272,
      "grad_norm": 4.31483209294794,
      "learning_rate": 1.3145841359208148e-07,
      "logits/chosen": -0.06046187877655029,
      "logits/rejected": 1.4396018981933594,
      "logps/chosen": -17.44058609008789,
      "logps/rejected": -37.39258575439453,
      "loss": 0.0422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2737378180027008,
      "rewards/margins": 5.390666484832764,
      "rewards/rejected": -5.116928577423096,
      "step": 816
    },
    {
      "epoch": 13.847457627118644,
      "grad_norm": 4.568010812319063,
      "learning_rate": 1.308078105663437e-07,
      "logits/chosen": -2.7601492404937744,
      "logits/rejected": -2.4725546836853027,
      "logps/chosen": -19.867713928222656,
      "logps/rejected": -28.408180236816406,
      "loss": 0.0605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.049654990434646606,
      "rewards/margins": 4.494874477386475,
      "rewards/rejected": -4.544529914855957,
      "step": 817
    },
    {
      "epoch": 13.864406779661017,
      "grad_norm": 4.69192569572312,
      "learning_rate": 1.3015825057395058e-07,
      "logits/chosen": -0.5765193700790405,
      "logits/rejected": 0.6422019004821777,
      "logps/chosen": -13.671000480651855,
      "logps/rejected": -26.479190826416016,
      "loss": 0.0508,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.473724365234375,
      "rewards/margins": 4.31511116027832,
      "rewards/rejected": -3.8413872718811035,
      "step": 818
    },
    {
      "epoch": 13.88135593220339,
      "grad_norm": 5.415611690257974,
      "learning_rate": 1.2950973929910619e-07,
      "logits/chosen": -3.1382057666778564,
      "logits/rejected": -0.9989707469940186,
      "logps/chosen": -17.182632446289062,
      "logps/rejected": -29.84030532836914,
      "loss": 0.073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07108063995838165,
      "rewards/margins": 4.959874629974365,
      "rewards/rejected": -4.8887939453125,
      "step": 819
    },
    {
      "epoch": 13.898305084745763,
      "grad_norm": 4.253278835211674,
      "learning_rate": 1.2886228241683748e-07,
      "logits/chosen": -2.7567286491394043,
      "logits/rejected": -1.1469461917877197,
      "logps/chosen": -20.566917419433594,
      "logps/rejected": -36.77910614013672,
      "loss": 0.047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2701362371444702,
      "rewards/margins": 6.22340202331543,
      "rewards/rejected": -5.95326566696167,
      "step": 820
    },
    {
      "epoch": 13.915254237288135,
      "grad_norm": 6.0534564497811,
      "learning_rate": 1.282158855929445e-07,
      "logits/chosen": -4.0090460777282715,
      "logits/rejected": -2.398519515991211,
      "logps/chosen": -17.34862518310547,
      "logps/rejected": -31.26947021484375,
      "loss": 0.066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38930952548980713,
      "rewards/margins": 4.36777925491333,
      "rewards/rejected": -3.9784698486328125,
      "step": 821
    },
    {
      "epoch": 13.932203389830509,
      "grad_norm": 4.712692482905532,
      "learning_rate": 1.275705544839509e-07,
      "logits/chosen": -2.769106864929199,
      "logits/rejected": -1.0413520336151123,
      "logps/chosen": -22.0438232421875,
      "logps/rejected": -37.239341735839844,
      "loss": 0.0632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0389874204993248,
      "rewards/margins": 5.474620819091797,
      "rewards/rejected": -5.513607978820801,
      "step": 822
    },
    {
      "epoch": 13.94915254237288,
      "grad_norm": 5.289659327917796,
      "learning_rate": 1.2692629473705452e-07,
      "logits/chosen": 0.8868709802627563,
      "logits/rejected": 1.507910966873169,
      "logps/chosen": -16.889572143554688,
      "logps/rejected": -29.149890899658203,
      "loss": 0.0543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1100674495100975,
      "rewards/margins": 4.167652130126953,
      "rewards/rejected": -4.277719497680664,
      "step": 823
    },
    {
      "epoch": 13.966101694915254,
      "grad_norm": 5.966734982532794,
      "learning_rate": 1.2628311199007762e-07,
      "logits/chosen": -2.237454652786255,
      "logits/rejected": -1.3493525981903076,
      "logps/chosen": -15.533367156982422,
      "logps/rejected": -27.488908767700195,
      "loss": 0.0649,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.02705906331539154,
      "rewards/margins": 4.565030574798584,
      "rewards/rejected": -4.5379719734191895,
      "step": 824
    },
    {
      "epoch": 13.983050847457626,
      "grad_norm": 4.76829853180543,
      "learning_rate": 1.2564101187141828e-07,
      "logits/chosen": -3.445924758911133,
      "logits/rejected": -2.606234550476074,
      "logps/chosen": -12.318361282348633,
      "logps/rejected": -33.33863830566406,
      "loss": 0.054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.674911618232727,
      "rewards/margins": 6.01348876953125,
      "rewards/rejected": -5.338577747344971,
      "step": 825
    },
    {
      "epoch": 14.0,
      "grad_norm": 4.855237519617607,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": -2.449713706970215,
      "logits/rejected": -1.7510132789611816,
      "logps/chosen": -21.224750518798828,
      "logps/rejected": -34.99382400512695,
      "loss": 0.0601,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.32820674777030945,
      "rewards/margins": 5.6509318351745605,
      "rewards/rejected": -5.9791388511657715,
      "step": 826
    },
    {
      "epoch": 14.016949152542374,
      "grad_norm": 4.54608207656459,
      "learning_rate": 1.2436008198522374e-07,
      "logits/chosen": -0.290882408618927,
      "logits/rejected": 0.5373449325561523,
      "logps/chosen": -18.350481033325195,
      "logps/rejected": -35.856468200683594,
      "loss": 0.0511,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.007371596992015839,
      "rewards/margins": 5.433517932891846,
      "rewards/rejected": -5.426146507263184,
      "step": 827
    },
    {
      "epoch": 14.033898305084746,
      "grad_norm": 4.908312293515908,
      "learning_rate": 1.2372126342691797e-07,
      "logits/chosen": -4.190606594085693,
      "logits/rejected": -3.0944652557373047,
      "logps/chosen": -19.034318923950195,
      "logps/rejected": -34.272186279296875,
      "loss": 0.0547,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2908100485801697,
      "rewards/margins": 4.515417575836182,
      "rewards/rejected": -4.224607467651367,
      "step": 828
    },
    {
      "epoch": 14.05084745762712,
      "grad_norm": 5.045804367551582,
      "learning_rate": 1.2308354991529006e-07,
      "logits/chosen": -0.760540246963501,
      "logits/rejected": -1.555609107017517,
      "logps/chosen": -14.472261428833008,
      "logps/rejected": -36.33820724487305,
      "loss": 0.0653,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.45639586448669434,
      "rewards/margins": 4.686809062957764,
      "rewards/rejected": -4.23041296005249,
      "step": 829
    },
    {
      "epoch": 14.067796610169491,
      "grad_norm": 4.199306543227042,
      "learning_rate": 1.2244694703087727e-07,
      "logits/chosen": -3.3680789470672607,
      "logits/rejected": -1.9466626644134521,
      "logps/chosen": -26.21337127685547,
      "logps/rejected": -34.70789337158203,
      "loss": 0.0512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30467984080314636,
      "rewards/margins": 4.528075695037842,
      "rewards/rejected": -4.223395824432373,
      "step": 830
    },
    {
      "epoch": 14.084745762711865,
      "grad_norm": 4.672521820477395,
      "learning_rate": 1.2181146034449807e-07,
      "logits/chosen": -2.1504950523376465,
      "logits/rejected": -1.46886146068573,
      "logps/chosen": -15.693143844604492,
      "logps/rejected": -26.54515838623047,
      "loss": 0.0565,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22424635291099548,
      "rewards/margins": 3.5846357345581055,
      "rewards/rejected": -3.3603897094726562,
      "step": 831
    },
    {
      "epoch": 14.101694915254237,
      "grad_norm": 5.058340149965996,
      "learning_rate": 1.2117709541720306e-07,
      "logits/chosen": -2.309311866760254,
      "logits/rejected": -2.2456936836242676,
      "logps/chosen": -20.746421813964844,
      "logps/rejected": -34.04157638549805,
      "loss": 0.0621,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.012225247919559479,
      "rewards/margins": 4.281233787536621,
      "rewards/rejected": -4.269008636474609,
      "step": 832
    },
    {
      "epoch": 14.11864406779661,
      "grad_norm": 4.80031946203614,
      "learning_rate": 1.2054385780022655e-07,
      "logits/chosen": 0.12395321577787399,
      "logits/rejected": 0.8067972660064697,
      "logps/chosen": -16.532777786254883,
      "logps/rejected": -38.36420440673828,
      "loss": 0.0545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06285473704338074,
      "rewards/margins": 5.238184928894043,
      "rewards/rejected": -5.175330638885498,
      "step": 833
    },
    {
      "epoch": 14.135593220338983,
      "grad_norm": 4.784487064199788,
      "learning_rate": 1.199117530349379e-07,
      "logits/chosen": -3.471977472305298,
      "logits/rejected": -3.575911521911621,
      "logps/chosen": -17.44545555114746,
      "logps/rejected": -32.76516342163086,
      "loss": 0.0494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09463706612586975,
      "rewards/margins": 5.1845011711120605,
      "rewards/rejected": -5.0898637771606445,
      "step": 834
    },
    {
      "epoch": 14.152542372881356,
      "grad_norm": 4.424987876484658,
      "learning_rate": 1.192807866527931e-07,
      "logits/chosen": -1.5066072940826416,
      "logits/rejected": -0.8759188055992126,
      "logps/chosen": -18.205663681030273,
      "logps/rejected": -31.757957458496094,
      "loss": 0.0455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26423490047454834,
      "rewards/margins": 4.786740303039551,
      "rewards/rejected": -4.522505760192871,
      "step": 835
    },
    {
      "epoch": 14.169491525423728,
      "grad_norm": 4.955141012463688,
      "learning_rate": 1.1865096417528633e-07,
      "logits/chosen": -0.48518604040145874,
      "logits/rejected": 0.1290414035320282,
      "logps/chosen": -13.834370613098145,
      "logps/rejected": -27.40927505493164,
      "loss": 0.0585,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2843541204929352,
      "rewards/margins": 4.416263103485107,
      "rewards/rejected": -4.131909370422363,
      "step": 836
    },
    {
      "epoch": 14.186440677966102,
      "grad_norm": 5.441205501869942,
      "learning_rate": 1.1802229111390155e-07,
      "logits/chosen": -1.3898885250091553,
      "logits/rejected": 0.1110721230506897,
      "logps/chosen": -15.802949905395508,
      "logps/rejected": -42.55592346191406,
      "loss": 0.0637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11498227715492249,
      "rewards/margins": 7.029240131378174,
      "rewards/rejected": -6.914258003234863,
      "step": 837
    },
    {
      "epoch": 14.203389830508474,
      "grad_norm": 4.2930763676154315,
      "learning_rate": 1.173947729700644e-07,
      "logits/chosen": -1.319080114364624,
      "logits/rejected": -0.31338560581207275,
      "logps/chosen": -20.780853271484375,
      "logps/rejected": -37.78578186035156,
      "loss": 0.0626,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14082486927509308,
      "rewards/margins": 5.361012935638428,
      "rewards/rejected": -5.501838684082031,
      "step": 838
    },
    {
      "epoch": 14.220338983050848,
      "grad_norm": 4.6566564145391975,
      "learning_rate": 1.1676841523509398e-07,
      "logits/chosen": -2.1716244220733643,
      "logits/rejected": -1.0681712627410889,
      "logps/chosen": -21.671016693115234,
      "logps/rejected": -32.90324401855469,
      "loss": 0.0541,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4317570924758911,
      "rewards/margins": 5.18146276473999,
      "rewards/rejected": -4.749706268310547,
      "step": 839
    },
    {
      "epoch": 14.23728813559322,
      "grad_norm": 4.056864733239223,
      "learning_rate": 1.1614322339015484e-07,
      "logits/chosen": -1.7106143236160278,
      "logits/rejected": -0.9364138245582581,
      "logps/chosen": -19.77383804321289,
      "logps/rejected": -35.89718246459961,
      "loss": 0.0434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35204410552978516,
      "rewards/margins": 6.058632850646973,
      "rewards/rejected": -5.7065887451171875,
      "step": 840
    },
    {
      "epoch": 14.254237288135593,
      "grad_norm": 4.983849879172364,
      "learning_rate": 1.1551920290620903e-07,
      "logits/chosen": -2.1713905334472656,
      "logits/rejected": -1.5407037734985352,
      "logps/chosen": -12.266080856323242,
      "logps/rejected": -29.389978408813477,
      "loss": 0.0455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4713742733001709,
      "rewards/margins": 4.8038177490234375,
      "rewards/rejected": -4.3324432373046875,
      "step": 841
    },
    {
      "epoch": 14.271186440677965,
      "grad_norm": 5.360406348411582,
      "learning_rate": 1.1489635924396815e-07,
      "logits/chosen": -1.4929659366607666,
      "logits/rejected": -1.120671033859253,
      "logps/chosen": -19.657901763916016,
      "logps/rejected": -41.733489990234375,
      "loss": 0.065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.41843128204345703,
      "rewards/margins": 5.889748573303223,
      "rewards/rejected": -6.30817985534668,
      "step": 842
    },
    {
      "epoch": 14.288135593220339,
      "grad_norm": 4.853679039074438,
      "learning_rate": 1.1427469785384558e-07,
      "logits/chosen": -1.4412063360214233,
      "logits/rejected": -0.877423882484436,
      "logps/chosen": -16.15688133239746,
      "logps/rejected": -35.47041320800781,
      "loss": 0.0489,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.431803822517395,
      "rewards/margins": 5.776800632476807,
      "rewards/rejected": -5.344996929168701,
      "step": 843
    },
    {
      "epoch": 14.305084745762711,
      "grad_norm": 4.74221982053372,
      "learning_rate": 1.1365422417590878e-07,
      "logits/chosen": 0.059105001389980316,
      "logits/rejected": 2.012216329574585,
      "logps/chosen": -20.8731689453125,
      "logps/rejected": -38.89690399169922,
      "loss": 0.058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27555304765701294,
      "rewards/margins": 6.34234619140625,
      "rewards/rejected": -6.066792964935303,
      "step": 844
    },
    {
      "epoch": 14.322033898305085,
      "grad_norm": 4.7538090273563025,
      "learning_rate": 1.1303494363983196e-07,
      "logits/chosen": -1.425571322441101,
      "logits/rejected": -0.497536838054657,
      "logps/chosen": -15.412924766540527,
      "logps/rejected": -29.72442626953125,
      "loss": 0.0542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09866355359554291,
      "rewards/margins": 4.266383647918701,
      "rewards/rejected": -4.167720317840576,
      "step": 845
    },
    {
      "epoch": 14.338983050847457,
      "grad_norm": 4.844067597917288,
      "learning_rate": 1.1241686166484804e-07,
      "logits/chosen": -1.9793237447738647,
      "logits/rejected": -0.8182030320167542,
      "logps/chosen": -15.300029754638672,
      "logps/rejected": -31.1224308013916,
      "loss": 0.0475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10771667957305908,
      "rewards/margins": 4.908604145050049,
      "rewards/rejected": -4.800887584686279,
      "step": 846
    },
    {
      "epoch": 14.35593220338983,
      "grad_norm": 4.878898863334605,
      "learning_rate": 1.1179998365970172e-07,
      "logits/chosen": -0.17034506797790527,
      "logits/rejected": 0.5397195219993591,
      "logps/chosen": -17.27823257446289,
      "logps/rejected": -29.693744659423828,
      "loss": 0.067,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09199562668800354,
      "rewards/margins": 4.912941932678223,
      "rewards/rejected": -4.820946216583252,
      "step": 847
    },
    {
      "epoch": 14.372881355932204,
      "grad_norm": 4.559682445793414,
      "learning_rate": 1.1118431502260162e-07,
      "logits/chosen": -0.6307776570320129,
      "logits/rejected": 0.6813377737998962,
      "logps/chosen": -11.645711898803711,
      "logps/rejected": -33.321189880371094,
      "loss": 0.0539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3501269221305847,
      "rewards/margins": 5.4904913902282715,
      "rewards/rejected": -5.140363693237305,
      "step": 848
    },
    {
      "epoch": 14.389830508474576,
      "grad_norm": 4.036758306534431,
      "learning_rate": 1.1056986114117367e-07,
      "logits/chosen": -1.4370734691619873,
      "logits/rejected": -0.3933512270450592,
      "logps/chosen": -14.565434455871582,
      "logps/rejected": -31.868255615234375,
      "loss": 0.0426,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3414157032966614,
      "rewards/margins": 5.2931809425354,
      "rewards/rejected": -4.951766014099121,
      "step": 849
    },
    {
      "epoch": 14.40677966101695,
      "grad_norm": 4.642153321658131,
      "learning_rate": 1.0995662739241346e-07,
      "logits/chosen": -3.0956766605377197,
      "logits/rejected": -2.5723183155059814,
      "logps/chosen": -21.624290466308594,
      "logps/rejected": -42.88805389404297,
      "loss": 0.0533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2431950867176056,
      "rewards/margins": 5.845188140869141,
      "rewards/rejected": -5.601992607116699,
      "step": 850
    },
    {
      "epoch": 14.423728813559322,
      "grad_norm": 4.307115009032184,
      "learning_rate": 1.0934461914263965e-07,
      "logits/chosen": -1.71174955368042,
      "logits/rejected": 0.0297551229596138,
      "logps/chosen": -16.369564056396484,
      "logps/rejected": -32.02233123779297,
      "loss": 0.0486,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.33911365270614624,
      "rewards/margins": 4.879045486450195,
      "rewards/rejected": -4.5399322509765625,
      "step": 851
    },
    {
      "epoch": 14.440677966101696,
      "grad_norm": 4.791655857038553,
      "learning_rate": 1.087338417474464e-07,
      "logits/chosen": -0.3006948232650757,
      "logits/rejected": 0.6420655250549316,
      "logps/chosen": -13.013580322265625,
      "logps/rejected": -35.55011749267578,
      "loss": 0.0476,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17818471789360046,
      "rewards/margins": 5.40357780456543,
      "rewards/rejected": -5.581762313842773,
      "step": 852
    },
    {
      "epoch": 14.457627118644067,
      "grad_norm": 5.0370461913788,
      "learning_rate": 1.0812430055165709e-07,
      "logits/chosen": -3.104762315750122,
      "logits/rejected": -2.9962918758392334,
      "logps/chosen": -22.40864372253418,
      "logps/rejected": -33.83332824707031,
      "loss": 0.0545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1751263439655304,
      "rewards/margins": 4.681097984313965,
      "rewards/rejected": -4.505971431732178,
      "step": 853
    },
    {
      "epoch": 14.474576271186441,
      "grad_norm": 4.114102239483635,
      "learning_rate": 1.0751600088927712e-07,
      "logits/chosen": -0.8753526210784912,
      "logits/rejected": -0.42285048961639404,
      "logps/chosen": -17.130706787109375,
      "logps/rejected": -36.58259582519531,
      "loss": 0.0521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19100093841552734,
      "rewards/margins": 5.029597282409668,
      "rewards/rejected": -4.838596343994141,
      "step": 854
    },
    {
      "epoch": 14.491525423728813,
      "grad_norm": 4.674717207809757,
      "learning_rate": 1.0690894808344756e-07,
      "logits/chosen": -0.887531042098999,
      "logits/rejected": 0.030666358768939972,
      "logps/chosen": -18.499759674072266,
      "logps/rejected": -33.44499206542969,
      "loss": 0.048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05299860239028931,
      "rewards/margins": 5.534433364868164,
      "rewards/rejected": -5.4814348220825195,
      "step": 855
    },
    {
      "epoch": 14.508474576271187,
      "grad_norm": 5.571055008821593,
      "learning_rate": 1.0630314744639829e-07,
      "logits/chosen": -2.3482229709625244,
      "logits/rejected": -1.5293030738830566,
      "logps/chosen": -20.04794692993164,
      "logps/rejected": -33.5222053527832,
      "loss": 0.0713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03216637670993805,
      "rewards/margins": 5.284518241882324,
      "rewards/rejected": -5.316684722900391,
      "step": 856
    },
    {
      "epoch": 14.525423728813559,
      "grad_norm": 4.036017883945882,
      "learning_rate": 1.0569860427940178e-07,
      "logits/chosen": -4.571822643280029,
      "logits/rejected": -3.6904447078704834,
      "logps/chosen": -17.9635066986084,
      "logps/rejected": -30.057472229003906,
      "loss": 0.0416,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03753848373889923,
      "rewards/margins": 4.019134521484375,
      "rewards/rejected": -3.981595993041992,
      "step": 857
    },
    {
      "epoch": 14.542372881355933,
      "grad_norm": 4.455559455534382,
      "learning_rate": 1.050953238727264e-07,
      "logits/chosen": -2.462489366531372,
      "logits/rejected": -2.7516651153564453,
      "logps/chosen": -14.60168743133545,
      "logps/rejected": -29.86794662475586,
      "loss": 0.0543,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3153627812862396,
      "rewards/margins": 4.536343574523926,
      "rewards/rejected": -4.220980644226074,
      "step": 858
    },
    {
      "epoch": 14.559322033898304,
      "grad_norm": 4.409069243450616,
      "learning_rate": 1.0449331150559063e-07,
      "logits/chosen": 0.21322837471961975,
      "logits/rejected": 0.560262143611908,
      "logps/chosen": -19.38181495666504,
      "logps/rejected": -37.284568786621094,
      "loss": 0.043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0371914803981781,
      "rewards/margins": 6.507880210876465,
      "rewards/rejected": -6.470687389373779,
      "step": 859
    },
    {
      "epoch": 14.576271186440678,
      "grad_norm": 3.8106192326465425,
      "learning_rate": 1.0389257244611601e-07,
      "logits/chosen": -3.911578893661499,
      "logits/rejected": -3.2801082134246826,
      "logps/chosen": -18.08879280090332,
      "logps/rejected": -32.2973747253418,
      "loss": 0.0411,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41884616017341614,
      "rewards/margins": 4.887192726135254,
      "rewards/rejected": -4.46834659576416,
      "step": 860
    },
    {
      "epoch": 14.59322033898305,
      "grad_norm": 4.10040976821886,
      "learning_rate": 1.0329311195128193e-07,
      "logits/chosen": -3.737536907196045,
      "logits/rejected": -1.8438628911972046,
      "logps/chosen": -13.185300827026367,
      "logps/rejected": -31.151159286499023,
      "loss": 0.0432,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.189189612865448,
      "rewards/margins": 5.370091915130615,
      "rewards/rejected": -5.559281826019287,
      "step": 861
    },
    {
      "epoch": 14.610169491525424,
      "grad_norm": 4.689298687726538,
      "learning_rate": 1.0269493526687914e-07,
      "logits/chosen": -4.051319122314453,
      "logits/rejected": -2.245673418045044,
      "logps/chosen": -21.487590789794922,
      "logps/rejected": -32.1451301574707,
      "loss": 0.0595,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4900110960006714,
      "rewards/margins": 4.4578471183776855,
      "rewards/rejected": -3.967834949493408,
      "step": 862
    },
    {
      "epoch": 14.627118644067796,
      "grad_norm": 4.629593885843378,
      "learning_rate": 1.0209804762746396e-07,
      "logits/chosen": -1.2015026807785034,
      "logits/rejected": 0.05276120454072952,
      "logps/chosen": -19.471660614013672,
      "logps/rejected": -33.03922653198242,
      "loss": 0.0626,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06236635148525238,
      "rewards/margins": 5.169223308563232,
      "rewards/rejected": -5.106856822967529,
      "step": 863
    },
    {
      "epoch": 14.64406779661017,
      "grad_norm": 5.0290811803860205,
      "learning_rate": 1.0150245425631235e-07,
      "logits/chosen": -0.8136476278305054,
      "logits/rejected": -0.6300673484802246,
      "logps/chosen": -14.205711364746094,
      "logps/rejected": -29.096141815185547,
      "loss": 0.0538,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31267741322517395,
      "rewards/margins": 4.294619560241699,
      "rewards/rejected": -3.9819421768188477,
      "step": 864
    },
    {
      "epoch": 14.661016949152543,
      "grad_norm": 4.984998268591449,
      "learning_rate": 1.0090816036537461e-07,
      "logits/chosen": -4.853797912597656,
      "logits/rejected": -4.035728931427002,
      "logps/chosen": -13.243602752685547,
      "logps/rejected": -32.274715423583984,
      "loss": 0.0488,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05507740378379822,
      "rewards/margins": 5.407604694366455,
      "rewards/rejected": -5.462681293487549,
      "step": 865
    },
    {
      "epoch": 14.677966101694915,
      "grad_norm": 4.935756536829011,
      "learning_rate": 1.0031517115522925e-07,
      "logits/chosen": -0.9398390054702759,
      "logits/rejected": -0.6894736289978027,
      "logps/chosen": -15.939778327941895,
      "logps/rejected": -29.07857894897461,
      "loss": 0.06,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2174006849527359,
      "rewards/margins": 4.617957592010498,
      "rewards/rejected": -4.400557041168213,
      "step": 866
    },
    {
      "epoch": 14.694915254237289,
      "grad_norm": 5.177827880857834,
      "learning_rate": 9.972349181503773e-08,
      "logits/chosen": -2.80765962600708,
      "logits/rejected": -2.379774570465088,
      "logps/chosen": -13.54705810546875,
      "logps/rejected": -27.66398811340332,
      "loss": 0.0548,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.011886119842529297,
      "rewards/margins": 4.244990348815918,
      "rewards/rejected": -4.233104228973389,
      "step": 867
    },
    {
      "epoch": 14.711864406779661,
      "grad_norm": 4.651526502576263,
      "learning_rate": 9.913312752249903e-08,
      "logits/chosen": 0.8737283945083618,
      "logits/rejected": 0.9006978273391724,
      "logps/chosen": -15.809414863586426,
      "logps/rejected": -31.1838436126709,
      "loss": 0.0568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.017194844782352448,
      "rewards/margins": 4.261214733123779,
      "rewards/rejected": -4.278409481048584,
      "step": 868
    },
    {
      "epoch": 14.728813559322035,
      "grad_norm": 3.8276656992018285,
      "learning_rate": 9.85440834438044e-08,
      "logits/chosen": -2.807131052017212,
      "logits/rejected": -1.8012992143630981,
      "logps/chosen": -19.008888244628906,
      "logps/rejected": -37.323631286621094,
      "loss": 0.0361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3896780014038086,
      "rewards/margins": 5.351907730102539,
      "rewards/rejected": -4.962228775024414,
      "step": 869
    },
    {
      "epoch": 14.745762711864407,
      "grad_norm": 5.226466365324867,
      "learning_rate": 9.795636473359207e-08,
      "logits/chosen": -1.8736178874969482,
      "logits/rejected": -1.2298340797424316,
      "logps/chosen": -20.007198333740234,
      "logps/rejected": -29.63496208190918,
      "loss": 0.0588,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11613491177558899,
      "rewards/margins": 3.2119195461273193,
      "rewards/rejected": -3.095784902572632,
      "step": 870
    },
    {
      "epoch": 14.76271186440678,
      "grad_norm": 4.897408411630909,
      "learning_rate": 9.736997653490214e-08,
      "logits/chosen": 2.0472121238708496,
      "logits/rejected": 2.940474033355713,
      "logps/chosen": -17.741928100585938,
      "logps/rejected": -37.018699645996094,
      "loss": 0.064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.27036699652671814,
      "rewards/margins": 5.808854103088379,
      "rewards/rejected": -6.079221725463867,
      "step": 871
    },
    {
      "epoch": 14.779661016949152,
      "grad_norm": 4.896869366507913,
      "learning_rate": 9.678492397913165e-08,
      "logits/chosen": -3.1093807220458984,
      "logits/rejected": -1.964849829673767,
      "logps/chosen": -24.06382179260254,
      "logps/rejected": -39.17333984375,
      "loss": 0.0504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.24048033356666565,
      "rewards/margins": 5.500030040740967,
      "rewards/rejected": -5.7405104637146,
      "step": 872
    },
    {
      "epoch": 14.796610169491526,
      "grad_norm": 4.236238856435463,
      "learning_rate": 9.620121218598957e-08,
      "logits/chosen": -3.5577645301818848,
      "logits/rejected": -3.226468563079834,
      "logps/chosen": -17.269495010375977,
      "logps/rejected": -29.61627960205078,
      "loss": 0.0463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20766395330429077,
      "rewards/margins": 5.623841285705566,
      "rewards/rejected": -5.416177272796631,
      "step": 873
    },
    {
      "epoch": 14.813559322033898,
      "grad_norm": 4.775367869659583,
      "learning_rate": 9.561884626345204e-08,
      "logits/chosen": -2.6089420318603516,
      "logits/rejected": -3.4899003505706787,
      "logps/chosen": -15.538010597229004,
      "logps/rejected": -29.64735221862793,
      "loss": 0.0515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.801546573638916,
      "rewards/margins": 4.463309288024902,
      "rewards/rejected": -3.6617627143859863,
      "step": 874
    },
    {
      "epoch": 14.830508474576272,
      "grad_norm": 5.311387814948821,
      "learning_rate": 9.503783130771778e-08,
      "logits/chosen": -2.0723490715026855,
      "logits/rejected": -0.9905359148979187,
      "logps/chosen": -16.402400970458984,
      "logps/rejected": -31.389341354370117,
      "loss": 0.0531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6416011452674866,
      "rewards/margins": 4.774829387664795,
      "rewards/rejected": -4.133228302001953,
      "step": 875
    },
    {
      "epoch": 14.847457627118644,
      "grad_norm": 4.576464275088217,
      "learning_rate": 9.445817240316332e-08,
      "logits/chosen": -0.9681025743484497,
      "logits/rejected": 1.3066233396530151,
      "logps/chosen": -15.319185256958008,
      "logps/rejected": -30.807912826538086,
      "loss": 0.0604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10009877383708954,
      "rewards/margins": 4.77067756652832,
      "rewards/rejected": -4.870776653289795,
      "step": 876
    },
    {
      "epoch": 14.864406779661017,
      "grad_norm": 3.566749084202065,
      "learning_rate": 9.387987462229857e-08,
      "logits/chosen": -3.0737295150756836,
      "logits/rejected": -1.5600917339324951,
      "logps/chosen": -13.265515327453613,
      "logps/rejected": -35.03205108642578,
      "loss": 0.0327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16393795609474182,
      "rewards/margins": 5.836406707763672,
      "rewards/rejected": -5.672469139099121,
      "step": 877
    },
    {
      "epoch": 14.88135593220339,
      "grad_norm": 3.732800979255398,
      "learning_rate": 9.330294302572242e-08,
      "logits/chosen": -1.8261581659317017,
      "logits/rejected": -1.2361934185028076,
      "logps/chosen": -15.713083267211914,
      "logps/rejected": -28.814865112304688,
      "loss": 0.0408,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16630926728248596,
      "rewards/margins": 5.071293354034424,
      "rewards/rejected": -4.904984474182129,
      "step": 878
    },
    {
      "epoch": 14.898305084745763,
      "grad_norm": 4.831490870008395,
      "learning_rate": 9.272738266207871e-08,
      "logits/chosen": -2.989412307739258,
      "logits/rejected": -2.152963161468506,
      "logps/chosen": -17.28832244873047,
      "logps/rejected": -35.29121780395508,
      "loss": 0.0598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2409454882144928,
      "rewards/margins": 4.79300594329834,
      "rewards/rejected": -4.552060127258301,
      "step": 879
    },
    {
      "epoch": 14.915254237288135,
      "grad_norm": 4.787771544995592,
      "learning_rate": 9.215319856801157e-08,
      "logits/chosen": -2.2084031105041504,
      "logits/rejected": -0.7729737758636475,
      "logps/chosen": -16.84943199157715,
      "logps/rejected": -31.124263763427734,
      "loss": 0.0571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07245789468288422,
      "rewards/margins": 4.6041388511657715,
      "rewards/rejected": -4.676596641540527,
      "step": 880
    }
  ],
  "logging_steps": 1,
  "max_steps": 1180,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 20,
  "save_steps": 80,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}