{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9984827495217362,
  "eval_steps": 400,
  "global_step": 473,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002110957187149548,
      "grad_norm": 4.06041781261902,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": -0.9878771901130676,
      "logits/rejected": -0.7230668663978577,
      "logps/chosen": -251.34963989257812,
      "logps/rejected": -287.15838623046875,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01055478593574774,
      "grad_norm": 4.105776204209711,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -0.9975427985191345,
      "logits/rejected": -0.6888133883476257,
      "logps/chosen": -272.86993408203125,
      "logps/rejected": -286.1126708984375,
      "loss": 0.6932,
      "rewards/accuracies": 0.359375,
      "rewards/chosen": -0.00011302110215183347,
      "rewards/margins": -0.00016400158347096294,
      "rewards/rejected": 5.0980423111468554e-05,
      "step": 5
    },
    {
      "epoch": 0.02110957187149548,
      "grad_norm": 4.582740122598074,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -1.0306963920593262,
      "logits/rejected": -0.7614760398864746,
      "logps/chosen": -274.9240417480469,
      "logps/rejected": -293.0102233886719,
      "loss": 0.6933,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0007906880346126854,
      "rewards/margins": 0.00033544833422638476,
      "rewards/rejected": 0.00045523978769779205,
      "step": 10
    },
    {
      "epoch": 0.03166435780724322,
      "grad_norm": 4.2774752538157,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -0.9785528182983398,
      "logits/rejected": -0.6860870122909546,
      "logps/chosen": -253.0674285888672,
      "logps/rejected": -277.63006591796875,
      "loss": 0.693,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.00013054809824097902,
      "rewards/margins": -4.017539322376251e-05,
      "rewards/rejected": 0.0001707235351204872,
      "step": 15
    },
    {
      "epoch": 0.04221914374299096,
      "grad_norm": 4.154426876518502,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -0.8861902356147766,
      "logits/rejected": -0.7466350793838501,
      "logps/chosen": -303.89990234375,
      "logps/rejected": -317.44354248046875,
      "loss": 0.6929,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.0017729544779285789,
      "rewards/margins": 0.0007360944291576743,
      "rewards/rejected": -0.0025090486742556095,
      "step": 20
    },
    {
      "epoch": 0.0527739296787387,
      "grad_norm": 4.391172268221911,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -0.9347459077835083,
      "logits/rejected": -0.7482324242591858,
      "logps/chosen": -257.277099609375,
      "logps/rejected": -276.6146240234375,
      "loss": 0.6925,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0021017056424170732,
      "rewards/margins": 0.001351111801341176,
      "rewards/rejected": -0.003452816978096962,
      "step": 25
    },
    {
      "epoch": 0.06332871561448644,
      "grad_norm": 4.197922988205761,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -0.9495643377304077,
      "logits/rejected": -0.722493052482605,
      "logps/chosen": -271.043212890625,
      "logps/rejected": -304.54779052734375,
      "loss": 0.6909,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.007976134307682514,
      "rewards/margins": 0.006574218161404133,
      "rewards/rejected": -0.014550352469086647,
      "step": 30
    },
    {
      "epoch": 0.07388350155023418,
      "grad_norm": 4.146928571575294,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -0.9740797281265259,
      "logits/rejected": -0.6511734127998352,
      "logps/chosen": -269.66558837890625,
      "logps/rejected": -282.5665588378906,
      "loss": 0.6901,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.014239413663744926,
      "rewards/margins": 0.006516980938613415,
      "rewards/rejected": -0.020756395533680916,
      "step": 35
    },
    {
      "epoch": 0.08443828748598192,
      "grad_norm": 4.434625304551974,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -0.9476411938667297,
      "logits/rejected": -0.8075205087661743,
      "logps/chosen": -254.89315795898438,
      "logps/rejected": -269.08843994140625,
      "loss": 0.6872,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.025963688269257545,
      "rewards/margins": 0.010111861862242222,
      "rewards/rejected": -0.03607555106282234,
      "step": 40
    },
    {
      "epoch": 0.09499307342172966,
      "grad_norm": 4.334333725960549,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -0.9668118357658386,
      "logits/rejected": -0.7818160057067871,
      "logps/chosen": -262.55804443359375,
      "logps/rejected": -279.33416748046875,
      "loss": 0.6841,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03984779864549637,
      "rewards/margins": 0.01882680132985115,
      "rewards/rejected": -0.05867459625005722,
      "step": 45
    },
    {
      "epoch": 0.1055478593574774,
      "grad_norm": 4.672589574927753,
      "learning_rate": 4.999726797933858e-07,
      "logits/chosen": -1.1188008785247803,
      "logits/rejected": -0.7495776414871216,
      "logps/chosen": -274.8739318847656,
      "logps/rejected": -298.7273254394531,
      "loss": 0.6736,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0697299912571907,
      "rewards/margins": 0.04492691531777382,
      "rewards/rejected": -0.11465690284967422,
      "step": 50
    },
    {
      "epoch": 0.11610264529322514,
      "grad_norm": 6.181545467894853,
      "learning_rate": 4.99665396039775e-07,
      "logits/chosen": -1.1051629781723022,
      "logits/rejected": -0.9252668619155884,
      "logps/chosen": -270.87841796875,
      "logps/rejected": -289.1105651855469,
      "loss": 0.6576,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.14251390099525452,
      "rewards/margins": 0.10730250179767609,
      "rewards/rejected": -0.2498163878917694,
      "step": 55
    },
    {
      "epoch": 0.12665743122897288,
      "grad_norm": 7.558458036407823,
      "learning_rate": 4.99017099386437e-07,
      "logits/chosen": -1.3753994703292847,
      "logits/rejected": -1.1374595165252686,
      "logps/chosen": -306.73138427734375,
      "logps/rejected": -341.1016845703125,
      "loss": 0.6281,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.37672942876815796,
      "rewards/margins": 0.20342817902565002,
      "rewards/rejected": -0.5801576375961304,
      "step": 60
    },
    {
      "epoch": 0.13721221716472062,
      "grad_norm": 66.9078697860869,
      "learning_rate": 4.980286753286194e-07,
      "logits/chosen": -1.5234780311584473,
      "logits/rejected": -1.2897632122039795,
      "logps/chosen": -359.58074951171875,
      "logps/rejected": -414.0733947753906,
      "loss": 0.6211,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.063786506652832,
      "rewards/margins": 0.41773301362991333,
      "rewards/rejected": -1.4815195798873901,
      "step": 65
    },
    {
      "epoch": 0.14776700310046836,
      "grad_norm": 12.864645326069963,
      "learning_rate": 4.967014739346915e-07,
      "logits/chosen": -1.4892756938934326,
      "logits/rejected": -1.3145514726638794,
      "logps/chosen": -410.1991271972656,
      "logps/rejected": -467.57623291015625,
      "loss": 0.6578,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.4497530460357666,
      "rewards/margins": 0.4113241136074066,
      "rewards/rejected": -1.8610769510269165,
      "step": 70
    },
    {
      "epoch": 0.1583217890362161,
      "grad_norm": 12.52823747211519,
      "learning_rate": 4.950373080021136e-07,
      "logits/chosen": -1.2871811389923096,
      "logits/rejected": -1.2107694149017334,
      "logps/chosen": -333.7770080566406,
      "logps/rejected": -356.77728271484375,
      "loss": 0.5936,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.579529881477356,
      "rewards/margins": 0.2453722506761551,
      "rewards/rejected": -0.8249020576477051,
      "step": 75
    },
    {
      "epoch": 0.16887657497196384,
      "grad_norm": 17.45417402457315,
      "learning_rate": 4.930384505813737e-07,
      "logits/chosen": -1.4544992446899414,
      "logits/rejected": -1.3903002738952637,
      "logps/chosen": -392.7482604980469,
      "logps/rejected": -472.0469665527344,
      "loss": 0.5588,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.0880345106124878,
      "rewards/margins": 0.6102110147476196,
      "rewards/rejected": -1.6982454061508179,
      "step": 80
    },
    {
      "epoch": 0.17943136090771158,
      "grad_norm": 36.09231440231977,
      "learning_rate": 4.907076318712738e-07,
      "logits/chosen": -1.5770976543426514,
      "logits/rejected": -1.4604318141937256,
      "logps/chosen": -478.90264892578125,
      "logps/rejected": -560.0563354492188,
      "loss": 0.557,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.8632774353027344,
      "rewards/margins": 0.6757477521896362,
      "rewards/rejected": -2.53902530670166,
      "step": 85
    },
    {
      "epoch": 0.18998614684345932,
      "grad_norm": 12.93821128098639,
      "learning_rate": 4.88048035489807e-07,
      "logits/chosen": -1.5192902088165283,
      "logits/rejected": -1.3578670024871826,
      "logps/chosen": -401.8139343261719,
      "logps/rejected": -476.27557373046875,
      "loss": 0.5413,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.2336255311965942,
      "rewards/margins": 0.6196562051773071,
      "rewards/rejected": -1.8532816171646118,
      "step": 90
    },
    {
      "epoch": 0.20054093277920707,
      "grad_norm": 10.174846260495245,
      "learning_rate": 4.85063294125718e-07,
      "logits/chosen": -1.4184257984161377,
      "logits/rejected": -1.349448561668396,
      "logps/chosen": -405.37030029296875,
      "logps/rejected": -465.87835693359375,
      "loss": 0.584,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.2496789693832397,
      "rewards/margins": 0.4809112548828125,
      "rewards/rejected": -1.7305902242660522,
      "step": 95
    },
    {
      "epoch": 0.2110957187149548,
      "grad_norm": 11.934921621793764,
      "learning_rate": 4.817574845766874e-07,
      "logits/chosen": -1.6645218133926392,
      "logits/rejected": -1.4362655878067017,
      "logps/chosen": -419.6376953125,
      "logps/rejected": -511.97955322265625,
      "loss": 0.5179,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.6008678674697876,
      "rewards/margins": 0.7449867129325867,
      "rewards/rejected": -2.3458542823791504,
      "step": 100
    },
    {
      "epoch": 0.22165050465070255,
      "grad_norm": 11.911318908319176,
      "learning_rate": 4.781351221809166e-07,
      "logits/chosen": -1.6439392566680908,
      "logits/rejected": -1.5432502031326294,
      "logps/chosen": -462.12249755859375,
      "logps/rejected": -561.0224609375,
      "loss": 0.563,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.0325913429260254,
      "rewards/margins": 0.8397369384765625,
      "rewards/rejected": -2.872328281402588,
      "step": 105
    },
    {
      "epoch": 0.23220529058645029,
      "grad_norm": 12.017204037189543,
      "learning_rate": 4.742011546497182e-07,
      "logits/chosen": -1.5051389932632446,
      "logits/rejected": -1.4414231777191162,
      "logps/chosen": -449.30657958984375,
      "logps/rejected": -550.5023803710938,
      "loss": 0.5068,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.5960687398910522,
      "rewards/margins": 0.8259360194206238,
      "rewards/rejected": -2.4220046997070312,
      "step": 110
    },
    {
      "epoch": 0.24276007652219803,
      "grad_norm": 16.38172474746581,
      "learning_rate": 4.6996095530953875e-07,
      "logits/chosen": -1.6456964015960693,
      "logits/rejected": -1.4991674423217773,
      "logps/chosen": -492.7826232910156,
      "logps/rejected": -631.8049926757812,
      "loss": 0.5084,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.1061949729919434,
      "rewards/margins": 1.1719015836715698,
      "rewards/rejected": -3.2780966758728027,
      "step": 115
    },
    {
      "epoch": 0.25331486245794577,
      "grad_norm": 19.153944069009537,
      "learning_rate": 4.654203157626399e-07,
      "logits/chosen": -1.7168292999267578,
      "logits/rejected": -1.5979254245758057,
      "logps/chosen": -499.25775146484375,
      "logps/rejected": -663.107177734375,
      "loss": 0.5023,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.420952558517456,
      "rewards/margins": 1.3582748174667358,
      "rewards/rejected": -3.7792270183563232,
      "step": 120
    },
    {
      "epoch": 0.26386964839369353,
      "grad_norm": 15.209663016177657,
      "learning_rate": 4.605854379764673e-07,
      "logits/chosen": -1.529827356338501,
      "logits/rejected": -1.4732040166854858,
      "logps/chosen": -422.29827880859375,
      "logps/rejected": -505.83154296875,
      "loss": 0.4999,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.6309483051300049,
      "rewards/margins": 0.74617999792099,
      "rewards/rejected": -2.3771283626556396,
      "step": 125
    },
    {
      "epoch": 0.27442443432944125,
      "grad_norm": 11.779884218050361,
      "learning_rate": 4.5546292581250857e-07,
      "logits/chosen": -1.5948470830917358,
      "logits/rejected": -1.4933321475982666,
      "logps/chosen": -455.269287109375,
      "logps/rejected": -543.4944458007812,
      "loss": 0.4868,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.7508251667022705,
      "rewards/margins": 0.8269698023796082,
      "rewards/rejected": -2.5777950286865234,
      "step": 130
    },
    {
      "epoch": 0.284979220265189,
      "grad_norm": 19.10994211444829,
      "learning_rate": 4.5005977600621275e-07,
      "logits/chosen": -1.7589390277862549,
      "logits/rejected": -1.5872992277145386,
      "logps/chosen": -568.4318237304688,
      "logps/rejected": -727.857666015625,
      "loss": 0.4781,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.8650763034820557,
      "rewards/margins": 1.3692649602890015,
      "rewards/rejected": -4.234341621398926,
      "step": 135
    },
    {
      "epoch": 0.2955340062009367,
      "grad_norm": 15.492507693472131,
      "learning_rate": 4.443833686102919e-07,
      "logits/chosen": -1.6332670450210571,
      "logits/rejected": -1.4034180641174316,
      "logps/chosen": -461.40826416015625,
      "logps/rejected": -608.6381225585938,
      "loss": 0.4705,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.9456875324249268,
      "rewards/margins": 1.1761558055877686,
      "rewards/rejected": -3.1218433380126953,
      "step": 140
    },
    {
      "epoch": 0.3060887921366845,
      "grad_norm": 16.867870056444893,
      "learning_rate": 4.384414569144561e-07,
      "logits/chosen": -1.5909126996994019,
      "logits/rejected": -1.5102109909057617,
      "logps/chosen": -441.18670654296875,
      "logps/rejected": -565.0006713867188,
      "loss": 0.5053,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.909717321395874,
      "rewards/margins": 1.0274746417999268,
      "rewards/rejected": -2.937191963195801,
      "step": 145
    },
    {
      "epoch": 0.3166435780724322,
      "grad_norm": 12.53246744632567,
      "learning_rate": 4.3224215685535287e-07,
      "logits/chosen": -1.6234986782073975,
      "logits/rejected": -1.4319360256195068,
      "logps/chosen": -540.7374267578125,
      "logps/rejected": -702.7229614257812,
      "loss": 0.4621,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.5035765171051025,
      "rewards/margins": 1.4901249408721924,
      "rewards/rejected": -3.993701457977295,
      "step": 150
    },
    {
      "epoch": 0.32719836400818,
      "grad_norm": 15.04185556896974,
      "learning_rate": 4.2579393593117364e-07,
      "logits/chosen": -1.533140778541565,
      "logits/rejected": -1.3760929107666016,
      "logps/chosen": -481.59954833984375,
      "logps/rejected": -652.427490234375,
      "loss": 0.4563,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.2860851287841797,
      "rewards/margins": 1.5269566774368286,
      "rewards/rejected": -3.8130416870117188,
      "step": 155
    },
    {
      "epoch": 0.3377531499439277,
      "grad_norm": 13.12290477739756,
      "learning_rate": 4.191056016360699e-07,
      "logits/chosen": -1.6271283626556396,
      "logits/rejected": -1.5437796115875244,
      "logps/chosen": -519.0302734375,
      "logps/rejected": -663.0972900390625,
      "loss": 0.4474,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.57169771194458,
      "rewards/margins": 1.3303985595703125,
      "rewards/rejected": -3.9020965099334717,
      "step": 160
    },
    {
      "epoch": 0.34830793587967546,
      "grad_norm": 15.784620667841397,
      "learning_rate": 4.121862894301754e-07,
      "logits/chosen": -1.5655263662338257,
      "logits/rejected": -1.5315742492675781,
      "logps/chosen": -580.234375,
      "logps/rejected": -725.3365478515625,
      "loss": 0.4634,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.9336700439453125,
      "rewards/margins": 1.4053281545639038,
      "rewards/rejected": -4.338997840881348,
      "step": 165
    },
    {
      "epoch": 0.35886272181542317,
      "grad_norm": 18.950750373982537,
      "learning_rate": 4.050454502616667e-07,
      "logits/chosen": -1.7045695781707764,
      "logits/rejected": -1.6517536640167236,
      "logps/chosen": -554.7913208007812,
      "logps/rejected": -716.6380615234375,
      "loss": 0.4487,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.823065757751465,
      "rewards/margins": 1.4535109996795654,
      "rewards/rejected": -4.276576519012451,
      "step": 170
    },
    {
      "epoch": 0.36941750775117094,
      "grad_norm": 17.91495880964947,
      "learning_rate": 3.976928376579047e-07,
      "logits/chosen": -1.6950843334197998,
      "logits/rejected": -1.582262396812439,
      "logps/chosen": -520.1227416992188,
      "logps/rejected": -664.7024536132812,
      "loss": 0.445,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.5456645488739014,
      "rewards/margins": 1.2352759838104248,
      "rewards/rejected": -3.780940532684326,
      "step": 175
    },
    {
      "epoch": 0.37997229368691865,
      "grad_norm": 15.17521580280272,
      "learning_rate": 3.9013849440328945e-07,
      "logits/chosen": -1.6940500736236572,
      "logits/rejected": -1.5467922687530518,
      "logps/chosen": -520.3582763671875,
      "logps/rejected": -680.7635498046875,
      "loss": 0.4665,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.551764726638794,
      "rewards/margins": 1.372143268585205,
      "rewards/rejected": -3.92390775680542,
      "step": 180
    },
    {
      "epoch": 0.3905270796226664,
      "grad_norm": 14.308326454629483,
      "learning_rate": 3.8239273882202473e-07,
      "logits/chosen": -1.6619869470596313,
      "logits/rejected": -1.5419933795928955,
      "logps/chosen": -506.9136657714844,
      "logps/rejected": -639.9857177734375,
      "loss": 0.4574,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -2.3956522941589355,
      "rewards/margins": 1.23929762840271,
      "rewards/rejected": -3.6349494457244873,
      "step": 185
    },
    {
      "epoch": 0.40108186555841413,
      "grad_norm": 15.767200038387838,
      "learning_rate": 3.7446615068452804e-07,
      "logits/chosen": -1.6123485565185547,
      "logits/rejected": -1.4534804821014404,
      "logps/chosen": -510.5772399902344,
      "logps/rejected": -681.8480224609375,
      "loss": 0.4326,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.398505687713623,
      "rewards/margins": 1.5248647928237915,
      "rewards/rejected": -3.923370838165283,
      "step": 190
    },
    {
      "epoch": 0.4116366514941619,
      "grad_norm": 16.282324670439472,
      "learning_rate": 3.6636955675673743e-07,
      "logits/chosen": -1.6264712810516357,
      "logits/rejected": -1.4750279188156128,
      "logps/chosen": -514.038330078125,
      "logps/rejected": -647.3781127929688,
      "loss": 0.4404,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.37794828414917,
      "rewards/margins": 1.2105674743652344,
      "rewards/rejected": -3.5885162353515625,
      "step": 195
    },
    {
      "epoch": 0.4221914374299096,
      "grad_norm": 21.312257244757074,
      "learning_rate": 3.5811401601205093e-07,
      "logits/chosen": -1.5191190242767334,
      "logits/rejected": -1.5369209051132202,
      "logps/chosen": -559.904052734375,
      "logps/rejected": -721.0767211914062,
      "loss": 0.4493,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.823049545288086,
      "rewards/margins": 1.5707520246505737,
      "rewards/rejected": -4.393801689147949,
      "step": 200
    },
    {
      "epoch": 0.4327462233656574,
      "grad_norm": 18.951174802021,
      "learning_rate": 3.497108045260995e-07,
      "logits/chosen": -1.6123138666152954,
      "logits/rejected": -1.5209693908691406,
      "logps/chosen": -508.78436279296875,
      "logps/rejected": -680.7193603515625,
      "loss": 0.4287,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.3978936672210693,
      "rewards/margins": 1.566540002822876,
      "rewards/rejected": -3.9644336700439453,
      "step": 205
    },
    {
      "epoch": 0.4433010093014051,
      "grad_norm": 28.219925216993275,
      "learning_rate": 3.411714000749838e-07,
      "logits/chosen": -1.661116361618042,
      "logits/rejected": -1.5592620372772217,
      "logps/chosen": -547.822998046875,
      "logps/rejected": -737.5689697265625,
      "loss": 0.4248,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.826833724975586,
      "rewards/margins": 1.727709412574768,
      "rewards/rejected": -4.5545430183410645,
      "step": 210
    },
    {
      "epoch": 0.45385579523715286,
      "grad_norm": 27.445717175281757,
      "learning_rate": 3.3250746645801287e-07,
      "logits/chosen": -1.6852436065673828,
      "logits/rejected": -1.6252915859222412,
      "logps/chosen": -603.220947265625,
      "logps/rejected": -795.3065185546875,
      "loss": 0.4409,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.4767067432403564,
      "rewards/margins": 1.7523845434188843,
      "rewards/rejected": -5.229090690612793,
      "step": 215
    },
    {
      "epoch": 0.46441058117290057,
      "grad_norm": 16.007752631407985,
      "learning_rate": 3.237308375663571e-07,
      "logits/chosen": -1.762291669845581,
      "logits/rejected": -1.5295162200927734,
      "logps/chosen": -531.2833862304688,
      "logps/rejected": -725.7946166992188,
      "loss": 0.3819,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.7329602241516113,
      "rewards/margins": 1.7636291980743408,
      "rewards/rejected": -4.496589660644531,
      "step": 220
    },
    {
      "epoch": 0.47496536710864834,
      "grad_norm": 18.506003391111292,
      "learning_rate": 3.148535012193767e-07,
      "logits/chosen": -1.754020094871521,
      "logits/rejected": -1.6287786960601807,
      "logps/chosen": -556.3197631835938,
      "logps/rejected": -728.1083984375,
      "loss": 0.3855,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.739107370376587,
      "rewards/margins": 1.6264499425888062,
      "rewards/rejected": -4.3655571937561035,
      "step": 225
    },
    {
      "epoch": 0.48552015304439605,
      "grad_norm": 20.68896480009483,
      "learning_rate": 3.0588758279070183e-07,
      "logits/chosen": -1.6177418231964111,
      "logits/rejected": -1.5763094425201416,
      "logps/chosen": -562.428955078125,
      "logps/rejected": -753.3512573242188,
      "loss": 0.4092,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.9177470207214355,
      "rewards/margins": 1.7720897197723389,
      "rewards/rejected": -4.6898369789123535,
      "step": 230
    },
    {
      "epoch": 0.4960749389801438,
      "grad_norm": 16.381736251716195,
      "learning_rate": 2.968453286464312e-07,
      "logits/chosen": -1.4933600425720215,
      "logits/rejected": -1.5784003734588623,
      "logps/chosen": -504.9471740722656,
      "logps/rejected": -648.5574951171875,
      "loss": 0.4179,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.378976345062256,
      "rewards/margins": 1.3860208988189697,
      "rewards/rejected": -3.7649970054626465,
      "step": 235
    },
    {
      "epoch": 0.5066297249158915,
      "grad_norm": 22.296205809275865,
      "learning_rate": 2.8773908941806877e-07,
      "logits/chosen": -1.6254231929779053,
      "logits/rejected": -1.621469259262085,
      "logps/chosen": -599.214599609375,
      "logps/rejected": -809.9522705078125,
      "loss": 0.39,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.3130722045898438,
      "rewards/margins": 1.939295768737793,
      "rewards/rejected": -5.2523674964904785,
      "step": 240
    },
    {
      "epoch": 0.5171845108516393,
      "grad_norm": 17.974418608052964,
      "learning_rate": 2.785813031330473e-07,
      "logits/chosen": -1.6836649179458618,
      "logits/rejected": -1.6946824789047241,
      "logps/chosen": -636.5640869140625,
      "logps/rejected": -862.1500244140625,
      "loss": 0.407,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.709088087081909,
      "rewards/margins": 2.079369068145752,
      "rewards/rejected": -5.788456916809082,
      "step": 245
    },
    {
      "epoch": 0.5277392967873871,
      "grad_norm": 21.869860477038394,
      "learning_rate": 2.693844782258779e-07,
      "logits/chosen": -1.597246766090393,
      "logits/rejected": -1.4808999300003052,
      "logps/chosen": -554.4249877929688,
      "logps/rejected": -735.31103515625,
      "loss": 0.377,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.8226776123046875,
      "rewards/margins": 1.673805832862854,
      "rewards/rejected": -4.49648380279541,
      "step": 250
    },
    {
      "epoch": 0.5382940827231347,
      "grad_norm": 35.342954704070486,
      "learning_rate": 2.601611764531342e-07,
      "logits/chosen": -1.599726676940918,
      "logits/rejected": -1.5378262996673584,
      "logps/chosen": -625.1596069335938,
      "logps/rejected": -838.8173828125,
      "loss": 0.3907,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.5563435554504395,
      "rewards/margins": 1.9609460830688477,
      "rewards/rejected": -5.517289638519287,
      "step": 255
    },
    {
      "epoch": 0.5488488686588825,
      "grad_norm": 17.9297074159325,
      "learning_rate": 2.5092399573560323e-07,
      "logits/chosen": -1.6411758661270142,
      "logits/rejected": -1.6522302627563477,
      "logps/chosen": -658.8689575195312,
      "logps/rejected": -871.1637573242188,
      "loss": 0.4168,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.866881847381592,
      "rewards/margins": 2.113030195236206,
      "rewards/rejected": -5.979912757873535,
      "step": 260
    },
    {
      "epoch": 0.5594036545946303,
      "grad_norm": 16.443040045807887,
      "learning_rate": 2.4168555295104124e-07,
      "logits/chosen": -1.5852059125900269,
      "logits/rejected": -1.5837304592132568,
      "logps/chosen": -594.8711547851562,
      "logps/rejected": -801.4606323242188,
      "loss": 0.4093,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.3196072578430176,
      "rewards/margins": 1.8402855396270752,
      "rewards/rejected": -5.159893035888672,
      "step": 265
    },
    {
      "epoch": 0.569958440530378,
      "grad_norm": 15.035798283991243,
      "learning_rate": 2.3245846670103626e-07,
      "logits/chosen": -1.5612332820892334,
      "logits/rejected": -1.5125606060028076,
      "logps/chosen": -579.2890014648438,
      "logps/rejected": -773.1954956054688,
      "loss": 0.3769,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.0682718753814697,
      "rewards/margins": 1.7783126831054688,
      "rewards/rejected": -4.846584320068359,
      "step": 270
    },
    {
      "epoch": 0.5805132264661257,
      "grad_norm": 20.761167206822886,
      "learning_rate": 2.232553400755159e-07,
      "logits/chosen": -1.6236953735351562,
      "logits/rejected": -1.5238358974456787,
      "logps/chosen": -613.8088989257812,
      "logps/rejected": -867.7283325195312,
      "loss": 0.3697,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.327275037765503,
      "rewards/margins": 2.372307538986206,
      "rewards/rejected": -5.699582576751709,
      "step": 275
    },
    {
      "epoch": 0.5910680124018735,
      "grad_norm": 20.416122820074975,
      "learning_rate": 2.1408874343844294e-07,
      "logits/chosen": -1.6903560161590576,
      "logits/rejected": -1.5515328645706177,
      "logps/chosen": -648.6663818359375,
      "logps/rejected": -949.74609375,
      "loss": 0.3701,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.67596435546875,
      "rewards/margins": 2.625474214553833,
      "rewards/rejected": -6.301438808441162,
      "step": 280
    },
    {
      "epoch": 0.6016227983376212,
      "grad_norm": 20.954462877817495,
      "learning_rate": 2.049711972582101e-07,
      "logits/chosen": -1.7400896549224854,
      "logits/rejected": -1.6318342685699463,
      "logps/chosen": -647.7393798828125,
      "logps/rejected": -899.9304809570312,
      "loss": 0.3719,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.6654484272003174,
      "rewards/margins": 2.371953010559082,
      "rewards/rejected": -6.03740119934082,
      "step": 285
    },
    {
      "epoch": 0.612177584273369,
      "grad_norm": 17.14928736018049,
      "learning_rate": 1.9591515500618588e-07,
      "logits/chosen": -1.6016016006469727,
      "logits/rejected": -1.5183677673339844,
      "logps/chosen": -670.8876342773438,
      "logps/rejected": -865.2810668945312,
      "loss": 0.4464,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -3.901280641555786,
      "rewards/margins": 1.737006425857544,
      "rewards/rejected": -5.638287544250488,
      "step": 290
    },
    {
      "epoch": 0.6227323702091166,
      "grad_norm": 15.631817925073218,
      "learning_rate": 1.8693298614677112e-07,
      "logits/chosen": -1.4730761051177979,
      "logits/rejected": -1.3837854862213135,
      "logps/chosen": -539.0037231445312,
      "logps/rejected": -730.4061279296875,
      "loss": 0.3841,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.58305025100708,
      "rewards/margins": 1.7921463251113892,
      "rewards/rejected": -4.37519645690918,
      "step": 295
    },
    {
      "epoch": 0.6332871561448644,
      "grad_norm": 17.780553626895177,
      "learning_rate": 1.7803695924219814e-07,
      "logits/chosen": -1.4669979810714722,
      "logits/rejected": -1.4254872798919678,
      "logps/chosen": -584.629150390625,
      "logps/rejected": -784.3005981445312,
      "loss": 0.4051,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.1204633712768555,
      "rewards/margins": 1.9275195598602295,
      "rewards/rejected": -5.047983169555664,
      "step": 300
    },
    {
      "epoch": 0.6438419420806122,
      "grad_norm": 19.866976638480853,
      "learning_rate": 1.6923922519515067e-07,
      "logits/chosen": -1.4443576335906982,
      "logits/rejected": -1.4355580806732178,
      "logps/chosen": -523.0612182617188,
      "logps/rejected": -729.1703491210938,
      "loss": 0.4069,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.749462842941284,
      "rewards/margins": 1.8047136068344116,
      "rewards/rejected": -4.554176330566406,
      "step": 305
    },
    {
      "epoch": 0.65439672801636,
      "grad_norm": 19.42045257459303,
      "learning_rate": 1.605518006520924e-07,
      "logits/chosen": -1.4894784688949585,
      "logits/rejected": -1.4559067487716675,
      "logps/chosen": -589.7485961914062,
      "logps/rejected": -808.5824584960938,
      "loss": 0.3816,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.1640734672546387,
      "rewards/margins": 2.0186519622802734,
      "rewards/rejected": -5.182725429534912,
      "step": 310
    },
    {
      "epoch": 0.6649515139521076,
      "grad_norm": 34.470860837004935,
      "learning_rate": 1.519865515899731e-07,
      "logits/chosen": -1.5242574214935303,
      "logits/rejected": -1.3928359746932983,
      "logps/chosen": -588.0692138671875,
      "logps/rejected": -841.7396240234375,
      "loss": 0.3858,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.3201167583465576,
      "rewards/margins": 2.1101181507110596,
      "rewards/rejected": -5.430234432220459,
      "step": 315
    },
    {
      "epoch": 0.6755062998878554,
      "grad_norm": 30.007742834102533,
      "learning_rate": 1.4355517710873182e-07,
      "logits/chosen": -1.546661615371704,
      "logits/rejected": -1.4665791988372803,
      "logps/chosen": -627.7368774414062,
      "logps/rejected": -869.39453125,
      "loss": 0.3914,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.6285042762756348,
      "rewards/margins": 2.2051749229431152,
      "rewards/rejected": -5.83367919921875,
      "step": 320
    },
    {
      "epoch": 0.6860610858236031,
      "grad_norm": 34.25630759725382,
      "learning_rate": 1.3526919345173318e-07,
      "logits/chosen": -1.5128138065338135,
      "logits/rejected": -1.4411523342132568,
      "logps/chosen": -602.6571044921875,
      "logps/rejected": -849.2667846679688,
      "loss": 0.4211,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.323326826095581,
      "rewards/margins": 2.333648681640625,
      "rewards/rejected": -5.656975269317627,
      "step": 325
    },
    {
      "epoch": 0.6966158717593509,
      "grad_norm": 28.233731363069086,
      "learning_rate": 1.2713991827596443e-07,
      "logits/chosen": -1.495444655418396,
      "logits/rejected": -1.4902544021606445,
      "logps/chosen": -605.8360595703125,
      "logps/rejected": -796.5411376953125,
      "loss": 0.3791,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.165205478668213,
      "rewards/margins": 1.914367914199829,
      "rewards/rejected": -5.079573631286621,
      "step": 330
    },
    {
      "epoch": 0.7071706576950986,
      "grad_norm": 28.532009996830972,
      "learning_rate": 1.191784551934773e-07,
      "logits/chosen": -1.5311321020126343,
      "logits/rejected": -1.444746971130371,
      "logps/chosen": -515.9117431640625,
      "logps/rejected": -714.9118041992188,
      "loss": 0.3985,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.741084575653076,
      "rewards/margins": 1.8353731632232666,
      "rewards/rejected": -4.5764570236206055,
      "step": 335
    },
    {
      "epoch": 0.7177254436308463,
      "grad_norm": 34.722496076575624,
      "learning_rate": 1.1139567860518953e-07,
      "logits/chosen": -1.3724250793457031,
      "logits/rejected": -1.3445093631744385,
      "logps/chosen": -516.8666381835938,
      "logps/rejected": -690.0352172851562,
      "loss": 0.4182,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.5948445796966553,
      "rewards/margins": 1.631908655166626,
      "rewards/rejected": -4.226753234863281,
      "step": 340
    },
    {
      "epoch": 0.7282802295665941,
      "grad_norm": 29.539595863207374,
      "learning_rate": 1.0380221884776128e-07,
      "logits/chosen": -1.4173920154571533,
      "logits/rejected": -1.3776549100875854,
      "logps/chosen": -542.7964477539062,
      "logps/rejected": -722.7305297851562,
      "loss": 0.4204,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.7417469024658203,
      "rewards/margins": 1.6534277200698853,
      "rewards/rejected": -4.395174980163574,
      "step": 345
    },
    {
      "epoch": 0.7388350155023419,
      "grad_norm": 17.47998143760053,
      "learning_rate": 9.640844767383405e-08,
      "logits/chosen": -1.3427600860595703,
      "logits/rejected": -1.2479599714279175,
      "logps/chosen": -540.9366455078125,
      "logps/rejected": -741.2656860351562,
      "loss": 0.4206,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.7238881587982178,
      "rewards/margins": 1.8479011058807373,
      "rewards/rejected": -4.571789741516113,
      "step": 350
    },
    {
      "epoch": 0.7493898014380895,
      "grad_norm": 22.53665974937468,
      "learning_rate": 8.922446408546378e-08,
      "logits/chosen": -1.3614610433578491,
      "logits/rejected": -1.3304545879364014,
      "logps/chosen": -564.0736694335938,
      "logps/rejected": -796.0197143554688,
      "loss": 0.4234,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.8989734649658203,
      "rewards/margins": 2.1152186393737793,
      "rewards/rejected": -5.0141921043396,
      "step": 355
    },
    {
      "epoch": 0.7599445873738373,
      "grad_norm": 34.016623930856696,
      "learning_rate": 8.22600805400994e-08,
      "logits/chosen": -1.4026951789855957,
      "logits/rejected": -1.3086416721343994,
      "logps/chosen": -557.1563720703125,
      "logps/rejected": -775.2821044921875,
      "loss": 0.3918,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.755949020385742,
      "rewards/margins": 1.9373924732208252,
      "rewards/rejected": -4.6933417320251465,
      "step": 360
    },
    {
      "epoch": 0.7704993733095851,
      "grad_norm": 19.583973928513963,
      "learning_rate": 7.552480954794558e-08,
      "logits/chosen": -1.4313266277313232,
      "logits/rejected": -1.3103513717651367,
      "logps/chosen": -564.1690673828125,
      "logps/rejected": -756.8692626953125,
      "loss": 0.4048,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.88138484954834,
      "rewards/margins": 1.8533366918563843,
      "rewards/rejected": -4.7347211837768555,
      "step": 365
    },
    {
      "epoch": 0.7810541592453328,
      "grad_norm": 18.206536925706505,
      "learning_rate": 6.902785067901854e-08,
      "logits/chosen": -1.3645613193511963,
      "logits/rejected": -1.3341350555419922,
      "logps/chosen": -579.579833984375,
      "logps/rejected": -779.4511108398438,
      "loss": 0.3792,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.0597000122070312,
      "rewards/margins": 1.8546708822250366,
      "rewards/rejected": -4.914370536804199,
      "step": 370
    },
    {
      "epoch": 0.7916089451810805,
      "grad_norm": 25.99340540536939,
      "learning_rate": 6.277807799763973e-08,
      "logits/chosen": -1.5334607362747192,
      "logits/rejected": -1.4499049186706543,
      "logps/chosen": -563.6980590820312,
      "logps/rejected": -789.72119140625,
      "loss": 0.3903,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.1820693016052246,
      "rewards/margins": 1.9633289575576782,
      "rewards/rejected": -5.145398139953613,
      "step": 375
    },
    {
      "epoch": 0.8021637311168283,
      "grad_norm": 27.363349380647865,
      "learning_rate": 5.678402794153145e-08,
      "logits/chosen": -1.490678071975708,
      "logits/rejected": -1.4118311405181885,
      "logps/chosen": -635.3906860351562,
      "logps/rejected": -841.986328125,
      "loss": 0.3978,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.6473312377929688,
      "rewards/margins": 1.8795642852783203,
      "rewards/rejected": -5.526895523071289,
      "step": 380
    },
    {
      "epoch": 0.812718517052576,
      "grad_norm": 22.024184023637442,
      "learning_rate": 5.105388766206969e-08,
      "logits/chosen": -1.6372134685516357,
      "logits/rejected": -1.4456651210784912,
      "logps/chosen": -670.1497802734375,
      "logps/rejected": -900.1561279296875,
      "loss": 0.4107,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.8800835609436035,
      "rewards/margins": 2.1356186866760254,
      "rewards/rejected": -6.015702724456787,
      "step": 385
    },
    {
      "epoch": 0.8232733029883238,
      "grad_norm": 18.957756630133876,
      "learning_rate": 4.5595483841620484e-08,
      "logits/chosen": -1.4949450492858887,
      "logits/rejected": -1.466933250427246,
      "logps/chosen": -592.1412963867188,
      "logps/rejected": -814.1785888671875,
      "loss": 0.3793,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.3024775981903076,
      "rewards/margins": 1.9800523519515991,
      "rewards/rejected": -5.282530307769775,
      "step": 390
    },
    {
      "epoch": 0.8338280889240715,
      "grad_norm": 24.130033467348618,
      "learning_rate": 4.0416272003232526e-08,
      "logits/chosen": -1.5286778211593628,
      "logits/rejected": -1.4018694162368774,
      "logps/chosen": -614.2500610351562,
      "logps/rejected": -848.65673828125,
      "loss": 0.4233,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.3793911933898926,
      "rewards/margins": 2.2501933574676514,
      "rewards/rejected": -5.629584312438965,
      "step": 395
    },
    {
      "epoch": 0.8443828748598192,
      "grad_norm": 21.924780448040742,
      "learning_rate": 3.552332632729041e-08,
      "logits/chosen": -1.3315622806549072,
      "logits/rejected": -1.4189374446868896,
      "logps/chosen": -591.8238525390625,
      "logps/rejected": -745.0551147460938,
      "loss": 0.4269,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.1886703968048096,
      "rewards/margins": 1.4928715229034424,
      "rewards/rejected": -4.68154239654541,
      "step": 400
    },
    {
      "epoch": 0.8443828748598192,
      "eval_logits/chosen": -1.3921160697937012,
      "eval_logits/rejected": -1.360321283340454,
      "eval_logps/chosen": -602.09814453125,
      "eval_logps/rejected": -797.6878051757812,
      "eval_loss": 0.38372838497161865,
      "eval_rewards/accuracies": 0.8643724918365479,
      "eval_rewards/chosen": -3.2510859966278076,
      "eval_rewards/margins": 1.8691294193267822,
      "eval_rewards/rejected": -5.120214939117432,
      "eval_runtime": 313.5439,
      "eval_samples_per_second": 6.302,
      "eval_steps_per_second": 1.576,
      "step": 400
    },
    {
      "epoch": 0.854937660795567,
      "grad_norm": 19.316578134763,
      "learning_rate": 3.092332998903416e-08,
      "logits/chosen": -1.4394410848617554,
      "logits/rejected": -1.3752410411834717,
      "logps/chosen": -601.6661987304688,
      "logps/rejected": -783.7593994140625,
      "loss": 0.3863,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -3.3254780769348145,
      "rewards/margins": 1.7059745788574219,
      "rewards/rejected": -5.031452655792236,
      "step": 405
    },
    {
      "epoch": 0.8654924467313148,
      "grad_norm": 28.757209433347523,
      "learning_rate": 2.6622566030146455e-08,
      "logits/chosen": -1.4320390224456787,
      "logits/rejected": -1.413570523262024,
      "logps/chosen": -565.2530517578125,
      "logps/rejected": -754.3922729492188,
      "loss": 0.4213,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.987544298171997,
      "rewards/margins": 1.7588831186294556,
      "rewards/rejected": -4.7464280128479,
      "step": 410
    },
    {
      "epoch": 0.8760472326670625,
      "grad_norm": 18.43535305501894,
      "learning_rate": 2.26269087768734e-08,
      "logits/chosen": -1.4583203792572021,
      "logits/rejected": -1.3761074542999268,
      "logps/chosen": -577.720703125,
      "logps/rejected": -806.8515014648438,
      "loss": 0.3829,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.2120673656463623,
      "rewards/margins": 2.084867477416992,
      "rewards/rejected": -5.296935081481934,
      "step": 415
    },
    {
      "epoch": 0.8866020186028102,
      "grad_norm": 20.96371705639774,
      "learning_rate": 1.894181581640106e-08,
      "logits/chosen": -1.384445071220398,
      "logits/rejected": -1.4241831302642822,
      "logps/chosen": -569.5962524414062,
      "logps/rejected": -749.0470581054688,
      "loss": 0.3819,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.081413745880127,
      "rewards/margins": 1.7126514911651611,
      "rewards/rejected": -4.794064998626709,
      "step": 420
    },
    {
      "epoch": 0.897156804538558,
      "grad_norm": 26.21197116684321,
      "learning_rate": 1.5572320542448143e-08,
      "logits/chosen": -1.3703296184539795,
      "logits/rejected": -1.3327410221099854,
      "logps/chosen": -617.3011474609375,
      "logps/rejected": -836.2391357421875,
      "loss": 0.3825,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.3691182136535645,
      "rewards/margins": 2.073474407196045,
      "rewards/rejected": -5.442592620849609,
      "step": 425
    },
    {
      "epoch": 0.9077115904743057,
      "grad_norm": 22.75331582162892,
      "learning_rate": 1.2523025280255729e-08,
      "logits/chosen": -1.4123306274414062,
      "logits/rejected": -1.4244548082351685,
      "logps/chosen": -593.9744262695312,
      "logps/rejected": -802.8662719726562,
      "loss": 0.3422,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.253312349319458,
      "rewards/margins": 1.9519180059432983,
      "rewards/rejected": -5.205230712890625,
      "step": 430
    },
    {
      "epoch": 0.9182663764100535,
      "grad_norm": 17.924586145441417,
      "learning_rate": 9.798095000364214e-09,
      "logits/chosen": -1.5641670227050781,
      "logits/rejected": -1.4084519147872925,
      "logps/chosen": -600.7659912109375,
      "logps/rejected": -869.5779418945312,
      "loss": 0.3553,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.2969956398010254,
      "rewards/margins": 2.371945381164551,
      "rewards/rejected": -5.668941020965576,
      "step": 435
    },
    {
      "epoch": 0.9288211623458011,
      "grad_norm": 25.268001512531786,
      "learning_rate": 7.401251629764876e-09,
      "logits/chosen": -1.5503554344177246,
      "logits/rejected": -1.3530725240707397,
      "logps/chosen": -560.983154296875,
      "logps/rejected": -772.5062255859375,
      "loss": 0.4149,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -3.1528074741363525,
      "rewards/margins": 1.8247524499893188,
      "rewards/rejected": -4.977559566497803,
      "step": 440
    },
    {
      "epoch": 0.9393759482815489,
      "grad_norm": 18.848350951492684,
      "learning_rate": 5.335768968195098e-09,
      "logits/chosen": -1.4744082689285278,
      "logits/rejected": -1.370416283607483,
      "logps/chosen": -589.2518920898438,
      "logps/rejected": -806.16943359375,
      "loss": 0.4069,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.1618690490722656,
      "rewards/margins": 2.018306255340576,
      "rewards/rejected": -5.180174827575684,
      "step": 445
    },
    {
      "epoch": 0.9499307342172967,
      "grad_norm": 14.44527353219914,
      "learning_rate": 3.604468216521883e-09,
      "logits/chosen": -1.3630057573318481,
      "logits/rejected": -1.2414896488189697,
      "logps/chosen": -591.0068359375,
      "logps/rejected": -788.7886352539062,
      "loss": 0.371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.234240770339966,
      "rewards/margins": 1.8283990621566772,
      "rewards/rejected": -5.0626397132873535,
      "step": 450
    },
    {
      "epoch": 0.9604855201530444,
      "grad_norm": 17.894278742200918,
      "learning_rate": 2.2097141233206884e-09,
      "logits/chosen": -1.5407589673995972,
      "logits/rejected": -1.4432313442230225,
      "logps/chosen": -616.9603271484375,
      "logps/rejected": -858.2979736328125,
      "loss": 0.3764,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.5265698432922363,
      "rewards/margins": 2.2084403038024902,
      "rewards/rejected": -5.735010623931885,
      "step": 455
    },
    {
      "epoch": 0.9710403060887921,
      "grad_norm": 18.751457793725724,
      "learning_rate": 1.1534117549133472e-09,
      "logits/chosen": -1.46907639503479,
      "logits/rejected": -1.3426740169525146,
      "logps/chosen": -629.7129516601562,
      "logps/rejected": -850.1477661132812,
      "loss": 0.3512,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.58038330078125,
      "rewards/margins": 2.005390167236328,
      "rewards/rejected": -5.585773468017578,
      "step": 460
    },
    {
      "epoch": 0.9815950920245399,
      "grad_norm": 29.39806840137654,
      "learning_rate": 4.3700389327672173e-10,
      "logits/chosen": -1.469868779182434,
      "logits/rejected": -1.3952000141143799,
      "logps/chosen": -604.9054565429688,
      "logps/rejected": -832.4953002929688,
      "loss": 0.3768,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.387165069580078,
      "rewards/margins": 2.0868310928344727,
      "rewards/rejected": -5.473996162414551,
      "step": 465
    },
    {
      "epoch": 0.9921498779602876,
      "grad_norm": 22.359148428267968,
      "learning_rate": 6.146906537587982e-11,
      "logits/chosen": -1.4432518482208252,
      "logits/rejected": -1.4169013500213623,
      "logps/chosen": -600.5186157226562,
      "logps/rejected": -811.341796875,
      "loss": 0.3939,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -3.3816261291503906,
      "rewards/margins": 2.0300629138946533,
      "rewards/rejected": -5.411688804626465,
      "step": 470
    },
    {
      "epoch": 0.9984827495217362,
      "step": 473,
      "total_flos": 0.0,
      "train_loss": 0.4645486564767285,
      "train_runtime": 24049.7915,
      "train_samples_per_second": 2.521,
      "train_steps_per_second": 0.02
    }
  ],
  "logging_steps": 5,
  "max_steps": 473,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}