{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9994767137624281,
  "eval_steps": 100,
  "global_step": 955,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0010465724751439038,
      "grad_norm": 127.36319300622706,
      "learning_rate": 5.208333333333333e-09,
      "logits/chosen": -2.9235777854919434,
      "logits/rejected": -2.7912116050720215,
      "logps/chosen": -380.68548583984375,
      "logps/rejected": -358.4158630371094,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.010465724751439037,
      "grad_norm": 107.9740860657244,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -2.595984935760498,
      "logits/rejected": -2.56972074508667,
      "logps/chosen": -256.59259033203125,
      "logps/rejected": -234.86553955078125,
      "loss": 0.6936,
      "rewards/accuracies": 0.4513888955116272,
      "rewards/chosen": 0.00012609982513822615,
      "rewards/margins": -5.737816172768362e-05,
      "rewards/rejected": 0.00018347776494920254,
      "step": 10
    },
    {
      "epoch": 0.020931449502878074,
      "grad_norm": 123.40563647833262,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.6136789321899414,
      "logits/rejected": -2.5759799480438232,
      "logps/chosen": -283.09326171875,
      "logps/rejected": -282.3014221191406,
      "loss": 0.6899,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.0011462386464700103,
      "rewards/margins": 0.004408253822475672,
      "rewards/rejected": -0.0032620157580822706,
      "step": 20
    },
    {
      "epoch": 0.03139717425431711,
      "grad_norm": 121.3818362170214,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -2.6919732093811035,
      "logits/rejected": -2.667656660079956,
      "logps/chosen": -270.1833801269531,
      "logps/rejected": -276.7445068359375,
      "loss": 0.683,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.01242474652826786,
      "rewards/margins": 0.02042328752577305,
      "rewards/rejected": -0.007998539134860039,
      "step": 30
    },
    {
      "epoch": 0.04186289900575615,
      "grad_norm": 115.66860622860622,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.665289878845215,
      "logits/rejected": -2.5902962684631348,
      "logps/chosen": -290.4902648925781,
      "logps/rejected": -282.1328125,
      "loss": 0.6686,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.04880904406309128,
      "rewards/margins": 0.05465905740857124,
      "rewards/rejected": -0.005850008223205805,
      "step": 40
    },
    {
      "epoch": 0.052328623757195186,
      "grad_norm": 113.05355403051037,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -2.672785997390747,
      "logits/rejected": -2.589231014251709,
      "logps/chosen": -266.2040100097656,
      "logps/rejected": -236.48739624023438,
      "loss": 0.641,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.14585763216018677,
      "rewards/margins": 0.13357332348823547,
      "rewards/rejected": 0.012284321710467339,
      "step": 50
    },
    {
      "epoch": 0.06279434850863422,
      "grad_norm": 106.6001336411059,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.6317365169525146,
      "logits/rejected": -2.596348285675049,
      "logps/chosen": -300.1541442871094,
      "logps/rejected": -274.40191650390625,
      "loss": 0.6134,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.3813043236732483,
      "rewards/margins": 0.24904949963092804,
      "rewards/rejected": 0.13225486874580383,
      "step": 60
    },
    {
      "epoch": 0.07326007326007326,
      "grad_norm": 105.44232045239558,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -2.5406157970428467,
      "logits/rejected": -2.5337507724761963,
      "logps/chosen": -256.82513427734375,
      "logps/rejected": -259.8224792480469,
      "loss": 0.5958,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.2853667438030243,
      "rewards/margins": 0.44680237770080566,
      "rewards/rejected": -0.16143563389778137,
      "step": 70
    },
    {
      "epoch": 0.0837257980115123,
      "grad_norm": 118.11259244631633,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.583888292312622,
      "logits/rejected": -2.516160249710083,
      "logps/chosen": -271.23541259765625,
      "logps/rejected": -252.84814453125,
      "loss": 0.5843,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.4816661775112152,
      "rewards/margins": 0.6344618797302246,
      "rewards/rejected": -0.15279565751552582,
      "step": 80
    },
    {
      "epoch": 0.09419152276295134,
      "grad_norm": 90.59677124687741,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -2.549140691757202,
      "logits/rejected": -2.518716335296631,
      "logps/chosen": -249.97048950195312,
      "logps/rejected": -263.16864013671875,
      "loss": 0.5806,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.5636419057846069,
      "rewards/margins": 0.4112882614135742,
      "rewards/rejected": 0.1523536890745163,
      "step": 90
    },
    {
      "epoch": 0.10465724751439037,
      "grad_norm": 89.93368824706333,
      "learning_rate": 4.999732492681437e-07,
      "logits/chosen": -2.5437731742858887,
      "logits/rejected": -2.4748952388763428,
      "logps/chosen": -290.6165771484375,
      "logps/rejected": -277.58087158203125,
      "loss": 0.5446,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.9238502383232117,
      "rewards/margins": 0.6838744878768921,
      "rewards/rejected": 0.23997588455677032,
      "step": 100
    },
    {
      "epoch": 0.10465724751439037,
      "eval_logits/chosen": -2.5742745399475098,
      "eval_logits/rejected": -2.516094923019409,
      "eval_logps/chosen": -271.84344482421875,
      "eval_logps/rejected": -256.6897888183594,
      "eval_loss": 0.5752763748168945,
      "eval_rewards/accuracies": 0.7242063283920288,
      "eval_rewards/chosen": 1.0110565423965454,
      "eval_rewards/margins": 0.6581434011459351,
      "eval_rewards/rejected": 0.35291311144828796,
      "eval_runtime": 206.982,
      "eval_samples_per_second": 9.663,
      "eval_steps_per_second": 0.304,
      "step": 100
    },
    {
      "epoch": 0.1151229722658294,
      "grad_norm": 91.21630112817637,
      "learning_rate": 4.996723692767926e-07,
      "logits/chosen": -2.5750699043273926,
      "logits/rejected": -2.528003215789795,
      "logps/chosen": -234.3852081298828,
      "logps/rejected": -236.4423370361328,
      "loss": 0.5648,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.705768346786499,
      "rewards/margins": 0.6249028444290161,
      "rewards/rejected": 0.08086539804935455,
      "step": 110
    },
    {
      "epoch": 0.12558869701726844,
      "grad_norm": 106.85996872671201,
      "learning_rate": 4.990375746213598e-07,
      "logits/chosen": -2.524847984313965,
      "logits/rejected": -2.4740397930145264,
      "logps/chosen": -258.1869201660156,
      "logps/rejected": -292.03643798828125,
      "loss": 0.5378,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.43742185831069946,
      "rewards/margins": 0.7921646237373352,
      "rewards/rejected": -0.3547428548336029,
      "step": 120
    },
    {
      "epoch": 0.1360544217687075,
      "grad_norm": 113.00637141782764,
      "learning_rate": 4.980697142834314e-07,
      "logits/chosen": -2.520475387573242,
      "logits/rejected": -2.4845001697540283,
      "logps/chosen": -278.5224304199219,
      "logps/rejected": -261.1920166015625,
      "loss": 0.5472,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.26921743154525757,
      "rewards/margins": 0.9498673677444458,
      "rewards/rejected": -0.6806498765945435,
      "step": 130
    },
    {
      "epoch": 0.14652014652014653,
      "grad_norm": 98.25615957326066,
      "learning_rate": 4.967700826904229e-07,
      "logits/chosen": -2.4557089805603027,
      "logits/rejected": -2.4235308170318604,
      "logps/chosen": -284.69012451171875,
      "logps/rejected": -292.3227233886719,
      "loss": 0.5192,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.6424055099487305,
      "rewards/margins": 0.9809519648551941,
      "rewards/rejected": -0.3385465741157532,
      "step": 140
    },
    {
      "epoch": 0.15698587127158556,
      "grad_norm": 99.56093992482657,
      "learning_rate": 4.951404179843962e-07,
      "logits/chosen": -2.601367473602295,
      "logits/rejected": -2.5492353439331055,
      "logps/chosen": -286.1108093261719,
      "logps/rejected": -252.67074584960938,
      "loss": 0.5511,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.17216625809669495,
      "rewards/margins": 0.7409141659736633,
      "rewards/rejected": -0.568747878074646,
      "step": 150
    },
    {
      "epoch": 0.1674515960230246,
      "grad_norm": 115.6693954933169,
      "learning_rate": 4.931828996974498e-07,
      "logits/chosen": -2.568312883377075,
      "logits/rejected": -2.4805312156677246,
      "logps/chosen": -291.2640380859375,
      "logps/rejected": -259.77508544921875,
      "loss": 0.5535,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.4559265077114105,
      "rewards/margins": 0.8324508666992188,
      "rewards/rejected": -0.3765243887901306,
      "step": 160
    },
    {
      "epoch": 0.17791732077446362,
      "grad_norm": 108.75023514338689,
      "learning_rate": 4.909001458367866e-07,
      "logits/chosen": -2.556863784790039,
      "logits/rejected": -2.4760687351226807,
      "logps/chosen": -274.7050476074219,
      "logps/rejected": -246.76708984375,
      "loss": 0.5534,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.37477999925613403,
      "rewards/margins": 0.9715734720230103,
      "rewards/rejected": -0.5967934727668762,
      "step": 170
    },
    {
      "epoch": 0.18838304552590268,
      "grad_norm": 99.91541524541664,
      "learning_rate": 4.882952093833627e-07,
      "logits/chosen": -2.543196201324463,
      "logits/rejected": -2.501505136489868,
      "logps/chosen": -242.51748657226562,
      "logps/rejected": -259.04241943359375,
      "loss": 0.5136,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.2591323256492615,
      "rewards/margins": 0.9666780233383179,
      "rewards/rejected": -1.2258104085922241,
      "step": 180
    },
    {
      "epoch": 0.1988487702773417,
      "grad_norm": 115.88419375588242,
      "learning_rate": 4.853715742087946e-07,
      "logits/chosen": -2.4405717849731445,
      "logits/rejected": -2.4257616996765137,
      "logps/chosen": -240.6494140625,
      "logps/rejected": -264.52215576171875,
      "loss": 0.5006,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.46298712491989136,
      "rewards/margins": 0.9989321827888489,
      "rewards/rejected": -1.4619193077087402,
      "step": 190
    },
    {
      "epoch": 0.20931449502878074,
      "grad_norm": 105.94881559115358,
      "learning_rate": 4.821331504159906e-07,
      "logits/chosen": -2.5927906036376953,
      "logits/rejected": -2.517674684524536,
      "logps/chosen": -309.7959289550781,
      "logps/rejected": -265.2268371582031,
      "loss": 0.5475,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.08473672717809677,
      "rewards/margins": 0.8970876932144165,
      "rewards/rejected": -0.812350869178772,
      "step": 200
    },
    {
      "epoch": 0.20931449502878074,
      "eval_logits/chosen": -2.5923333168029785,
      "eval_logits/rejected": -2.537993907928467,
      "eval_logps/chosen": -277.6067810058594,
      "eval_logps/rejected": -265.0432434082031,
      "eval_loss": 0.5464460849761963,
      "eval_rewards/accuracies": 0.7638888955116272,
      "eval_rewards/chosen": 0.4347245991230011,
      "eval_rewards/margins": 0.9171593189239502,
      "eval_rewards/rejected": -0.48243483901023865,
      "eval_runtime": 206.9145,
      "eval_samples_per_second": 9.666,
      "eval_steps_per_second": 0.304,
      "step": 200
    },
    {
      "epoch": 0.21978021978021978,
      "grad_norm": 118.18724049066705,
      "learning_rate": 4.785842691097342e-07,
      "logits/chosen": -2.5883002281188965,
      "logits/rejected": -2.524261951446533,
      "logps/chosen": -278.22625732421875,
      "logps/rejected": -275.1640625,
      "loss": 0.5643,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.45807909965515137,
      "rewards/margins": 0.7460596561431885,
      "rewards/rejected": -0.2879805266857147,
      "step": 210
    },
    {
      "epoch": 0.2302459445316588,
      "grad_norm": 156.31813233608617,
      "learning_rate": 4.7472967660421603e-07,
      "logits/chosen": -2.5972952842712402,
      "logits/rejected": -2.5259041786193848,
      "logps/chosen": -284.1155700683594,
      "logps/rejected": -242.37039184570312,
      "loss": 0.5191,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.30692246556282043,
      "rewards/margins": 0.9284420013427734,
      "rewards/rejected": -0.6215195655822754,
      "step": 220
    },
    {
      "epoch": 0.24071166928309787,
      "grad_norm": 103.00603061075961,
      "learning_rate": 4.705745280752585e-07,
      "logits/chosen": -2.527318000793457,
      "logits/rejected": -2.505685329437256,
      "logps/chosen": -287.8115539550781,
      "logps/rejected": -275.6258850097656,
      "loss": 0.5225,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.09247669577598572,
      "rewards/margins": 0.7993501424789429,
      "rewards/rejected": -0.7068735361099243,
      "step": 230
    },
    {
      "epoch": 0.25117739403453687,
      "grad_norm": 187.82863477728128,
      "learning_rate": 4.6612438066572555e-07,
      "logits/chosen": -2.536621570587158,
      "logits/rejected": -2.525778293609619,
      "logps/chosen": -265.84332275390625,
      "logps/rejected": -262.84210205078125,
      "loss": 0.517,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.17310741543769836,
      "rewards/margins": 1.0395452976226807,
      "rewards/rejected": -0.8664379119873047,
      "step": 240
    },
    {
      "epoch": 0.2616431187859759,
      "grad_norm": 98.47328118938334,
      "learning_rate": 4.6138518605333664e-07,
      "logits/chosen": -2.5257980823516846,
      "logits/rejected": -2.50608229637146,
      "logps/chosen": -275.45831298828125,
      "logps/rejected": -265.85247802734375,
      "loss": 0.4916,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.23112091422080994,
      "rewards/margins": 1.057886004447937,
      "rewards/rejected": -0.8267651796340942,
      "step": 250
    },
    {
      "epoch": 0.272108843537415,
      "grad_norm": 132.81269847647957,
      "learning_rate": 4.5636328249082514e-07,
      "logits/chosen": -2.6113829612731934,
      "logits/rejected": -2.5672502517700195,
      "logps/chosen": -288.1708068847656,
      "logps/rejected": -290.2517395019531,
      "loss": 0.5332,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.08042006194591522,
      "rewards/margins": 0.8779838681221008,
      "rewards/rejected": -0.7975638508796692,
      "step": 260
    },
    {
      "epoch": 0.282574568288854,
      "grad_norm": 109.69739978094053,
      "learning_rate": 4.510653863290871e-07,
      "logits/chosen": -2.5312352180480957,
      "logits/rejected": -2.514791965484619,
      "logps/chosen": -294.85821533203125,
      "logps/rejected": -259.7037658691406,
      "loss": 0.4994,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.4837077260017395,
      "rewards/margins": 1.097794771194458,
      "rewards/rejected": -0.614086925983429,
      "step": 270
    },
    {
      "epoch": 0.29304029304029305,
      "grad_norm": 112.60908808433207,
      "learning_rate": 4.4549858303465737e-07,
      "logits/chosen": -2.577178478240967,
      "logits/rejected": -2.527773380279541,
      "logps/chosen": -293.6539001464844,
      "logps/rejected": -294.2825927734375,
      "loss": 0.5343,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.25686317682266235,
      "rewards/margins": 1.3584675788879395,
      "rewards/rejected": -1.1016044616699219,
      "step": 280
    },
    {
      "epoch": 0.3035060177917321,
      "grad_norm": 136.22152391063145,
      "learning_rate": 4.396703177135261e-07,
      "logits/chosen": -2.5472209453582764,
      "logits/rejected": -2.470147132873535,
      "logps/chosen": -272.1517028808594,
      "logps/rejected": -246.69384765625,
      "loss": 0.5047,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.06049398332834244,
      "rewards/margins": 1.0268157720565796,
      "rewards/rejected": -0.9663218259811401,
      "step": 290
    },
    {
      "epoch": 0.3139717425431711,
      "grad_norm": 103.27642265277515,
      "learning_rate": 4.335883851539693e-07,
      "logits/chosen": -2.571641206741333,
      "logits/rejected": -2.5657002925872803,
      "logps/chosen": -269.2393493652344,
      "logps/rejected": -277.0874938964844,
      "loss": 0.5359,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.024735305458307266,
      "rewards/margins": 0.9094675183296204,
      "rewards/rejected": -0.9342027902603149,
      "step": 300
    },
    {
      "epoch": 0.3139717425431711,
      "eval_logits/chosen": -2.559617757797241,
      "eval_logits/rejected": -2.5065526962280273,
      "eval_logps/chosen": -281.25714111328125,
      "eval_logps/rejected": -270.3889465332031,
      "eval_loss": 0.5473023653030396,
      "eval_rewards/accuracies": 0.7579365372657776,
      "eval_rewards/chosen": 0.0696859061717987,
      "eval_rewards/margins": 1.0866899490356445,
      "eval_rewards/rejected": -1.017004132270813,
      "eval_runtime": 206.5677,
      "eval_samples_per_second": 9.682,
      "eval_steps_per_second": 0.305,
      "step": 300
    },
    {
      "epoch": 0.32443746729461015,
      "grad_norm": 113.48472631776741,
      "learning_rate": 4.272609194017105e-07,
      "logits/chosen": -2.5980334281921387,
      "logits/rejected": -2.5018410682678223,
      "logps/chosen": -284.5079650878906,
      "logps/rejected": -245.64804077148438,
      "loss": 0.5341,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07721372693777084,
      "rewards/margins": 1.208656907081604,
      "rewards/rejected": -1.2858705520629883,
      "step": 310
    },
    {
      "epoch": 0.3349031920460492,
      "grad_norm": 94.90226304989493,
      "learning_rate": 4.2069638288135547e-07,
      "logits/chosen": -2.53539776802063,
      "logits/rejected": -2.496711254119873,
      "logps/chosen": -254.16464233398438,
      "logps/rejected": -263.13336181640625,
      "loss": 0.5831,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.1118149608373642,
      "rewards/margins": 1.1433535814285278,
      "rewards/rejected": -1.2551685571670532,
      "step": 320
    },
    {
      "epoch": 0.3453689167974882,
      "grad_norm": 111.02501519646566,
      "learning_rate": 4.139035550786494e-07,
      "logits/chosen": -2.503291606903076,
      "logits/rejected": -2.466763973236084,
      "logps/chosen": -260.0853271484375,
      "logps/rejected": -246.05831909179688,
      "loss": 0.5226,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3735058307647705,
      "rewards/margins": 1.021803379058838,
      "rewards/rejected": -1.395309329032898,
      "step": 330
    },
    {
      "epoch": 0.35583464154892724,
      "grad_norm": 120.37840185839579,
      "learning_rate": 4.0689152079869306e-07,
      "logits/chosen": -2.419663667678833,
      "logits/rejected": -2.3767175674438477,
      "logps/chosen": -270.08642578125,
      "logps/rejected": -246.24227905273438,
      "loss": 0.6048,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.054160237312316895,
      "rewards/margins": 0.966072678565979,
      "rewards/rejected": -1.0202327966690063,
      "step": 340
    },
    {
      "epoch": 0.3663003663003663,
      "grad_norm": 135.62587519571315,
      "learning_rate": 3.99669658015821e-07,
      "logits/chosen": -2.512122631072998,
      "logits/rejected": -2.480046510696411,
      "logps/chosen": -289.42449951171875,
      "logps/rejected": -262.649169921875,
      "loss": 0.537,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.17944186925888062,
      "rewards/margins": 0.8316653370857239,
      "rewards/rejected": -0.6522234678268433,
      "step": 350
    },
    {
      "epoch": 0.37676609105180536,
      "grad_norm": 95.8864031595669,
      "learning_rate": 3.92247625331392e-07,
      "logits/chosen": -2.4578399658203125,
      "logits/rejected": -2.4470133781433105,
      "logps/chosen": -281.417236328125,
      "logps/rejected": -271.0404357910156,
      "loss": 0.4816,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.4489572048187256,
      "rewards/margins": 1.2753773927688599,
      "rewards/rejected": -0.8264200091362,
      "step": 360
    },
    {
      "epoch": 0.3872318158032444,
      "grad_norm": 100.50656645709512,
      "learning_rate": 3.846353490562664e-07,
      "logits/chosen": -2.5093467235565186,
      "logits/rejected": -2.4688096046447754,
      "logps/chosen": -260.3323669433594,
      "logps/rejected": -242.1940155029297,
      "loss": 0.5303,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.2971992492675781,
      "rewards/margins": 0.9509322047233582,
      "rewards/rejected": -0.65373295545578,
      "step": 370
    },
    {
      "epoch": 0.3976975405546834,
      "grad_norm": 106.58256559416274,
      "learning_rate": 3.768430099352445e-07,
      "logits/chosen": -2.487880229949951,
      "logits/rejected": -2.454012870788574,
      "logps/chosen": -255.39425659179688,
      "logps/rejected": -257.49078369140625,
      "loss": 0.5166,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07468878477811813,
      "rewards/margins": 1.0435364246368408,
      "rewards/rejected": -1.118225336074829,
      "step": 380
    },
    {
      "epoch": 0.40816326530612246,
      "grad_norm": 101.08281656389065,
      "learning_rate": 3.6888102953122304e-07,
      "logits/chosen": -2.5439352989196777,
      "logits/rejected": -2.506777286529541,
      "logps/chosen": -287.0313720703125,
      "logps/rejected": -284.038818359375,
      "loss": 0.5404,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.5431143045425415,
      "rewards/margins": 1.0249578952789307,
      "rewards/rejected": -1.5680720806121826,
      "step": 390
    },
    {
      "epoch": 0.4186289900575615,
      "grad_norm": 295.883680569105,
      "learning_rate": 3.607600562872785e-07,
      "logits/chosen": -2.568805456161499,
      "logits/rejected": -2.5805678367614746,
      "logps/chosen": -274.0014953613281,
      "logps/rejected": -281.6232604980469,
      "loss": 0.5228,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.4060361385345459,
      "rewards/margins": 0.9878374338150024,
      "rewards/rejected": -1.3938738107681274,
      "step": 400
    },
    {
      "epoch": 0.4186289900575615,
      "eval_logits/chosen": -2.6470746994018555,
      "eval_logits/rejected": -2.59334397315979,
      "eval_logps/chosen": -284.2651672363281,
      "eval_logps/rejected": -273.2837219238281,
      "eval_loss": 0.5321290493011475,
      "eval_rewards/accuracies": 0.7539682388305664,
      "eval_rewards/chosen": -0.23111547529697418,
      "eval_rewards/margins": 1.0753650665283203,
      "eval_rewards/rejected": -1.3064805269241333,
      "eval_runtime": 206.5734,
      "eval_samples_per_second": 9.682,
      "eval_steps_per_second": 0.305,
      "step": 400
    },
    {
      "epoch": 0.4290947148090005,
      "grad_norm": 103.72761100708047,
      "learning_rate": 3.5249095128531856e-07,
      "logits/chosen": -2.588397741317749,
      "logits/rejected": -2.5449459552764893,
      "logps/chosen": -274.9619140625,
      "logps/rejected": -259.3703308105469,
      "loss": 0.5364,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.32226118445396423,
      "rewards/margins": 0.9323698878288269,
      "rewards/rejected": -1.2546310424804688,
      "step": 410
    },
    {
      "epoch": 0.43956043956043955,
      "grad_norm": 143.18448896096157,
      "learning_rate": 3.4408477372034736e-07,
      "logits/chosen": -2.6011955738067627,
      "logits/rejected": -2.511427402496338,
      "logps/chosen": -281.31475830078125,
      "logps/rejected": -264.31573486328125,
      "loss": 0.5605,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.08763083815574646,
      "rewards/margins": 1.3145806789398193,
      "rewards/rejected": -1.226949691772461,
      "step": 420
    },
    {
      "epoch": 0.4500261643118786,
      "grad_norm": 105.39381768622819,
      "learning_rate": 3.3555276610977276e-07,
      "logits/chosen": -2.582556962966919,
      "logits/rejected": -2.563215732574463,
      "logps/chosen": -251.7899932861328,
      "logps/rejected": -261.0627746582031,
      "loss": 0.5028,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.19878244400024414,
      "rewards/margins": 1.2880423069000244,
      "rewards/rejected": -1.0892596244812012,
      "step": 430
    },
    {
      "epoch": 0.4604918890633176,
      "grad_norm": 97.95625671060284,
      "learning_rate": 3.269063392575352e-07,
      "logits/chosen": -2.5603010654449463,
      "logits/rejected": -2.5315120220184326,
      "logps/chosen": -294.2579040527344,
      "logps/rejected": -261.4208679199219,
      "loss": 0.5781,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.100133016705513,
      "rewards/margins": 0.6995735168457031,
      "rewards/rejected": -0.5994404554367065,
      "step": 440
    },
    {
      "epoch": 0.47095761381475665,
      "grad_norm": 115.32814881542778,
      "learning_rate": 3.1815705699316964e-07,
      "logits/chosen": -2.5920796394348145,
      "logits/rejected": -2.5575454235076904,
      "logps/chosen": -281.134033203125,
      "logps/rejected": -289.2639465332031,
      "loss": 0.5217,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.12630310654640198,
      "rewards/margins": 1.2271454334259033,
      "rewards/rejected": -1.1008423566818237,
      "step": 450
    },
    {
      "epoch": 0.48142333856619574,
      "grad_norm": 118.44955170701016,
      "learning_rate": 3.0931662070620794e-07,
      "logits/chosen": -2.5641932487487793,
      "logits/rejected": -2.5097055435180664,
      "logps/chosen": -257.33160400390625,
      "logps/rejected": -255.80221557617188,
      "loss": 0.5178,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.01436941884458065,
      "rewards/margins": 1.0947532653808594,
      "rewards/rejected": -1.0803838968276978,
      "step": 460
    },
    {
      "epoch": 0.49188906331763477,
      "grad_norm": 95.30482890686108,
      "learning_rate": 3.003968536966078e-07,
      "logits/chosen": -2.6018574237823486,
      "logits/rejected": -2.559516429901123,
      "logps/chosen": -282.80145263671875,
      "logps/rejected": -259.4977111816406,
      "loss": 0.4768,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.09861959517002106,
      "rewards/margins": 1.0542025566101074,
      "rewards/rejected": -0.9555829763412476,
      "step": 470
    },
    {
      "epoch": 0.5023547880690737,
      "grad_norm": 105.76775426891116,
      "learning_rate": 2.9140968536213693e-07,
      "logits/chosen": -2.546515464782715,
      "logits/rejected": -2.4800972938537598,
      "logps/chosen": -268.1258850097656,
      "logps/rejected": -265.9749450683594,
      "loss": 0.4923,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09133951365947723,
      "rewards/margins": 1.3749868869781494,
      "rewards/rejected": -1.466326355934143,
      "step": 480
    },
    {
      "epoch": 0.5128205128205128,
      "grad_norm": 99.04770574478776,
      "learning_rate": 2.823671352438608e-07,
      "logits/chosen": -2.5936622619628906,
      "logits/rejected": -2.4636528491973877,
      "logps/chosen": -310.77911376953125,
      "logps/rejected": -268.76055908203125,
      "loss": 0.5039,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.11457942426204681,
      "rewards/margins": 1.352496862411499,
      "rewards/rejected": -1.237917423248291,
      "step": 490
    },
    {
      "epoch": 0.5232862375719518,
      "grad_norm": 89.42678222589365,
      "learning_rate": 2.73281296951072e-07,
      "logits/chosen": -2.482358694076538,
      "logits/rejected": -2.4588165283203125,
      "logps/chosen": -258.74224853515625,
      "logps/rejected": -238.3992156982422,
      "loss": 0.5217,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.006418359465897083,
      "rewards/margins": 1.0917348861694336,
      "rewards/rejected": -1.0981531143188477,
      "step": 500
    },
    {
      "epoch": 0.5232862375719518,
      "eval_logits/chosen": -2.5772807598114014,
      "eval_logits/rejected": -2.5195107460021973,
      "eval_logps/chosen": -281.8111267089844,
      "eval_logps/rejected": -272.2919006347656,
      "eval_loss": 0.526000440120697,
      "eval_rewards/accuracies": 0.7876983880996704,
      "eval_rewards/chosen": 0.014284107834100723,
      "eval_rewards/margins": 1.2215818166732788,
      "eval_rewards/rejected": -1.2072975635528564,
      "eval_runtime": 206.5403,
      "eval_samples_per_second": 9.683,
      "eval_steps_per_second": 0.305,
      "step": 500
    },
    {
      "epoch": 0.533751962323391,
      "grad_norm": 95.80547597223038,
      "learning_rate": 2.641643219871597e-07,
      "logits/chosen": -2.5779619216918945,
      "logits/rejected": -2.567397356033325,
      "logps/chosen": -281.93707275390625,
      "logps/rejected": -275.4586486816406,
      "loss": 0.4822,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.19970571994781494,
      "rewards/margins": 1.36869215965271,
      "rewards/rejected": -1.1689863204956055,
      "step": 510
    },
    {
      "epoch": 0.54421768707483,
      "grad_norm": 88.16712559993171,
      "learning_rate": 2.550284034980507e-07,
      "logits/chosen": -2.5944199562072754,
      "logits/rejected": -2.571303129196167,
      "logps/chosen": -274.64373779296875,
      "logps/rejected": -263.65179443359375,
      "loss": 0.5305,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.2565365731716156,
      "rewards/margins": 1.113057255744934,
      "rewards/rejected": -1.369593858718872,
      "step": 520
    },
    {
      "epoch": 0.554683411826269,
      "grad_norm": 130.8715598080221,
      "learning_rate": 2.4588575996495794e-07,
      "logits/chosen": -2.616055965423584,
      "logits/rejected": -2.5736825466156006,
      "logps/chosen": -271.31500244140625,
      "logps/rejected": -268.7001037597656,
      "loss": 0.5681,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.25552767515182495,
      "rewards/margins": 1.114168405532837,
      "rewards/rejected": -1.3696961402893066,
      "step": 530
    },
    {
      "epoch": 0.565149136577708,
      "grad_norm": 105.92360698073946,
      "learning_rate": 2.367486188632446e-07,
      "logits/chosen": -2.6313700675964355,
      "logits/rejected": -2.5749824047088623,
      "logps/chosen": -290.64703369140625,
      "logps/rejected": -272.3555908203125,
      "loss": 0.517,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02643487975001335,
      "rewards/margins": 1.1456679105758667,
      "rewards/rejected": -1.172102689743042,
      "step": 540
    },
    {
      "epoch": 0.5756148613291471,
      "grad_norm": 125.07827114011454,
      "learning_rate": 2.276292003092593e-07,
      "logits/chosen": -2.5747551918029785,
      "logits/rejected": -2.5299413204193115,
      "logps/chosen": -273.75311279296875,
      "logps/rejected": -243.8401641845703,
      "loss": 0.566,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.24802279472351074,
      "rewards/margins": 1.2050869464874268,
      "rewards/rejected": -1.453109860420227,
      "step": 550
    },
    {
      "epoch": 0.5860805860805861,
      "grad_norm": 119.35091354587506,
      "learning_rate": 2.185397007170141e-07,
      "logits/chosen": -2.539616584777832,
      "logits/rejected": -2.483868360519409,
      "logps/chosen": -284.36090087890625,
      "logps/rejected": -269.1393127441406,
      "loss": 0.5529,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.15324077010154724,
      "rewards/margins": 1.269974708557129,
      "rewards/rejected": -1.4232155084609985,
      "step": 560
    },
    {
      "epoch": 0.5965463108320251,
      "grad_norm": 121.35724227176418,
      "learning_rate": 2.094922764865619e-07,
      "logits/chosen": -2.620558977127075,
      "logits/rejected": -2.552006244659424,
      "logps/chosen": -282.87921142578125,
      "logps/rejected": -279.60931396484375,
      "loss": 0.5355,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3395797610282898,
      "rewards/margins": 1.0040457248687744,
      "rewards/rejected": -1.343625545501709,
      "step": 570
    },
    {
      "epoch": 0.6070120355834642,
      "grad_norm": 114.95741828344511,
      "learning_rate": 2.0049902774588797e-07,
      "logits/chosen": -2.604160785675049,
      "logits/rejected": -2.53875994682312,
      "logps/chosen": -254.87548828125,
      "logps/rejected": -224.6983184814453,
      "loss": 0.5335,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.15681779384613037,
      "rewards/margins": 1.0275707244873047,
      "rewards/rejected": -1.1843883991241455,
      "step": 580
    },
    {
      "epoch": 0.6174777603349032,
      "grad_norm": 116.63197017075697,
      "learning_rate": 1.9157198216806238e-07,
      "logits/chosen": -2.5892062187194824,
      "logits/rejected": -2.523660659790039,
      "logps/chosen": -282.70709228515625,
      "logps/rejected": -283.9153137207031,
      "loss": 0.5121,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.4720216691493988,
      "rewards/margins": 1.0485048294067383,
      "rewards/rejected": -1.52052640914917,
      "step": 590
    },
    {
      "epoch": 0.6279434850863422,
      "grad_norm": 90.93991377080742,
      "learning_rate": 1.8272307888529274e-07,
      "logits/chosen": -2.502859592437744,
      "logits/rejected": -2.44142746925354,
      "logps/chosen": -288.04864501953125,
      "logps/rejected": -274.6844787597656,
      "loss": 0.517,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5497933626174927,
      "rewards/margins": 0.9119491577148438,
      "rewards/rejected": -1.461742639541626,
      "step": 600
    },
    {
      "epoch": 0.6279434850863422,
      "eval_logits/chosen": -2.5743534564971924,
      "eval_logits/rejected": -2.518336296081543,
      "eval_logps/chosen": -284.8755187988281,
      "eval_logps/rejected": -274.78082275390625,
      "eval_loss": 0.5262279510498047,
      "eval_rewards/accuracies": 0.7698412537574768,
      "eval_rewards/chosen": -0.29215142130851746,
      "eval_rewards/margins": 1.1640383005142212,
      "eval_rewards/rejected": -1.4561898708343506,
      "eval_runtime": 206.851,
      "eval_samples_per_second": 9.669,
      "eval_steps_per_second": 0.305,
      "step": 600
    },
    {
      "epoch": 0.6384092098377813,
      "grad_norm": 109.87505024379428,
      "learning_rate": 1.7396415252139288e-07,
      "logits/chosen": -2.54246187210083,
      "logits/rejected": -2.4643430709838867,
      "logps/chosen": -308.33099365234375,
      "logps/rejected": -259.29583740234375,
      "loss": 0.5252,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.2741188406944275,
      "rewards/margins": 1.187243103981018,
      "rewards/rejected": -1.4613618850708008,
      "step": 610
    },
    {
      "epoch": 0.6488749345892203,
      "grad_norm": 92.62549956674485,
      "learning_rate": 1.6530691736402316e-07,
      "logits/chosen": -2.4948151111602783,
      "logits/rejected": -2.4823691844940186,
      "logps/chosen": -271.5950927734375,
      "logps/rejected": -249.6556396484375,
      "loss": 0.4543,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.05750391632318497,
      "rewards/margins": 1.2674978971481323,
      "rewards/rejected": -1.3250019550323486,
      "step": 620
    },
    {
      "epoch": 0.6593406593406593,
      "grad_norm": 115.38512973551775,
      "learning_rate": 1.5676295169786864e-07,
      "logits/chosen": -2.588836193084717,
      "logits/rejected": -2.5063748359680176,
      "logps/chosen": -272.53277587890625,
      "logps/rejected": -246.722900390625,
      "loss": 0.4912,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.21716764569282532,
      "rewards/margins": 1.2922747135162354,
      "rewards/rejected": -1.0751070976257324,
      "step": 630
    },
    {
      "epoch": 0.6698063840920984,
      "grad_norm": 98.58039473996509,
      "learning_rate": 1.483436823197092e-07,
      "logits/chosen": -2.5013747215270996,
      "logits/rejected": -2.4492290019989014,
      "logps/chosen": -267.44415283203125,
      "logps/rejected": -240.54727172851562,
      "loss": 0.4593,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.04387538880109787,
      "rewards/margins": 1.1440480947494507,
      "rewards/rejected": -1.1879234313964844,
      "step": 640
    },
    {
      "epoch": 0.6802721088435374,
      "grad_norm": 108.19817847647745,
      "learning_rate": 1.4006036925609243e-07,
      "logits/chosen": -2.545503616333008,
      "logits/rejected": -2.5265164375305176,
      "logps/chosen": -264.9919738769531,
      "logps/rejected": -279.6017150878906,
      "loss": 0.5124,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.26580336689949036,
      "rewards/margins": 1.0504064559936523,
      "rewards/rejected": -0.7846031188964844,
      "step": 650
    },
    {
      "epoch": 0.6907378335949764,
      "grad_norm": 103.96694126480867,
      "learning_rate": 1.319240907040458e-07,
      "logits/chosen": -2.545365571975708,
      "logits/rejected": -2.4856820106506348,
      "logps/chosen": -279.41656494140625,
      "logps/rejected": -267.95294189453125,
      "loss": 0.5078,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.28060275316238403,
      "rewards/margins": 1.3304187059402466,
      "rewards/rejected": -1.0498158931732178,
      "step": 660
    },
    {
      "epoch": 0.7012035583464155,
      "grad_norm": 126.63379779239241,
      "learning_rate": 1.239457282149695e-07,
      "logits/chosen": -2.5796685218811035,
      "logits/rejected": -2.5014591217041016,
      "logps/chosen": -288.4901428222656,
      "logps/rejected": -296.0643615722656,
      "loss": 0.4813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.004715624265372753,
      "rewards/margins": 1.2996948957443237,
      "rewards/rejected": -1.2949790954589844,
      "step": 670
    },
    {
      "epoch": 0.7116692830978545,
      "grad_norm": 104.43312059048667,
      "learning_rate": 1.1613595214152711e-07,
      "logits/chosen": -2.5504367351531982,
      "logits/rejected": -2.536426067352295,
      "logps/chosen": -310.7039489746094,
      "logps/rejected": -300.8048400878906,
      "loss": 0.5083,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.10008208453655243,
      "rewards/margins": 1.3875254392623901,
      "rewards/rejected": -1.2874433994293213,
      "step": 680
    },
    {
      "epoch": 0.7221350078492935,
      "grad_norm": 86.78408797627893,
      "learning_rate": 1.0850520736699362e-07,
      "logits/chosen": -2.5602102279663086,
      "logits/rejected": -2.476207971572876,
      "logps/chosen": -272.1448974609375,
      "logps/rejected": -247.1821746826172,
      "loss": 0.4535,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.14905604720115662,
      "rewards/margins": 1.301754355430603,
      "rewards/rejected": -1.4508103132247925,
      "step": 690
    },
    {
      "epoch": 0.7326007326007326,
      "grad_norm": 101.551211026177,
      "learning_rate": 1.0106369933615042e-07,
      "logits/chosen": -2.5397322177886963,
      "logits/rejected": -2.493654251098633,
      "logps/chosen": -256.18463134765625,
      "logps/rejected": -252.2621612548828,
      "loss": 0.4766,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.2093852460384369,
      "rewards/margins": 1.1600673198699951,
      "rewards/rejected": -1.369452714920044,
      "step": 700
    },
    {
      "epoch": 0.7326007326007326,
      "eval_logits/chosen": -2.575119972229004,
      "eval_logits/rejected": -2.5193893909454346,
      "eval_logps/chosen": -282.1366271972656,
      "eval_logps/rejected": -273.1544494628906,
      "eval_loss": 0.527851402759552,
      "eval_rewards/accuracies": 0.7797619104385376,
      "eval_rewards/chosen": -0.018264232203364372,
      "eval_rewards/margins": 1.2752907276153564,
      "eval_rewards/rejected": -1.2935550212860107,
      "eval_runtime": 206.8622,
      "eval_samples_per_second": 9.668,
      "eval_steps_per_second": 0.305,
      "step": 700
    },
    {
      "epoch": 0.7430664573521716,
      "grad_norm": 195.86678751064096,
      "learning_rate": 9.382138040640714e-08,
      "logits/chosen": -2.5042312145233154,
      "logits/rejected": -2.4359371662139893,
      "logps/chosen": -261.2381896972656,
      "logps/rejected": -257.90142822265625,
      "loss": 0.5532,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.27821090817451477,
      "rewards/margins": 1.218544602394104,
      "rewards/rejected": -1.4967554807662964,
      "step": 710
    },
    {
      "epoch": 0.7535321821036107,
      "grad_norm": 97.34497217574805,
      "learning_rate": 8.678793653740632e-08,
      "logits/chosen": -2.5442166328430176,
      "logits/rejected": -2.506136417388916,
      "logps/chosen": -283.2102966308594,
      "logps/rejected": -260.00079345703125,
      "loss": 0.5108,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.05546964332461357,
      "rewards/margins": 1.3087044954299927,
      "rewards/rejected": -1.25323486328125,
      "step": 720
    },
    {
      "epoch": 0.7639979068550498,
      "grad_norm": 112.03077715841201,
      "learning_rate": 7.997277433690983e-08,
      "logits/chosen": -2.5620975494384766,
      "logits/rejected": -2.50207781791687,
      "logps/chosen": -268.3021545410156,
      "logps/rejected": -249.5152587890625,
      "loss": 0.4936,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.2925592064857483,
      "rewards/margins": 0.9584562182426453,
      "rewards/rejected": -1.251015543937683,
      "step": 730
    },
    {
      "epoch": 0.7744636316064888,
      "grad_norm": 104.0919829579398,
      "learning_rate": 7.338500848029602e-08,
      "logits/chosen": -2.5124528408050537,
      "logits/rejected": -2.4893691539764404,
      "logps/chosen": -255.79995727539062,
      "logps/rejected": -245.2656707763672,
      "loss": 0.557,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2949948012828827,
      "rewards/margins": 1.0080726146697998,
      "rewards/rejected": -1.3030673265457153,
      "step": 740
    },
    {
      "epoch": 0.7849293563579278,
      "grad_norm": 99.74133982899394,
      "learning_rate": 6.70334495204884e-08,
      "logits/chosen": -2.4846601486206055,
      "logits/rejected": -2.44720458984375,
      "logps/chosen": -250.17013549804688,
      "logps/rejected": -258.7621765136719,
      "loss": 0.4753,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3299402594566345,
      "rewards/margins": 1.1778974533081055,
      "rewards/rejected": -1.5078377723693848,
      "step": 750
    },
    {
      "epoch": 0.7953950811093669,
      "grad_norm": 73.11096892518727,
      "learning_rate": 6.092659210462231e-08,
      "logits/chosen": -2.5241293907165527,
      "logits/rejected": -2.492896318435669,
      "logps/chosen": -244.87533569335938,
      "logps/rejected": -258.718017578125,
      "loss": 0.5085,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.351382315158844,
      "rewards/margins": 1.2206757068634033,
      "rewards/rejected": -1.5720579624176025,
      "step": 760
    },
    {
      "epoch": 0.8058608058608059,
      "grad_norm": 75.76619723672131,
      "learning_rate": 5.507260361320737e-08,
      "logits/chosen": -2.559986114501953,
      "logits/rejected": -2.5477547645568848,
      "logps/chosen": -305.89483642578125,
      "logps/rejected": -336.5893859863281,
      "loss": 0.4732,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.25339916348457336,
      "rewards/margins": 1.2115254402160645,
      "rewards/rejected": -1.4649248123168945,
      "step": 770
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 105.8401947481118,
      "learning_rate": 4.947931323697982e-08,
      "logits/chosen": -2.517803430557251,
      "logits/rejected": -2.4777727127075195,
      "logps/chosen": -319.5518493652344,
      "logps/rejected": -279.0348815917969,
      "loss": 0.5237,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0607873797416687,
      "rewards/margins": 1.0895849466323853,
      "rewards/rejected": -1.1503721475601196,
      "step": 780
    },
    {
      "epoch": 0.826792255363684,
      "grad_norm": 113.35027564232854,
      "learning_rate": 4.415420150605398e-08,
      "logits/chosen": -2.5854082107543945,
      "logits/rejected": -2.4739928245544434,
      "logps/chosen": -280.592529296875,
      "logps/rejected": -259.7514343261719,
      "loss": 0.5226,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.20229856669902802,
      "rewards/margins": 1.2445086240768433,
      "rewards/rejected": -1.4468071460723877,
      "step": 790
    },
    {
      "epoch": 0.837257980115123,
      "grad_norm": 115.9328163130989,
      "learning_rate": 3.9104390285376374e-08,
      "logits/chosen": -2.4794726371765137,
      "logits/rejected": -2.463261365890503,
      "logps/chosen": -268.74591064453125,
      "logps/rejected": -277.9842834472656,
      "loss": 0.4894,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2617167830467224,
      "rewards/margins": 1.2013636827468872,
      "rewards/rejected": -1.4630804061889648,
      "step": 800
    },
    {
      "epoch": 0.837257980115123,
      "eval_logits/chosen": -2.5850720405578613,
      "eval_logits/rejected": -2.531074047088623,
      "eval_logps/chosen": -282.5210876464844,
      "eval_logps/rejected": -272.81268310546875,
      "eval_loss": 0.5257476568222046,
      "eval_rewards/accuracies": 0.7777777910232544,
      "eval_rewards/chosen": -0.05670681595802307,
      "eval_rewards/margins": 1.2026678323745728,
      "eval_rewards/rejected": -1.259374737739563,
      "eval_runtime": 207.4453,
      "eval_samples_per_second": 9.641,
      "eval_steps_per_second": 0.304,
      "step": 800
    },
    {
      "epoch": 0.847723704866562,
      "grad_norm": 87.80187909830617,
      "learning_rate": 3.433663324986208e-08,
      "logits/chosen": -2.5537991523742676,
      "logits/rejected": -2.495201826095581,
      "logps/chosen": -292.41900634765625,
      "logps/rejected": -258.72625732421875,
      "loss": 0.5129,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.12009086459875107,
      "rewards/margins": 1.1076496839523315,
      "rewards/rejected": -1.2277406454086304,
      "step": 810
    },
    {
      "epoch": 0.858189429618001,
      "grad_norm": 113.52567438362996,
      "learning_rate": 2.9857306851953897e-08,
      "logits/chosen": -2.5736160278320312,
      "logits/rejected": -2.5441737174987793,
      "logps/chosen": -305.28387451171875,
      "logps/rejected": -295.17120361328125,
      "loss": 0.508,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05176614597439766,
      "rewards/margins": 1.149815320968628,
      "rewards/rejected": -1.201581358909607,
      "step": 820
    },
    {
      "epoch": 0.8686551543694401,
      "grad_norm": 94.35513967454145,
      "learning_rate": 2.567240179368185e-08,
      "logits/chosen": -2.5325002670288086,
      "logits/rejected": -2.5152266025543213,
      "logps/chosen": -259.08319091796875,
      "logps/rejected": -262.1970520019531,
      "loss": 0.5463,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.09561798721551895,
      "rewards/margins": 1.1467230319976807,
      "rewards/rejected": -1.242341160774231,
      "step": 830
    },
    {
      "epoch": 0.8791208791208791,
      "grad_norm": 129.97386022506407,
      "learning_rate": 2.1787515014630357e-08,
      "logits/chosen": -2.540358304977417,
      "logits/rejected": -2.4877097606658936,
      "logps/chosen": -282.8505859375,
      "logps/rejected": -278.73236083984375,
      "loss": 0.4725,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.05927306413650513,
      "rewards/margins": 1.2071725130081177,
      "rewards/rejected": -1.266445517539978,
      "step": 840
    },
    {
      "epoch": 0.8895866038723181,
      "grad_norm": 79.91861676789057,
      "learning_rate": 1.820784220652766e-08,
      "logits/chosen": -2.480196714401245,
      "logits/rejected": -2.413062572479248,
      "logps/chosen": -282.0146179199219,
      "logps/rejected": -246.9482421875,
      "loss": 0.4622,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.1173391118645668,
      "rewards/margins": 1.2294926643371582,
      "rewards/rejected": -1.3468319177627563,
      "step": 850
    },
    {
      "epoch": 0.9000523286237572,
      "grad_norm": 108.90343136375496,
      "learning_rate": 1.4938170864468636e-08,
      "logits/chosen": -2.517470359802246,
      "logits/rejected": -2.458211898803711,
      "logps/chosen": -289.65374755859375,
      "logps/rejected": -274.5342102050781,
      "loss": 0.4958,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.1372273564338684,
      "rewards/margins": 1.1825356483459473,
      "rewards/rejected": -1.3197630643844604,
      "step": 860
    },
    {
      "epoch": 0.9105180533751962,
      "grad_norm": 153.358042630482,
      "learning_rate": 1.1982873884064465e-08,
      "logits/chosen": -2.578099012374878,
      "logits/rejected": -2.503145933151245,
      "logps/chosen": -271.9369201660156,
      "logps/rejected": -258.82427978515625,
      "loss": 0.5166,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.014210554771125317,
      "rewards/margins": 1.2173117399215698,
      "rewards/rejected": -1.2315223217010498,
      "step": 870
    },
    {
      "epoch": 0.9209837781266352,
      "grad_norm": 88.58371346729444,
      "learning_rate": 9.345903713082304e-09,
      "logits/chosen": -2.5621767044067383,
      "logits/rejected": -2.545060396194458,
      "logps/chosen": -303.8287658691406,
      "logps/rejected": -306.15374755859375,
      "loss": 0.4932,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.02834125980734825,
      "rewards/margins": 1.1778244972229004,
      "rewards/rejected": -1.149483323097229,
      "step": 880
    },
    {
      "epoch": 0.9314495028780743,
      "grad_norm": 123.70760581405742,
      "learning_rate": 7.030787065396865e-09,
      "logits/chosen": -2.5063490867614746,
      "logits/rejected": -2.4980926513671875,
      "logps/chosen": -288.7598571777344,
      "logps/rejected": -291.5402526855469,
      "loss": 0.5139,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.11085305362939835,
      "rewards/margins": 1.0220136642456055,
      "rewards/rejected": -1.132866621017456,
      "step": 890
    },
    {
      "epoch": 0.9419152276295133,
      "grad_norm": 123.75723552086023,
      "learning_rate": 5.04062020432286e-09,
      "logits/chosen": -2.5003843307495117,
      "logits/rejected": -2.4942610263824463,
      "logps/chosen": -281.16094970703125,
      "logps/rejected": -294.83892822265625,
      "loss": 0.4722,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08161640912294388,
      "rewards/margins": 1.2573249340057373,
      "rewards/rejected": -1.3389413356781006,
      "step": 900
    },
    {
      "epoch": 0.9419152276295133,
      "eval_logits/chosen": -2.590116024017334,
      "eval_logits/rejected": -2.5362370014190674,
      "eval_logps/chosen": -282.11407470703125,
      "eval_logps/rejected": -272.7223205566406,
      "eval_loss": 0.5279979109764099,
      "eval_rewards/accuracies": 0.7797619104385376,
      "eval_rewards/chosen": -0.01600908488035202,
      "eval_rewards/margins": 1.2343299388885498,
      "eval_rewards/rejected": -1.2503389120101929,
      "eval_runtime": 207.5627,
      "eval_samples_per_second": 9.636,
      "eval_steps_per_second": 0.304,
      "step": 900
    },
    {
      "epoch": 0.9523809523809523,
      "grad_norm": 117.42904817244555,
      "learning_rate": 3.3780648016376866e-09,
      "logits/chosen": -2.4798412322998047,
      "logits/rejected": -2.4555060863494873,
      "logps/chosen": -224.1981658935547,
      "logps/rejected": -224.94448852539062,
      "loss": 0.4853,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0745500773191452,
      "rewards/margins": 1.1322568655014038,
      "rewards/rejected": -1.206807017326355,
      "step": 910
    },
    {
      "epoch": 0.9628466771323915,
      "grad_norm": 118.07550601245399,
      "learning_rate": 2.0453443778310766e-09,
      "logits/chosen": -2.59162974357605,
      "logits/rejected": -2.536531925201416,
      "logps/chosen": -287.6225280761719,
      "logps/rejected": -264.23828125,
      "loss": 0.5345,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.09747320413589478,
      "rewards/margins": 1.166621208190918,
      "rewards/rejected": -1.264094591140747,
      "step": 920
    },
    {
      "epoch": 0.9733124018838305,
      "grad_norm": 76.103065820101,
      "learning_rate": 1.0442413283435758e-09,
      "logits/chosen": -2.567437171936035,
      "logits/rejected": -2.467308521270752,
      "logps/chosen": -315.1402893066406,
      "logps/rejected": -279.08343505859375,
      "loss": 0.4831,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.03391196206212044,
      "rewards/margins": 1.449866533279419,
      "rewards/rejected": -1.4837785959243774,
      "step": 930
    },
    {
      "epoch": 0.9837781266352695,
      "grad_norm": 105.2080573269562,
      "learning_rate": 3.760945397705828e-10,
      "logits/chosen": -2.5792009830474854,
      "logits/rejected": -2.542511463165283,
      "logps/chosen": -261.95806884765625,
      "logps/rejected": -271.60552978515625,
      "loss": 0.4913,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.12502527236938477,
      "rewards/margins": 1.313679575920105,
      "rewards/rejected": -1.1886541843414307,
      "step": 940
    },
    {
      "epoch": 0.9942438513867086,
      "grad_norm": 121.4273155412146,
      "learning_rate": 4.17975992204056e-11,
      "logits/chosen": -2.5649020671844482,
      "logits/rejected": -2.5450055599212646,
      "logps/chosen": -303.64208984375,
      "logps/rejected": -295.95794677734375,
      "loss": 0.4961,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08054888248443604,
      "rewards/margins": 1.018601417541504,
      "rewards/rejected": -1.0991504192352295,
      "step": 950
    },
    {
      "epoch": 0.9994767137624281,
      "step": 955,
      "total_flos": 0.0,
      "train_loss": 0.528570184657711,
      "train_runtime": 17823.4182,
      "train_samples_per_second": 3.43,
      "train_steps_per_second": 0.054
    }
  ],
  "logging_steps": 10,
  "max_steps": 955,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}