{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.984,
  "eval_steps": 62,
  "global_step": 620,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0032,
      "grad_norm": 110.22585351543883,
      "learning_rate": 5.3191489361702125e-09,
      "logits/generated": -2.55141019821167,
      "logits/real": -2.4245524406433105,
      "logps/generated": -132.02703857421875,
      "logps/real": -160.4918212890625,
      "loss": 0.8087,
      "rewards/accuracies": 0.0,
      "rewards/generated": 0.0,
      "rewards/margins": 0.0,
      "rewards/real": 0.0,
      "step": 1
    },
    {
      "epoch": 0.032,
      "grad_norm": 94.60389227241137,
      "learning_rate": 5.3191489361702123e-08,
      "logits/generated": -2.485980749130249,
      "logits/real": -2.513298988342285,
      "logps/generated": -123.58778381347656,
      "logps/real": -124.10762786865234,
      "loss": 0.8229,
      "rewards/accuracies": 0.3888888955116272,
      "rewards/generated": 0.012898079119622707,
      "rewards/margins": -0.0016533082816749811,
      "rewards/real": 0.01124477107077837,
      "step": 10
    },
    {
      "epoch": 0.064,
      "grad_norm": 124.28348177080606,
      "learning_rate": 1.0638297872340425e-07,
      "logits/generated": -2.4363627433776855,
      "logits/real": -2.505277156829834,
      "logps/generated": -126.88423156738281,
      "logps/real": -140.9454345703125,
      "loss": 0.8195,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/generated": 0.18230387568473816,
      "rewards/margins": 0.028107190504670143,
      "rewards/real": 0.21041104197502136,
      "step": 20
    },
    {
      "epoch": 0.096,
      "grad_norm": 93.87242798974545,
      "learning_rate": 1.5957446808510638e-07,
      "logits/generated": -2.436490535736084,
      "logits/real": -2.544337511062622,
      "logps/generated": -114.72662353515625,
      "logps/real": -130.33639526367188,
      "loss": 0.8044,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": 0.4327339231967926,
      "rewards/margins": 0.02861124277114868,
      "rewards/real": 0.4613451361656189,
      "step": 30
    },
    {
      "epoch": 0.128,
      "grad_norm": 128.4599654276774,
      "learning_rate": 2.127659574468085e-07,
      "logits/generated": -2.614637851715088,
      "logits/real": -2.5557188987731934,
      "logps/generated": -126.87288665771484,
      "logps/real": -128.26336669921875,
      "loss": 0.8092,
      "rewards/accuracies": 0.574999988079071,
      "rewards/generated": 0.6153943538665771,
      "rewards/margins": 0.0003047659993171692,
      "rewards/real": 0.6156991720199585,
      "step": 40
    },
    {
      "epoch": 0.16,
      "grad_norm": 102.61066490262962,
      "learning_rate": 2.659574468085106e-07,
      "logits/generated": -2.4596478939056396,
      "logits/real": -2.5249321460723877,
      "logps/generated": -120.49662017822266,
      "logps/real": -123.62603759765625,
      "loss": 0.7869,
      "rewards/accuracies": 0.612500011920929,
      "rewards/generated": 0.4423678517341614,
      "rewards/margins": 0.11444463580846786,
      "rewards/real": 0.5568125247955322,
      "step": 50
    },
    {
      "epoch": 0.192,
      "grad_norm": 97.46977954003668,
      "learning_rate": 3.1914893617021275e-07,
      "logits/generated": -2.441962957382202,
      "logits/real": -2.4571471214294434,
      "logps/generated": -117.2234115600586,
      "logps/real": -133.79359436035156,
      "loss": 0.74,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": 0.5936142802238464,
      "rewards/margins": 0.31819960474967957,
      "rewards/real": 0.9118140339851379,
      "step": 60
    },
    {
      "epoch": 0.1984,
      "eval_logits/generated": -2.422919988632202,
      "eval_logits/real": -2.4710988998413086,
      "eval_logps/generated": -112.486328125,
      "eval_logps/real": -127.55885314941406,
      "eval_loss": 0.7414137125015259,
      "eval_rewards/accuracies": 0.6153846383094788,
      "eval_rewards/generated": 0.8829215168952942,
      "eval_rewards/margins": 0.2526075839996338,
      "eval_rewards/real": 1.1355290412902832,
      "eval_runtime": 27.4999,
      "eval_samples_per_second": 7.273,
      "eval_steps_per_second": 0.473,
      "step": 62
    },
    {
      "epoch": 0.224,
      "grad_norm": 111.31786558317472,
      "learning_rate": 3.7234042553191484e-07,
      "logits/generated": -2.4125685691833496,
      "logits/real": -2.412478446960449,
      "logps/generated": -122.41896057128906,
      "logps/real": -142.01275634765625,
      "loss": 0.7703,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": 1.1598608493804932,
      "rewards/margins": 0.22506161034107208,
      "rewards/real": 1.3849226236343384,
      "step": 70
    },
    {
      "epoch": 0.256,
      "grad_norm": 108.77845029282479,
      "learning_rate": 4.25531914893617e-07,
      "logits/generated": -2.425182819366455,
      "logits/real": -2.4704861640930176,
      "logps/generated": -111.57796478271484,
      "logps/real": -133.41946411132812,
      "loss": 0.7575,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": 1.273758888244629,
      "rewards/margins": 0.2473028153181076,
      "rewards/real": 1.521061658859253,
      "step": 80
    },
    {
      "epoch": 0.288,
      "grad_norm": 87.55111605760202,
      "learning_rate": 4.787234042553192e-07,
      "logits/generated": -2.457540988922119,
      "logits/real": -2.5106961727142334,
      "logps/generated": -118.1611099243164,
      "logps/real": -128.29928588867188,
      "loss": 0.6781,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": 0.9840036630630493,
      "rewards/margins": 0.5047105550765991,
      "rewards/real": 1.4887142181396484,
      "step": 90
    },
    {
      "epoch": 0.32,
      "grad_norm": 86.71987338916492,
      "learning_rate": 4.96437054631829e-07,
      "logits/generated": -2.4262959957122803,
      "logits/real": -2.446533679962158,
      "logps/generated": -118.33076477050781,
      "logps/real": -132.8785400390625,
      "loss": 0.7601,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": 1.1130377054214478,
      "rewards/margins": 0.3733757734298706,
      "rewards/real": 1.4864133596420288,
      "step": 100
    },
    {
      "epoch": 0.352,
      "grad_norm": 85.8458816841653,
      "learning_rate": 4.904988123515439e-07,
      "logits/generated": -2.3812742233276367,
      "logits/real": -2.4494009017944336,
      "logps/generated": -113.8266372680664,
      "logps/real": -131.19769287109375,
      "loss": 0.7051,
      "rewards/accuracies": 0.762499988079071,
      "rewards/generated": 0.42378121614456177,
      "rewards/margins": 0.6939576268196106,
      "rewards/real": 1.1177388429641724,
      "step": 110
    },
    {
      "epoch": 0.384,
      "grad_norm": 71.18315494374211,
      "learning_rate": 4.845605700712589e-07,
      "logits/generated": -2.4884400367736816,
      "logits/real": -2.6123955249786377,
      "logps/generated": -122.7501220703125,
      "logps/real": -125.4404067993164,
      "loss": 0.7524,
      "rewards/accuracies": 0.5625,
      "rewards/generated": 1.1129395961761475,
      "rewards/margins": 0.42210373282432556,
      "rewards/real": 1.5350432395935059,
      "step": 120
    },
    {
      "epoch": 0.3968,
      "eval_logits/generated": -2.5561394691467285,
      "eval_logits/real": -2.586364269256592,
      "eval_logps/generated": -108.77558898925781,
      "eval_logps/real": -121.60958099365234,
      "eval_loss": 0.7001773715019226,
      "eval_rewards/accuracies": 0.692307710647583,
      "eval_rewards/generated": 1.2539945840835571,
      "eval_rewards/margins": 0.47646236419677734,
      "eval_rewards/real": 1.7304571866989136,
      "eval_runtime": 36.5599,
      "eval_samples_per_second": 5.47,
      "eval_steps_per_second": 0.356,
      "step": 124
    },
    {
      "epoch": 0.416,
      "grad_norm": 70.47568521829318,
      "learning_rate": 4.786223277909738e-07,
      "logits/generated": -2.4682154655456543,
      "logits/real": -2.564629554748535,
      "logps/generated": -114.67634582519531,
      "logps/real": -131.61488342285156,
      "loss": 0.7435,
      "rewards/accuracies": 0.612500011920929,
      "rewards/generated": 0.9963735342025757,
      "rewards/margins": 0.43854427337646484,
      "rewards/real": 1.4349180459976196,
      "step": 130
    },
    {
      "epoch": 0.448,
      "grad_norm": 73.46355856068281,
      "learning_rate": 4.7268408551068883e-07,
      "logits/generated": -2.59525728225708,
      "logits/real": -2.6275722980499268,
      "logps/generated": -116.23558044433594,
      "logps/real": -126.39945220947266,
      "loss": 0.7297,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": 1.0241186618804932,
      "rewards/margins": 0.5995591878890991,
      "rewards/real": 1.6236778497695923,
      "step": 140
    },
    {
      "epoch": 0.48,
      "grad_norm": 66.15478984380387,
      "learning_rate": 4.667458432304038e-07,
      "logits/generated": -2.5851664543151855,
      "logits/real": -2.584568977355957,
      "logps/generated": -115.7433853149414,
      "logps/real": -126.77409362792969,
      "loss": 0.7703,
      "rewards/accuracies": 0.6875,
      "rewards/generated": 1.2251970767974854,
      "rewards/margins": 0.4628323018550873,
      "rewards/real": 1.6880295276641846,
      "step": 150
    },
    {
      "epoch": 0.512,
      "grad_norm": 63.89907025375523,
      "learning_rate": 4.6080760095011875e-07,
      "logits/generated": -2.606046438217163,
      "logits/real": -2.6311333179473877,
      "logps/generated": -135.02236938476562,
      "logps/real": -124.39349365234375,
      "loss": 0.7752,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/generated": 0.7536696195602417,
      "rewards/margins": 0.8498827815055847,
      "rewards/real": 1.6035524606704712,
      "step": 160
    },
    {
      "epoch": 0.544,
      "grad_norm": 77.16459067147474,
      "learning_rate": 4.548693586698337e-07,
      "logits/generated": -2.3408329486846924,
      "logits/real": -2.348755121231079,
      "logps/generated": -111.637939453125,
      "logps/real": -126.47406005859375,
      "loss": 0.7802,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": 1.1328538656234741,
      "rewards/margins": 0.37310591340065,
      "rewards/real": 1.5059595108032227,
      "step": 170
    },
    {
      "epoch": 0.576,
      "grad_norm": 71.90278136552813,
      "learning_rate": 4.4893111638954866e-07,
      "logits/generated": -2.3571982383728027,
      "logits/real": -2.394517660140991,
      "logps/generated": -125.56034088134766,
      "logps/real": -134.2845001220703,
      "loss": 0.8028,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/generated": 0.9942463040351868,
      "rewards/margins": 0.24001729488372803,
      "rewards/real": 1.2342634201049805,
      "step": 180
    },
    {
      "epoch": 0.5952,
      "eval_logits/generated": -2.410170316696167,
      "eval_logits/real": -2.3984007835388184,
      "eval_logps/generated": -108.79092407226562,
      "eval_logps/real": -121.71672821044922,
      "eval_loss": 0.7025200724601746,
      "eval_rewards/accuracies": 0.6538461446762085,
      "eval_rewards/generated": 1.2524622678756714,
      "eval_rewards/margins": 0.4672794044017792,
      "eval_rewards/real": 1.7197418212890625,
      "eval_runtime": 36.467,
      "eval_samples_per_second": 5.484,
      "eval_steps_per_second": 0.356,
      "step": 186
    },
    {
      "epoch": 0.608,
      "grad_norm": 86.7511219785863,
      "learning_rate": 4.429928741092636e-07,
      "logits/generated": -2.4148917198181152,
      "logits/real": -2.3717541694641113,
      "logps/generated": -118.5252685546875,
      "logps/real": -148.78070068359375,
      "loss": 0.7774,
      "rewards/accuracies": 0.75,
      "rewards/generated": 0.8947796821594238,
      "rewards/margins": 0.7275075912475586,
      "rewards/real": 1.6222871541976929,
      "step": 190
    },
    {
      "epoch": 0.64,
      "grad_norm": 81.51532186606975,
      "learning_rate": 4.3705463182897863e-07,
      "logits/generated": -2.333613872528076,
      "logits/real": -2.317903757095337,
      "logps/generated": -106.9345703125,
      "logps/real": -122.63179779052734,
      "loss": 0.7231,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": 1.007331132888794,
      "rewards/margins": 0.8109349012374878,
      "rewards/real": 1.8182659149169922,
      "step": 200
    },
    {
      "epoch": 0.672,
      "grad_norm": 66.81550093382229,
      "learning_rate": 4.311163895486936e-07,
      "logits/generated": -2.3513429164886475,
      "logits/real": -2.4845714569091797,
      "logps/generated": -115.06642150878906,
      "logps/real": -125.45219421386719,
      "loss": 0.7643,
      "rewards/accuracies": 0.625,
      "rewards/generated": 1.3189352750778198,
      "rewards/margins": 0.4965497851371765,
      "rewards/real": 1.8154850006103516,
      "step": 210
    },
    {
      "epoch": 0.704,
      "grad_norm": 69.67950191082039,
      "learning_rate": 4.251781472684085e-07,
      "logits/generated": -2.458786725997925,
      "logits/real": -2.4684672355651855,
      "logps/generated": -104.95294189453125,
      "logps/real": -110.86625671386719,
      "loss": 0.7933,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/generated": 1.5485458374023438,
      "rewards/margins": 0.44912296533584595,
      "rewards/real": 1.9976688623428345,
      "step": 220
    },
    {
      "epoch": 0.736,
      "grad_norm": 67.77084563505426,
      "learning_rate": 4.192399049881235e-07,
      "logits/generated": -2.4709246158599854,
      "logits/real": -2.4637975692749023,
      "logps/generated": -109.6999282836914,
      "logps/real": -118.01756286621094,
      "loss": 0.7679,
      "rewards/accuracies": 0.8125,
      "rewards/generated": 1.0844991207122803,
      "rewards/margins": 0.8938215374946594,
      "rewards/real": 1.978320837020874,
      "step": 230
    },
    {
      "epoch": 0.768,
      "grad_norm": 72.48546532520147,
      "learning_rate": 4.1330166270783846e-07,
      "logits/generated": -2.4811666011810303,
      "logits/real": -2.499788284301758,
      "logps/generated": -121.445068359375,
      "logps/real": -134.60311889648438,
      "loss": 0.7502,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": 1.3976621627807617,
      "rewards/margins": 0.5193124413490295,
      "rewards/real": 1.916974425315857,
      "step": 240
    },
    {
      "epoch": 0.7936,
      "eval_logits/generated": -2.5031919479370117,
      "eval_logits/real": -2.5135350227355957,
      "eval_logps/generated": -111.80171966552734,
      "eval_logps/real": -123.52568054199219,
      "eval_loss": 0.7087541222572327,
      "eval_rewards/accuracies": 0.6346153616905212,
      "eval_rewards/generated": 0.9513813257217407,
      "eval_rewards/margins": 0.5874653458595276,
      "eval_rewards/real": 1.538846731185913,
      "eval_runtime": 37.1383,
      "eval_samples_per_second": 5.385,
      "eval_steps_per_second": 0.35,
      "step": 248
    },
    {
      "epoch": 0.8,
      "grad_norm": 69.53844560870881,
      "learning_rate": 4.0736342042755347e-07,
      "logits/generated": -2.528850793838501,
      "logits/real": -2.5651848316192627,
      "logps/generated": -111.78643798828125,
      "logps/real": -129.1221160888672,
      "loss": 0.7458,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": 1.0328727960586548,
      "rewards/margins": 0.48747485876083374,
      "rewards/real": 1.5203477144241333,
      "step": 250
    },
    {
      "epoch": 0.832,
      "grad_norm": 58.53751612377979,
      "learning_rate": 4.0142517814726837e-07,
      "logits/generated": -2.4891209602355957,
      "logits/real": -2.54585599899292,
      "logps/generated": -112.5472640991211,
      "logps/real": -133.5475311279297,
      "loss": 0.7326,
      "rewards/accuracies": 0.6875,
      "rewards/generated": 0.7736698389053345,
      "rewards/margins": 0.7718098163604736,
      "rewards/real": 1.5454795360565186,
      "step": 260
    },
    {
      "epoch": 0.864,
      "grad_norm": 66.38243191502328,
      "learning_rate": 3.9548693586698333e-07,
      "logits/generated": -2.566307544708252,
      "logits/real": -2.5040535926818848,
      "logps/generated": -109.7750015258789,
      "logps/real": -133.3238067626953,
      "loss": 0.6923,
      "rewards/accuracies": 0.737500011920929,
      "rewards/generated": 1.3040878772735596,
      "rewards/margins": 0.9263402819633484,
      "rewards/real": 2.2304282188415527,
      "step": 270
    },
    {
      "epoch": 0.896,
      "grad_norm": 58.807926451156646,
      "learning_rate": 3.8954869358669834e-07,
      "logits/generated": -2.472594738006592,
      "logits/real": -2.5396690368652344,
      "logps/generated": -109.0926513671875,
      "logps/real": -112.8726577758789,
      "loss": 0.7444,
      "rewards/accuracies": 0.612500011920929,
      "rewards/generated": 1.3761478662490845,
      "rewards/margins": 0.6517657041549683,
      "rewards/real": 2.0279135704040527,
      "step": 280
    },
    {
      "epoch": 0.928,
      "grad_norm": 77.38897475693065,
      "learning_rate": 3.836104513064133e-07,
      "logits/generated": -2.561764717102051,
      "logits/real": -2.614086627960205,
      "logps/generated": -119.01023864746094,
      "logps/real": -130.38446044921875,
      "loss": 0.8491,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": 1.8400380611419678,
      "rewards/margins": 0.4446924328804016,
      "rewards/real": 2.2847301959991455,
      "step": 290
    },
    {
      "epoch": 0.96,
      "grad_norm": 84.12176593503071,
      "learning_rate": 3.7767220902612825e-07,
      "logits/generated": -2.5616064071655273,
      "logits/real": -2.5765159130096436,
      "logps/generated": -110.4354476928711,
      "logps/real": -120.6690444946289,
      "loss": 0.7953,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": 1.4392826557159424,
      "rewards/margins": 0.5520261526107788,
      "rewards/real": 1.9913088083267212,
      "step": 300
    },
    {
      "epoch": 0.992,
      "grad_norm": 77.21737579365974,
      "learning_rate": 3.717339667458432e-07,
      "logits/generated": -2.4197304248809814,
      "logits/real": -2.5655956268310547,
      "logps/generated": -115.2034912109375,
      "logps/real": -136.11380004882812,
      "loss": 0.8621,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/generated": 1.4656058549880981,
      "rewards/margins": 0.20878872275352478,
      "rewards/real": 1.6743943691253662,
      "step": 310
    },
    {
      "epoch": 0.992,
      "eval_logits/generated": -2.496532678604126,
      "eval_logits/real": -2.5021958351135254,
      "eval_logps/generated": -110.10234832763672,
      "eval_logps/real": -123.74348449707031,
      "eval_loss": 0.7444469928741455,
      "eval_rewards/accuracies": 0.6730769276618958,
      "eval_rewards/generated": 1.1213189363479614,
      "eval_rewards/margins": 0.39574724435806274,
      "eval_rewards/real": 1.517066240310669,
      "eval_runtime": 36.2384,
      "eval_samples_per_second": 5.519,
      "eval_steps_per_second": 0.359,
      "step": 310
    },
    {
      "epoch": 1.024,
      "grad_norm": 23.478068074273793,
      "learning_rate": 3.6579572446555817e-07,
      "logits/generated": -2.4226808547973633,
      "logits/real": -2.4831509590148926,
      "logps/generated": -105.56974792480469,
      "logps/real": -120.84535217285156,
      "loss": 0.4414,
      "rewards/accuracies": 0.9375,
      "rewards/generated": 0.5887820720672607,
      "rewards/margins": 2.0551419258117676,
      "rewards/real": 2.6439242362976074,
      "step": 320
    },
    {
      "epoch": 1.056,
      "grad_norm": 53.84789861907458,
      "learning_rate": 3.598574821852731e-07,
      "logits/generated": -2.562809944152832,
      "logits/real": -2.54683256149292,
      "logps/generated": -110.83638000488281,
      "logps/real": -106.26161193847656,
      "loss": 0.3126,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": 0.8259130716323853,
      "rewards/margins": 2.4829070568084717,
      "rewards/real": 3.3088200092315674,
      "step": 330
    },
    {
      "epoch": 1.088,
      "grad_norm": 25.743224152634117,
      "learning_rate": 3.5391923990498813e-07,
      "logits/generated": -2.567718982696533,
      "logits/real": -2.5672295093536377,
      "logps/generated": -133.78305053710938,
      "logps/real": -108.94974517822266,
      "loss": 0.3081,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": 0.4089571535587311,
      "rewards/margins": 3.111121416091919,
      "rewards/real": 3.520078659057617,
      "step": 340
    },
    {
      "epoch": 1.12,
      "grad_norm": 27.156393795142105,
      "learning_rate": 3.479809976247031e-07,
      "logits/generated": -2.552985668182373,
      "logits/real": -2.639397144317627,
      "logps/generated": -118.40696716308594,
      "logps/real": -106.28694152832031,
      "loss": 0.3855,
      "rewards/accuracies": 0.9375,
      "rewards/generated": 0.319620817899704,
      "rewards/margins": 2.3786392211914062,
      "rewards/real": 2.6982600688934326,
      "step": 350
    },
    {
      "epoch": 1.152,
      "grad_norm": 17.27004888873941,
      "learning_rate": 3.42042755344418e-07,
      "logits/generated": -2.629342555999756,
      "logits/real": -2.6742465496063232,
      "logps/generated": -119.6379165649414,
      "logps/real": -97.76652526855469,
      "loss": 0.2743,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": 0.05346187204122543,
      "rewards/margins": 2.755115270614624,
      "rewards/real": 2.808577537536621,
      "step": 360
    },
    {
      "epoch": 1.184,
      "grad_norm": 41.22537338864181,
      "learning_rate": 3.36104513064133e-07,
      "logits/generated": -2.5502161979675293,
      "logits/real": -2.5699377059936523,
      "logps/generated": -116.44935607910156,
      "logps/real": -108.02735900878906,
      "loss": 0.3246,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": 0.30488744378089905,
      "rewards/margins": 2.962125301361084,
      "rewards/real": 3.26701283454895,
      "step": 370
    },
    {
      "epoch": 1.1904,
      "eval_logits/generated": -2.5670902729034424,
      "eval_logits/real": -2.5847625732421875,
      "eval_logps/generated": -104.2799072265625,
      "eval_logps/real": -115.69155883789062,
      "eval_loss": 0.7214540839195251,
      "eval_rewards/accuracies": 0.6730769276618958,
      "eval_rewards/generated": 1.7035623788833618,
      "eval_rewards/margins": 0.618696928024292,
      "eval_rewards/real": 2.3222594261169434,
      "eval_runtime": 36.7301,
      "eval_samples_per_second": 5.445,
      "eval_steps_per_second": 0.354,
      "step": 372
    },
    {
      "epoch": 1.216,
      "grad_norm": 42.19482089195438,
      "learning_rate": 3.3016627078384796e-07,
      "logits/generated": -2.6010684967041016,
      "logits/real": -2.563749313354492,
      "logps/generated": -127.38044738769531,
      "logps/real": -122.81330871582031,
      "loss": 0.3494,
      "rewards/accuracies": 0.875,
      "rewards/generated": 0.5199904441833496,
      "rewards/margins": 2.884294033050537,
      "rewards/real": 3.4042842388153076,
      "step": 380
    },
    {
      "epoch": 1.248,
      "grad_norm": 53.916948193274834,
      "learning_rate": 3.2422802850356297e-07,
      "logits/generated": -2.5801901817321777,
      "logits/real": -2.5469508171081543,
      "logps/generated": -126.4742202758789,
      "logps/real": -122.43565368652344,
      "loss": 0.3655,
      "rewards/accuracies": 0.875,
      "rewards/generated": 0.2614704668521881,
      "rewards/margins": 2.877016544342041,
      "rewards/real": 3.1384871006011963,
      "step": 390
    },
    {
      "epoch": 1.28,
      "grad_norm": 37.850375132000444,
      "learning_rate": 3.182897862232779e-07,
      "logits/generated": -2.5271100997924805,
      "logits/real": -2.5569581985473633,
      "logps/generated": -135.80126953125,
      "logps/real": -107.57261657714844,
      "loss": 0.3173,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": -0.10970146954059601,
      "rewards/margins": 2.853787422180176,
      "rewards/real": 2.7440860271453857,
      "step": 400
    },
    {
      "epoch": 1.312,
      "grad_norm": 41.572987230872016,
      "learning_rate": 3.1235154394299283e-07,
      "logits/generated": -2.5878734588623047,
      "logits/real": -2.721327066421509,
      "logps/generated": -112.33902740478516,
      "logps/real": -97.48545837402344,
      "loss": 0.3206,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": 0.2867007851600647,
      "rewards/margins": 2.679920196533203,
      "rewards/real": 2.966620922088623,
      "step": 410
    },
    {
      "epoch": 1.3439999999999999,
      "grad_norm": 30.741827558770616,
      "learning_rate": 3.0641330166270784e-07,
      "logits/generated": -2.4893412590026855,
      "logits/real": -2.570223331451416,
      "logps/generated": -110.69290924072266,
      "logps/real": -98.38088989257812,
      "loss": 0.3303,
      "rewards/accuracies": 0.9375,
      "rewards/generated": 0.0641188770532608,
      "rewards/margins": 2.817840099334717,
      "rewards/real": 2.8819591999053955,
      "step": 420
    },
    {
      "epoch": 1.376,
      "grad_norm": 37.05106236999541,
      "learning_rate": 3.004750593824228e-07,
      "logits/generated": -2.5790863037109375,
      "logits/real": -2.562358856201172,
      "logps/generated": -122.13496398925781,
      "logps/real": -102.71966552734375,
      "loss": 0.3153,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -0.3958076536655426,
      "rewards/margins": 3.1295769214630127,
      "rewards/real": 2.733769655227661,
      "step": 430
    },
    {
      "epoch": 1.3888,
      "eval_logits/generated": -2.499913454055786,
      "eval_logits/real": -2.525455951690674,
      "eval_logps/generated": -104.29454040527344,
      "eval_logps/real": -115.43984985351562,
      "eval_loss": 0.7150266170501709,
      "eval_rewards/accuracies": 0.6538461446762085,
      "eval_rewards/generated": 1.7021007537841797,
      "eval_rewards/margins": 0.6453297138214111,
      "eval_rewards/real": 2.3474302291870117,
      "eval_runtime": 37.3175,
      "eval_samples_per_second": 5.359,
      "eval_steps_per_second": 0.348,
      "step": 434
    },
    {
      "epoch": 1.408,
      "grad_norm": 59.86596162814969,
      "learning_rate": 2.9453681710213776e-07,
      "logits/generated": -2.5212674140930176,
      "logits/real": -2.6616861820220947,
      "logps/generated": -124.64134216308594,
      "logps/real": -119.5413818359375,
      "loss": 0.4341,
      "rewards/accuracies": 0.9375,
      "rewards/generated": 0.546046257019043,
      "rewards/margins": 2.7997288703918457,
      "rewards/real": 3.3457748889923096,
      "step": 440
    },
    {
      "epoch": 1.44,
      "grad_norm": 25.921657044180346,
      "learning_rate": 2.885985748218527e-07,
      "logits/generated": -2.4825379848480225,
      "logits/real": -2.5802297592163086,
      "logps/generated": -134.5606231689453,
      "logps/real": -103.09733581542969,
      "loss": 0.2921,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": 0.3280000686645508,
      "rewards/margins": 3.0266566276550293,
      "rewards/real": 3.35465669631958,
      "step": 450
    },
    {
      "epoch": 1.472,
      "grad_norm": 60.488510196254985,
      "learning_rate": 2.8266033254156767e-07,
      "logits/generated": -2.463170289993286,
      "logits/real": -2.5204989910125732,
      "logps/generated": -117.78157043457031,
      "logps/real": -94.84367370605469,
      "loss": 0.4117,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": 0.7249609231948853,
      "rewards/margins": 2.5204477310180664,
      "rewards/real": 3.245408535003662,
      "step": 460
    },
    {
      "epoch": 1.504,
      "grad_norm": 42.83019835762152,
      "learning_rate": 2.7672209026128263e-07,
      "logits/generated": -2.474457263946533,
      "logits/real": -2.513317584991455,
      "logps/generated": -133.95883178710938,
      "logps/real": -112.54573059082031,
      "loss": 0.3241,
      "rewards/accuracies": 0.887499988079071,
      "rewards/generated": -0.05931330472230911,
      "rewards/margins": 2.9567954540252686,
      "rewards/real": 2.897481918334961,
      "step": 470
    },
    {
      "epoch": 1.536,
      "grad_norm": 55.17709070072599,
      "learning_rate": 2.7078384798099764e-07,
      "logits/generated": -2.442061185836792,
      "logits/real": -2.411407947540283,
      "logps/generated": -124.24302673339844,
      "logps/real": -105.8341293334961,
      "loss": 0.3585,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": 0.2026868313550949,
      "rewards/margins": 2.592716932296753,
      "rewards/real": 2.7954039573669434,
      "step": 480
    },
    {
      "epoch": 1.568,
      "grad_norm": 38.62958702650794,
      "learning_rate": 2.648456057007126e-07,
      "logits/generated": -2.338289737701416,
      "logits/real": -2.3997902870178223,
      "logps/generated": -128.10601806640625,
      "logps/real": -111.6781234741211,
      "loss": 0.4053,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -0.04702238366007805,
      "rewards/margins": 3.0088653564453125,
      "rewards/real": 2.961843490600586,
      "step": 490
    },
    {
      "epoch": 1.5872000000000002,
      "eval_logits/generated": -2.403900623321533,
      "eval_logits/real": -2.4069132804870605,
      "eval_logps/generated": -104.69697570800781,
      "eval_logps/real": -115.92328643798828,
      "eval_loss": 0.7082696557044983,
      "eval_rewards/accuracies": 0.6730769276618958,
      "eval_rewards/generated": 1.6618565320968628,
      "eval_rewards/margins": 0.6372315287590027,
      "eval_rewards/real": 2.2990880012512207,
      "eval_runtime": 37.8337,
      "eval_samples_per_second": 5.286,
      "eval_steps_per_second": 0.344,
      "step": 496
    },
    {
      "epoch": 1.6,
      "grad_norm": 33.14482905038865,
      "learning_rate": 2.589073634204275e-07,
      "logits/generated": -2.348078727722168,
      "logits/real": -2.3059887886047363,
      "logps/generated": -123.2630615234375,
      "logps/real": -101.59033966064453,
      "loss": 0.3313,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -0.09043073654174805,
      "rewards/margins": 3.036623477935791,
      "rewards/real": 2.946192502975464,
      "step": 500
    },
    {
      "epoch": 1.6320000000000001,
      "grad_norm": 58.37181281947162,
      "learning_rate": 2.529691211401425e-07,
      "logits/generated": -2.3784213066101074,
      "logits/real": -2.445460557937622,
      "logps/generated": -137.46278381347656,
      "logps/real": -98.52521514892578,
      "loss": 0.3326,
      "rewards/accuracies": 0.887499988079071,
      "rewards/generated": -0.19049878418445587,
      "rewards/margins": 3.1443653106689453,
      "rewards/real": 2.953866958618164,
      "step": 510
    },
    {
      "epoch": 1.6640000000000001,
      "grad_norm": 51.7755109683276,
      "learning_rate": 2.4703087885985747e-07,
      "logits/generated": -2.6047139167785645,
      "logits/real": -2.575225353240967,
      "logps/generated": -125.86954498291016,
      "logps/real": -115.58686828613281,
      "loss": 0.3731,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": 0.010144609026610851,
      "rewards/margins": 2.9819352626800537,
      "rewards/real": 2.9920802116394043,
      "step": 520
    },
    {
      "epoch": 1.696,
      "grad_norm": 61.11300533777711,
      "learning_rate": 2.410926365795724e-07,
      "logits/generated": -2.598510503768921,
      "logits/real": -2.6134636402130127,
      "logps/generated": -130.40304565429688,
      "logps/real": -119.8432846069336,
      "loss": 0.4011,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": 0.22392193973064423,
      "rewards/margins": 3.1454458236694336,
      "rewards/real": 3.3693675994873047,
      "step": 530
    },
    {
      "epoch": 1.728,
      "grad_norm": 52.94216878715562,
      "learning_rate": 2.351543942992874e-07,
      "logits/generated": -2.54017972946167,
      "logits/real": -2.611607074737549,
      "logps/generated": -123.71136474609375,
      "logps/real": -117.9095687866211,
      "loss": 0.3699,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": 0.34511905908584595,
      "rewards/margins": 2.8268630504608154,
      "rewards/real": 3.1719822883605957,
      "step": 540
    },
    {
      "epoch": 1.76,
      "grad_norm": 34.291195649117704,
      "learning_rate": 2.2921615201900234e-07,
      "logits/generated": -2.530885696411133,
      "logits/real": -2.5769619941711426,
      "logps/generated": -129.57070922851562,
      "logps/real": -107.8799057006836,
      "loss": 0.3611,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": 0.188289612531662,
      "rewards/margins": 3.124110460281372,
      "rewards/real": 3.3124001026153564,
      "step": 550
    },
    {
      "epoch": 1.7856,
      "eval_logits/generated": -2.432302951812744,
      "eval_logits/real": -2.436417818069458,
      "eval_logps/generated": -104.27023315429688,
      "eval_logps/real": -115.58291625976562,
      "eval_loss": 0.7119041681289673,
      "eval_rewards/accuracies": 0.6730769276618958,
      "eval_rewards/generated": 1.704530119895935,
      "eval_rewards/margins": 0.6285931468009949,
      "eval_rewards/real": 2.333123207092285,
      "eval_runtime": 37.2029,
      "eval_samples_per_second": 5.376,
      "eval_steps_per_second": 0.349,
      "step": 558
    },
    {
      "epoch": 1.792,
      "grad_norm": 36.45272717171445,
      "learning_rate": 2.2327790973871732e-07,
      "logits/generated": -2.4495067596435547,
      "logits/real": -2.4794678688049316,
      "logps/generated": -132.83074951171875,
      "logps/real": -119.1983413696289,
      "loss": 0.3839,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": 0.6143507957458496,
      "rewards/margins": 2.7835047245025635,
      "rewards/real": 3.397855281829834,
      "step": 560
    },
    {
      "epoch": 1.8239999999999998,
      "grad_norm": 47.41784238396275,
      "learning_rate": 2.173396674584323e-07,
      "logits/generated": -2.371821641921997,
      "logits/real": -2.4102165699005127,
      "logps/generated": -118.35310363769531,
      "logps/real": -104.9133529663086,
      "loss": 0.3493,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": 0.35018137097358704,
      "rewards/margins": 2.667248487472534,
      "rewards/real": 3.017429828643799,
      "step": 570
    },
    {
      "epoch": 1.8559999999999999,
      "grad_norm": 49.63769156744519,
      "learning_rate": 2.1140142517814726e-07,
      "logits/generated": -2.4235947132110596,
      "logits/real": -2.425346851348877,
      "logps/generated": -111.2136001586914,
      "logps/real": -107.47050476074219,
      "loss": 0.3654,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": 0.31659263372421265,
      "rewards/margins": 2.903308153152466,
      "rewards/real": 3.219900608062744,
      "step": 580
    },
    {
      "epoch": 1.888,
      "grad_norm": 58.661910753155134,
      "learning_rate": 2.0546318289786222e-07,
      "logits/generated": -2.431994676589966,
      "logits/real": -2.488813638687134,
      "logps/generated": -129.04559326171875,
      "logps/real": -119.28230285644531,
      "loss": 0.4107,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": 0.3402375876903534,
      "rewards/margins": 2.8488047122955322,
      "rewards/real": 3.189042091369629,
      "step": 590
    },
    {
      "epoch": 1.92,
      "grad_norm": 21.619481492021077,
      "learning_rate": 1.9952494061757718e-07,
      "logits/generated": -2.436246871948242,
      "logits/real": -2.466956377029419,
      "logps/generated": -116.15250396728516,
      "logps/real": -107.74141693115234,
      "loss": 0.4577,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": 0.5984365344047546,
      "rewards/margins": 2.591343641281128,
      "rewards/real": 3.1897804737091064,
      "step": 600
    },
    {
      "epoch": 1.952,
      "grad_norm": 45.346853109328364,
      "learning_rate": 1.9358669833729216e-07,
      "logits/generated": -2.4542717933654785,
      "logits/real": -2.4808132648468018,
      "logps/generated": -101.80671691894531,
      "logps/real": -96.19842529296875,
      "loss": 0.4562,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": 0.5814850330352783,
      "rewards/margins": 2.2760722637176514,
      "rewards/real": 2.8575568199157715,
      "step": 610
    },
    {
      "epoch": 1.984,
      "grad_norm": 44.76458950174624,
      "learning_rate": 1.876484560570071e-07,
      "logits/generated": -2.506688356399536,
      "logits/real": -2.5051321983337402,
      "logps/generated": -109.52842712402344,
      "logps/real": -100.06086730957031,
      "loss": 0.3933,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": 0.37385421991348267,
      "rewards/margins": 2.584712266921997,
      "rewards/real": 2.958566665649414,
      "step": 620
    },
    {
      "epoch": 1.984,
      "eval_logits/generated": -2.432065725326538,
      "eval_logits/real": -2.426744222640991,
      "eval_logps/generated": -104.29174041748047,
      "eval_logps/real": -115.62232971191406,
      "eval_loss": 0.7168198227882385,
      "eval_rewards/accuracies": 0.6730769276618958,
      "eval_rewards/generated": 1.7023800611495972,
      "eval_rewards/margins": 0.6268020868301392,
      "eval_rewards/real": 2.3291821479797363,
      "eval_runtime": 36.5982,
      "eval_samples_per_second": 5.465,
      "eval_steps_per_second": 0.355,
      "step": 620
    }
  ],
  "logging_steps": 10,
  "max_steps": 936,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 62,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}