{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9952,
  "eval_steps": 62,
  "global_step": 936,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0032,
      "grad_norm": 110.22585351543883,
      "learning_rate": 5.3191489361702125e-09,
      "logits/generated": -2.55141019821167,
      "logits/real": -2.4245524406433105,
      "logps/generated": -132.02703857421875,
      "logps/real": -160.4918212890625,
      "loss": 0.8087,
      "rewards/accuracies": 0.0,
      "rewards/generated": 0.0,
      "rewards/margins": 0.0,
      "rewards/real": 0.0,
      "step": 1
    },
    {
      "epoch": 0.032,
      "grad_norm": 94.60389227241137,
      "learning_rate": 5.3191489361702123e-08,
      "logits/generated": -2.485980749130249,
      "logits/real": -2.513298988342285,
      "logps/generated": -123.58778381347656,
      "logps/real": -124.10762786865234,
      "loss": 0.8229,
      "rewards/accuracies": 0.3888888955116272,
      "rewards/generated": 0.012898079119622707,
      "rewards/margins": -0.0016533082816749811,
      "rewards/real": 0.01124477107077837,
      "step": 10
    },
    {
      "epoch": 0.064,
      "grad_norm": 124.28348177080606,
      "learning_rate": 1.0638297872340425e-07,
      "logits/generated": -2.4363627433776855,
      "logits/real": -2.505277156829834,
      "logps/generated": -126.88423156738281,
      "logps/real": -140.9454345703125,
      "loss": 0.8195,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/generated": 0.18230387568473816,
      "rewards/margins": 0.028107190504670143,
      "rewards/real": 0.21041104197502136,
      "step": 20
    },
    {
      "epoch": 0.096,
      "grad_norm": 93.87242798974545,
      "learning_rate": 1.5957446808510638e-07,
      "logits/generated": -2.436490535736084,
      "logits/real": -2.544337511062622,
      "logps/generated": -114.72662353515625,
      "logps/real": -130.33639526367188,
      "loss": 0.8044,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": 0.4327339231967926,
      "rewards/margins": 0.02861124277114868,
      "rewards/real": 0.4613451361656189,
      "step": 30
    },
    {
      "epoch": 0.128,
      "grad_norm": 128.4599654276774,
      "learning_rate": 2.127659574468085e-07,
      "logits/generated": -2.614637851715088,
      "logits/real": -2.5557188987731934,
      "logps/generated": -126.87288665771484,
      "logps/real": -128.26336669921875,
      "loss": 0.8092,
      "rewards/accuracies": 0.574999988079071,
      "rewards/generated": 0.6153943538665771,
      "rewards/margins": 0.0003047659993171692,
      "rewards/real": 0.6156991720199585,
      "step": 40
    },
    {
      "epoch": 0.16,
      "grad_norm": 102.61066490262962,
      "learning_rate": 2.659574468085106e-07,
      "logits/generated": -2.4596478939056396,
      "logits/real": -2.5249321460723877,
      "logps/generated": -120.49662017822266,
      "logps/real": -123.62603759765625,
      "loss": 0.7869,
      "rewards/accuracies": 0.612500011920929,
      "rewards/generated": 0.4423678517341614,
      "rewards/margins": 0.11444463580846786,
      "rewards/real": 0.5568125247955322,
      "step": 50
    },
    {
      "epoch": 0.192,
      "grad_norm": 97.46977954003668,
      "learning_rate": 3.1914893617021275e-07,
      "logits/generated": -2.441962957382202,
      "logits/real": -2.4571471214294434,
      "logps/generated": -117.2234115600586,
      "logps/real": -133.79359436035156,
      "loss": 0.74,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": 0.5936142802238464,
      "rewards/margins": 0.31819960474967957,
      "rewards/real": 0.9118140339851379,
      "step": 60
    },
    {
      "epoch": 0.1984,
      "eval_logits/generated": -2.422919988632202,
      "eval_logits/real": -2.4710988998413086,
      "eval_logps/generated": -112.486328125,
      "eval_logps/real": -127.55885314941406,
      "eval_loss": 0.7414137125015259,
      "eval_rewards/accuracies": 0.6153846383094788,
      "eval_rewards/generated": 0.8829215168952942,
      "eval_rewards/margins": 0.2526075839996338,
      "eval_rewards/real": 1.1355290412902832,
      "eval_runtime": 27.4999,
      "eval_samples_per_second": 7.273,
      "eval_steps_per_second": 0.473,
      "step": 62
    },
    {
      "epoch": 0.224,
      "grad_norm": 111.31786558317472,
      "learning_rate": 3.7234042553191484e-07,
      "logits/generated": -2.4125685691833496,
      "logits/real": -2.412478446960449,
      "logps/generated": -122.41896057128906,
      "logps/real": -142.01275634765625,
      "loss": 0.7703,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": 1.1598608493804932,
      "rewards/margins": 0.22506161034107208,
      "rewards/real": 1.3849226236343384,
      "step": 70
    },
    {
      "epoch": 0.256,
      "grad_norm": 108.77845029282479,
      "learning_rate": 4.25531914893617e-07,
      "logits/generated": -2.425182819366455,
      "logits/real": -2.4704861640930176,
      "logps/generated": -111.57796478271484,
      "logps/real": -133.41946411132812,
      "loss": 0.7575,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": 1.273758888244629,
      "rewards/margins": 0.2473028153181076,
      "rewards/real": 1.521061658859253,
      "step": 80
    },
    {
      "epoch": 0.288,
      "grad_norm": 87.55111605760202,
      "learning_rate": 4.787234042553192e-07,
      "logits/generated": -2.457540988922119,
      "logits/real": -2.5106961727142334,
      "logps/generated": -118.1611099243164,
      "logps/real": -128.29928588867188,
      "loss": 0.6781,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": 0.9840036630630493,
      "rewards/margins": 0.5047105550765991,
      "rewards/real": 1.4887142181396484,
      "step": 90
    },
    {
      "epoch": 0.32,
      "grad_norm": 86.71987338916492,
      "learning_rate": 4.96437054631829e-07,
      "logits/generated": -2.4262959957122803,
      "logits/real": -2.446533679962158,
      "logps/generated": -118.33076477050781,
      "logps/real": -132.8785400390625,
      "loss": 0.7601,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": 1.1130377054214478,
      "rewards/margins": 0.3733757734298706,
      "rewards/real": 1.4864133596420288,
      "step": 100
    },
    {
      "epoch": 0.352,
      "grad_norm": 85.8458816841653,
      "learning_rate": 4.904988123515439e-07,
      "logits/generated": -2.3812742233276367,
      "logits/real": -2.4494009017944336,
      "logps/generated": -113.8266372680664,
      "logps/real": -131.19769287109375,
      "loss": 0.7051,
      "rewards/accuracies": 0.762499988079071,
      "rewards/generated": 0.42378121614456177,
      "rewards/margins": 0.6939576268196106,
      "rewards/real": 1.1177388429641724,
      "step": 110
    },
    {
      "epoch": 0.384,
      "grad_norm": 71.18315494374211,
      "learning_rate": 4.845605700712589e-07,
      "logits/generated": -2.4884400367736816,
      "logits/real": -2.6123955249786377,
      "logps/generated": -122.7501220703125,
      "logps/real": -125.4404067993164,
      "loss": 0.7524,
      "rewards/accuracies": 0.5625,
      "rewards/generated": 1.1129395961761475,
      "rewards/margins": 0.42210373282432556,
      "rewards/real": 1.5350432395935059,
      "step": 120
    },
    {
      "epoch": 0.3968,
      "eval_logits/generated": -2.5561394691467285,
      "eval_logits/real": -2.586364269256592,
      "eval_logps/generated": -108.77558898925781,
      "eval_logps/real": -121.60958099365234,
      "eval_loss": 0.7001773715019226,
      "eval_rewards/accuracies": 0.692307710647583,
      "eval_rewards/generated": 1.2539945840835571,
      "eval_rewards/margins": 0.47646236419677734,
      "eval_rewards/real": 1.7304571866989136,
      "eval_runtime": 36.5599,
      "eval_samples_per_second": 5.47,
      "eval_steps_per_second": 0.356,
      "step": 124
    },
    {
      "epoch": 0.416,
      "grad_norm": 70.47568521829318,
      "learning_rate": 4.786223277909738e-07,
      "logits/generated": -2.4682154655456543,
      "logits/real": -2.564629554748535,
      "logps/generated": -114.67634582519531,
      "logps/real": -131.61488342285156,
      "loss": 0.7435,
      "rewards/accuracies": 0.612500011920929,
      "rewards/generated": 0.9963735342025757,
      "rewards/margins": 0.43854427337646484,
      "rewards/real": 1.4349180459976196,
      "step": 130
    },
    {
      "epoch": 0.448,
      "grad_norm": 73.46355856068281,
      "learning_rate": 4.7268408551068883e-07,
      "logits/generated": -2.59525728225708,
      "logits/real": -2.6275722980499268,
      "logps/generated": -116.23558044433594,
      "logps/real": -126.39945220947266,
      "loss": 0.7297,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": 1.0241186618804932,
      "rewards/margins": 0.5995591878890991,
      "rewards/real": 1.6236778497695923,
      "step": 140
    },
    {
      "epoch": 0.48,
      "grad_norm": 66.15478984380387,
      "learning_rate": 4.667458432304038e-07,
      "logits/generated": -2.5851664543151855,
      "logits/real": -2.584568977355957,
      "logps/generated": -115.7433853149414,
      "logps/real": -126.77409362792969,
      "loss": 0.7703,
      "rewards/accuracies": 0.6875,
      "rewards/generated": 1.2251970767974854,
      "rewards/margins": 0.4628323018550873,
      "rewards/real": 1.6880295276641846,
      "step": 150
    },
    {
      "epoch": 0.512,
      "grad_norm": 63.89907025375523,
      "learning_rate": 4.6080760095011875e-07,
      "logits/generated": -2.606046438217163,
      "logits/real": -2.6311333179473877,
      "logps/generated": -135.02236938476562,
      "logps/real": -124.39349365234375,
      "loss": 0.7752,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/generated": 0.7536696195602417,
      "rewards/margins": 0.8498827815055847,
      "rewards/real": 1.6035524606704712,
      "step": 160
    },
    {
      "epoch": 0.544,
      "grad_norm": 77.16459067147474,
      "learning_rate": 4.548693586698337e-07,
      "logits/generated": -2.3408329486846924,
      "logits/real": -2.348755121231079,
      "logps/generated": -111.637939453125,
      "logps/real": -126.47406005859375,
      "loss": 0.7802,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": 1.1328538656234741,
      "rewards/margins": 0.37310591340065,
      "rewards/real": 1.5059595108032227,
      "step": 170
    },
    {
      "epoch": 0.576,
      "grad_norm": 71.90278136552813,
      "learning_rate": 4.4893111638954866e-07,
      "logits/generated": -2.3571982383728027,
      "logits/real": -2.394517660140991,
      "logps/generated": -125.56034088134766,
      "logps/real": -134.2845001220703,
      "loss": 0.8028,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/generated": 0.9942463040351868,
      "rewards/margins": 0.24001729488372803,
      "rewards/real": 1.2342634201049805,
      "step": 180
    },
    {
      "epoch": 0.5952,
      "eval_logits/generated": -2.410170316696167,
      "eval_logits/real": -2.3984007835388184,
      "eval_logps/generated": -108.79092407226562,
      "eval_logps/real": -121.71672821044922,
      "eval_loss": 0.7025200724601746,
      "eval_rewards/accuracies": 0.6538461446762085,
      "eval_rewards/generated": 1.2524622678756714,
      "eval_rewards/margins": 0.4672794044017792,
      "eval_rewards/real": 1.7197418212890625,
      "eval_runtime": 36.467,
      "eval_samples_per_second": 5.484,
      "eval_steps_per_second": 0.356,
      "step": 186
    },
    {
      "epoch": 0.608,
      "grad_norm": 86.7511219785863,
      "learning_rate": 4.429928741092636e-07,
      "logits/generated": -2.4148917198181152,
      "logits/real": -2.3717541694641113,
      "logps/generated": -118.5252685546875,
      "logps/real": -148.78070068359375,
      "loss": 0.7774,
      "rewards/accuracies": 0.75,
      "rewards/generated": 0.8947796821594238,
      "rewards/margins": 0.7275075912475586,
      "rewards/real": 1.6222871541976929,
      "step": 190
    },
    {
      "epoch": 0.64,
      "grad_norm": 81.51532186606975,
      "learning_rate": 4.3705463182897863e-07,
      "logits/generated": -2.333613872528076,
      "logits/real": -2.317903757095337,
      "logps/generated": -106.9345703125,
      "logps/real": -122.63179779052734,
      "loss": 0.7231,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": 1.007331132888794,
      "rewards/margins": 0.8109349012374878,
      "rewards/real": 1.8182659149169922,
      "step": 200
    },
    {
      "epoch": 0.672,
      "grad_norm": 66.81550093382229,
      "learning_rate": 4.311163895486936e-07,
      "logits/generated": -2.3513429164886475,
      "logits/real": -2.4845714569091797,
      "logps/generated": -115.06642150878906,
      "logps/real": -125.45219421386719,
      "loss": 0.7643,
      "rewards/accuracies": 0.625,
      "rewards/generated": 1.3189352750778198,
      "rewards/margins": 0.4965497851371765,
      "rewards/real": 1.8154850006103516,
      "step": 210
    },
    {
      "epoch": 0.704,
      "grad_norm": 69.67950191082039,
      "learning_rate": 4.251781472684085e-07,
      "logits/generated": -2.458786725997925,
      "logits/real": -2.4684672355651855,
      "logps/generated": -104.95294189453125,
      "logps/real": -110.86625671386719,
      "loss": 0.7933,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/generated": 1.5485458374023438,
      "rewards/margins": 0.44912296533584595,
      "rewards/real": 1.9976688623428345,
      "step": 220
    },
    {
      "epoch": 0.736,
      "grad_norm": 67.77084563505426,
      "learning_rate": 4.192399049881235e-07,
      "logits/generated": -2.4709246158599854,
      "logits/real": -2.4637975692749023,
      "logps/generated": -109.6999282836914,
      "logps/real": -118.01756286621094,
      "loss": 0.7679,
      "rewards/accuracies": 0.8125,
      "rewards/generated": 1.0844991207122803,
      "rewards/margins": 0.8938215374946594,
      "rewards/real": 1.978320837020874,
      "step": 230
    },
    {
      "epoch": 0.768,
      "grad_norm": 72.48546532520147,
      "learning_rate": 4.1330166270783846e-07,
      "logits/generated": -2.4811666011810303,
      "logits/real": -2.499788284301758,
      "logps/generated": -121.445068359375,
      "logps/real": -134.60311889648438,
      "loss": 0.7502,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": 1.3976621627807617,
      "rewards/margins": 0.5193124413490295,
      "rewards/real": 1.916974425315857,
      "step": 240
    },
    {
      "epoch": 0.7936,
      "eval_logits/generated": -2.5031919479370117,
      "eval_logits/real": -2.5135350227355957,
      "eval_logps/generated": -111.80171966552734,
      "eval_logps/real": -123.52568054199219,
      "eval_loss": 0.7087541222572327,
      "eval_rewards/accuracies": 0.6346153616905212,
      "eval_rewards/generated": 0.9513813257217407,
      "eval_rewards/margins": 0.5874653458595276,
      "eval_rewards/real": 1.538846731185913,
      "eval_runtime": 37.1383,
      "eval_samples_per_second": 5.385,
      "eval_steps_per_second": 0.35,
      "step": 248
    },
    {
      "epoch": 0.8,
      "grad_norm": 69.53844560870881,
      "learning_rate": 4.0736342042755347e-07,
      "logits/generated": -2.528850793838501,
      "logits/real": -2.5651848316192627,
      "logps/generated": -111.78643798828125,
      "logps/real": -129.1221160888672,
      "loss": 0.7458,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": 1.0328727960586548,
      "rewards/margins": 0.48747485876083374,
      "rewards/real": 1.5203477144241333,
      "step": 250
    },
    {
      "epoch": 0.832,
      "grad_norm": 58.53751612377979,
      "learning_rate": 4.0142517814726837e-07,
      "logits/generated": -2.4891209602355957,
      "logits/real": -2.54585599899292,
      "logps/generated": -112.5472640991211,
      "logps/real": -133.5475311279297,
      "loss": 0.7326,
      "rewards/accuracies": 0.6875,
      "rewards/generated": 0.7736698389053345,
      "rewards/margins": 0.7718098163604736,
      "rewards/real": 1.5454795360565186,
      "step": 260
    },
    {
      "epoch": 0.864,
      "grad_norm": 66.38243191502328,
      "learning_rate": 3.9548693586698333e-07,
      "logits/generated": -2.566307544708252,
      "logits/real": -2.5040535926818848,
      "logps/generated": -109.7750015258789,
      "logps/real": -133.3238067626953,
      "loss": 0.6923,
      "rewards/accuracies": 0.737500011920929,
      "rewards/generated": 1.3040878772735596,
      "rewards/margins": 0.9263402819633484,
      "rewards/real": 2.2304282188415527,
      "step": 270
    },
    {
      "epoch": 0.896,
      "grad_norm": 58.807926451156646,
      "learning_rate": 3.8954869358669834e-07,
      "logits/generated": -2.472594738006592,
      "logits/real": -2.5396690368652344,
      "logps/generated": -109.0926513671875,
      "logps/real": -112.8726577758789,
      "loss": 0.7444,
      "rewards/accuracies": 0.612500011920929,
      "rewards/generated": 1.3761478662490845,
      "rewards/margins": 0.6517657041549683,
      "rewards/real": 2.0279135704040527,
      "step": 280
    },
    {
      "epoch": 0.928,
      "grad_norm": 77.38897475693065,
      "learning_rate": 3.836104513064133e-07,
      "logits/generated": -2.561764717102051,
      "logits/real": -2.614086627960205,
      "logps/generated": -119.01023864746094,
      "logps/real": -130.38446044921875,
      "loss": 0.8491,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": 1.8400380611419678,
      "rewards/margins": 0.4446924328804016,
      "rewards/real": 2.2847301959991455,
      "step": 290
    },
    {
      "epoch": 0.96,
      "grad_norm": 84.12176593503071,
      "learning_rate": 3.7767220902612825e-07,
      "logits/generated": -2.5616064071655273,
      "logits/real": -2.5765159130096436,
      "logps/generated": -110.4354476928711,
      "logps/real": -120.6690444946289,
      "loss": 0.7953,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": 1.4392826557159424,
      "rewards/margins": 0.5520261526107788,
      "rewards/real": 1.9913088083267212,
      "step": 300
    },
    {
      "epoch": 0.992,
      "grad_norm": 77.21737579365974,
      "learning_rate": 3.717339667458432e-07,
      "logits/generated": -2.4197304248809814,
      "logits/real": -2.5655956268310547,
      "logps/generated": -115.2034912109375,
      "logps/real": -136.11380004882812,
      "loss": 0.8621,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/generated": 1.4656058549880981,
      "rewards/margins": 0.20878872275352478,
      "rewards/real": 1.6743943691253662,
      "step": 310
    },
    {
      "epoch": 0.992,
      "eval_logits/generated": -2.496532678604126,
      "eval_logits/real": -2.5021958351135254,
      "eval_logps/generated": -110.10234832763672,
      "eval_logps/real": -123.74348449707031,
      "eval_loss": 0.7444469928741455,
      "eval_rewards/accuracies": 0.6730769276618958,
      "eval_rewards/generated": 1.1213189363479614,
      "eval_rewards/margins": 0.39574724435806274,
      "eval_rewards/real": 1.517066240310669,
      "eval_runtime": 36.2384,
      "eval_samples_per_second": 5.519,
      "eval_steps_per_second": 0.359,
      "step": 310
    },
    {
      "epoch": 1.024,
      "grad_norm": 23.478068074273793,
      "learning_rate": 3.6579572446555817e-07,
      "logits/generated": -2.4226808547973633,
      "logits/real": -2.4831509590148926,
      "logps/generated": -105.56974792480469,
      "logps/real": -120.84535217285156,
      "loss": 0.4414,
      "rewards/accuracies": 0.9375,
      "rewards/generated": 0.5887820720672607,
      "rewards/margins": 2.0551419258117676,
      "rewards/real": 2.6439242362976074,
      "step": 320
    },
    {
      "epoch": 1.056,
      "grad_norm": 53.84789861907458,
      "learning_rate": 3.598574821852731e-07,
      "logits/generated": -2.562809944152832,
      "logits/real": -2.54683256149292,
      "logps/generated": -110.83638000488281,
      "logps/real": -106.26161193847656,
      "loss": 0.3126,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": 0.8259130716323853,
      "rewards/margins": 2.4829070568084717,
      "rewards/real": 3.3088200092315674,
      "step": 330
    },
    {
      "epoch": 1.088,
      "grad_norm": 25.743224152634117,
      "learning_rate": 3.5391923990498813e-07,
      "logits/generated": -2.567718982696533,
      "logits/real": -2.5672295093536377,
      "logps/generated": -133.78305053710938,
      "logps/real": -108.94974517822266,
      "loss": 0.3081,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": 0.4089571535587311,
      "rewards/margins": 3.111121416091919,
      "rewards/real": 3.520078659057617,
      "step": 340
    },
    {
      "epoch": 1.12,
      "grad_norm": 27.156393795142105,
      "learning_rate": 3.479809976247031e-07,
      "logits/generated": -2.552985668182373,
      "logits/real": -2.639397144317627,
      "logps/generated": -118.40696716308594,
      "logps/real": -106.28694152832031,
      "loss": 0.3855,
      "rewards/accuracies": 0.9375,
      "rewards/generated": 0.319620817899704,
      "rewards/margins": 2.3786392211914062,
      "rewards/real": 2.6982600688934326,
      "step": 350
    },
    {
      "epoch": 1.152,
      "grad_norm": 17.27004888873941,
      "learning_rate": 3.42042755344418e-07,
      "logits/generated": -2.629342555999756,
      "logits/real": -2.6742465496063232,
      "logps/generated": -119.6379165649414,
      "logps/real": -97.76652526855469,
      "loss": 0.2743,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": 0.05346187204122543,
      "rewards/margins": 2.755115270614624,
      "rewards/real": 2.808577537536621,
      "step": 360
    },
    {
      "epoch": 1.184,
      "grad_norm": 41.22537338864181,
      "learning_rate": 3.36104513064133e-07,
      "logits/generated": -2.5502161979675293,
      "logits/real": -2.5699377059936523,
      "logps/generated": -116.44935607910156,
      "logps/real": -108.02735900878906,
      "loss": 0.3246,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": 0.30488744378089905,
      "rewards/margins": 2.962125301361084,
      "rewards/real": 3.26701283454895,
      "step": 370
    },
    {
      "epoch": 1.1904,
      "eval_logits/generated": -2.5670902729034424,
      "eval_logits/real": -2.5847625732421875,
      "eval_logps/generated": -104.2799072265625,
      "eval_logps/real": -115.69155883789062,
      "eval_loss": 0.7214540839195251,
      "eval_rewards/accuracies": 0.6730769276618958,
      "eval_rewards/generated": 1.7035623788833618,
      "eval_rewards/margins": 0.618696928024292,
      "eval_rewards/real": 2.3222594261169434,
      "eval_runtime": 36.7301,
      "eval_samples_per_second": 5.445,
      "eval_steps_per_second": 0.354,
      "step": 372
    },
    {
      "epoch": 1.216,
      "grad_norm": 42.19482089195438,
      "learning_rate": 3.3016627078384796e-07,
      "logits/generated": -2.6010684967041016,
      "logits/real": -2.563749313354492,
      "logps/generated": -127.38044738769531,
      "logps/real": -122.81330871582031,
      "loss": 0.3494,
      "rewards/accuracies": 0.875,
      "rewards/generated": 0.5199904441833496,
      "rewards/margins": 2.884294033050537,
      "rewards/real": 3.4042842388153076,
      "step": 380
    },
    {
      "epoch": 1.248,
      "grad_norm": 53.916948193274834,
      "learning_rate": 3.2422802850356297e-07,
      "logits/generated": -2.5801901817321777,
      "logits/real": -2.5469508171081543,
      "logps/generated": -126.4742202758789,
      "logps/real": -122.43565368652344,
      "loss": 0.3655,
      "rewards/accuracies": 0.875,
      "rewards/generated": 0.2614704668521881,
      "rewards/margins": 2.877016544342041,
      "rewards/real": 3.1384871006011963,
      "step": 390
    },
    {
      "epoch": 1.28,
      "grad_norm": 37.850375132000444,
      "learning_rate": 3.182897862232779e-07,
      "logits/generated": -2.5271100997924805,
      "logits/real": -2.5569581985473633,
      "logps/generated": -135.80126953125,
      "logps/real": -107.57261657714844,
      "loss": 0.3173,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": -0.10970146954059601,
      "rewards/margins": 2.853787422180176,
      "rewards/real": 2.7440860271453857,
      "step": 400
    },
    {
      "epoch": 1.312,
      "grad_norm": 41.572987230872016,
      "learning_rate": 3.1235154394299283e-07,
      "logits/generated": -2.5878734588623047,
      "logits/real": -2.721327066421509,
      "logps/generated": -112.33902740478516,
      "logps/real": -97.48545837402344,
      "loss": 0.3206,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": 0.2867007851600647,
      "rewards/margins": 2.679920196533203,
      "rewards/real": 2.966620922088623,
      "step": 410
    },
    {
      "epoch": 1.3439999999999999,
      "grad_norm": 30.741827558770616,
      "learning_rate": 3.0641330166270784e-07,
      "logits/generated": -2.4893412590026855,
      "logits/real": -2.570223331451416,
      "logps/generated": -110.69290924072266,
      "logps/real": -98.38088989257812,
      "loss": 0.3303,
      "rewards/accuracies": 0.9375,
      "rewards/generated": 0.0641188770532608,
      "rewards/margins": 2.817840099334717,
      "rewards/real": 2.8819591999053955,
      "step": 420
    },
    {
      "epoch": 1.376,
      "grad_norm": 37.05106236999541,
      "learning_rate": 3.004750593824228e-07,
      "logits/generated": -2.5790863037109375,
      "logits/real": -2.562358856201172,
      "logps/generated": -122.13496398925781,
      "logps/real": -102.71966552734375,
      "loss": 0.3153,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -0.3958076536655426,
      "rewards/margins": 3.1295769214630127,
      "rewards/real": 2.733769655227661,
      "step": 430
    },
    {
      "epoch": 1.3888,
      "eval_logits/generated": -2.499913454055786,
      "eval_logits/real": -2.525455951690674,
      "eval_logps/generated": -104.29454040527344,
      "eval_logps/real": -115.43984985351562,
      "eval_loss": 0.7150266170501709,
      "eval_rewards/accuracies": 0.6538461446762085,
      "eval_rewards/generated": 1.7021007537841797,
      "eval_rewards/margins": 0.6453297138214111,
      "eval_rewards/real": 2.3474302291870117,
      "eval_runtime": 37.3175,
      "eval_samples_per_second": 5.359,
      "eval_steps_per_second": 0.348,
      "step": 434
    },
    {
      "epoch": 1.408,
      "grad_norm": 59.86596162814969,
      "learning_rate": 2.9453681710213776e-07,
      "logits/generated": -2.5212674140930176,
      "logits/real": -2.6616861820220947,
      "logps/generated": -124.64134216308594,
      "logps/real": -119.5413818359375,
      "loss": 0.4341,
      "rewards/accuracies": 0.9375,
      "rewards/generated": 0.546046257019043,
      "rewards/margins": 2.7997288703918457,
      "rewards/real": 3.3457748889923096,
      "step": 440
    },
    {
      "epoch": 1.44,
      "grad_norm": 25.921657044180346,
      "learning_rate": 2.885985748218527e-07,
      "logits/generated": -2.4825379848480225,
      "logits/real": -2.5802297592163086,
      "logps/generated": -134.5606231689453,
      "logps/real": -103.09733581542969,
      "loss": 0.2921,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": 0.3280000686645508,
      "rewards/margins": 3.0266566276550293,
      "rewards/real": 3.35465669631958,
      "step": 450
    },
    {
      "epoch": 1.472,
      "grad_norm": 60.488510196254985,
      "learning_rate": 2.8266033254156767e-07,
      "logits/generated": -2.463170289993286,
      "logits/real": -2.5204989910125732,
      "logps/generated": -117.78157043457031,
      "logps/real": -94.84367370605469,
      "loss": 0.4117,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": 0.7249609231948853,
      "rewards/margins": 2.5204477310180664,
      "rewards/real": 3.245408535003662,
      "step": 460
    },
    {
      "epoch": 1.504,
      "grad_norm": 42.83019835762152,
      "learning_rate": 2.7672209026128263e-07,
      "logits/generated": -2.474457263946533,
      "logits/real": -2.513317584991455,
      "logps/generated": -133.95883178710938,
      "logps/real": -112.54573059082031,
      "loss": 0.3241,
      "rewards/accuracies": 0.887499988079071,
      "rewards/generated": -0.05931330472230911,
      "rewards/margins": 2.9567954540252686,
      "rewards/real": 2.897481918334961,
      "step": 470
    },
    {
      "epoch": 1.536,
      "grad_norm": 55.17709070072599,
      "learning_rate": 2.7078384798099764e-07,
      "logits/generated": -2.442061185836792,
      "logits/real": -2.411407947540283,
      "logps/generated": -124.24302673339844,
      "logps/real": -105.8341293334961,
      "loss": 0.3585,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": 0.2026868313550949,
      "rewards/margins": 2.592716932296753,
      "rewards/real": 2.7954039573669434,
      "step": 480
    },
    {
      "epoch": 1.568,
      "grad_norm": 38.62958702650794,
      "learning_rate": 2.648456057007126e-07,
      "logits/generated": -2.338289737701416,
      "logits/real": -2.3997902870178223,
      "logps/generated": -128.10601806640625,
      "logps/real": -111.6781234741211,
      "loss": 0.4053,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -0.04702238366007805,
      "rewards/margins": 3.0088653564453125,
      "rewards/real": 2.961843490600586,
      "step": 490
    },
    {
      "epoch": 1.5872000000000002,
      "eval_logits/generated": -2.403900623321533,
      "eval_logits/real": -2.4069132804870605,
      "eval_logps/generated": -104.69697570800781,
      "eval_logps/real": -115.92328643798828,
      "eval_loss": 0.7082696557044983,
      "eval_rewards/accuracies": 0.6730769276618958,
      "eval_rewards/generated": 1.6618565320968628,
      "eval_rewards/margins": 0.6372315287590027,
      "eval_rewards/real": 2.2990880012512207,
      "eval_runtime": 37.8337,
      "eval_samples_per_second": 5.286,
      "eval_steps_per_second": 0.344,
      "step": 496
    },
    {
      "epoch": 1.6,
      "grad_norm": 33.14482905038865,
      "learning_rate": 2.589073634204275e-07,
      "logits/generated": -2.348078727722168,
      "logits/real": -2.3059887886047363,
      "logps/generated": -123.2630615234375,
      "logps/real": -101.59033966064453,
      "loss": 0.3313,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -0.09043073654174805,
      "rewards/margins": 3.036623477935791,
      "rewards/real": 2.946192502975464,
      "step": 500
    },
    {
      "epoch": 1.6320000000000001,
      "grad_norm": 58.37181281947162,
      "learning_rate": 2.529691211401425e-07,
      "logits/generated": -2.3784213066101074,
      "logits/real": -2.445460557937622,
      "logps/generated": -137.46278381347656,
      "logps/real": -98.52521514892578,
      "loss": 0.3326,
      "rewards/accuracies": 0.887499988079071,
      "rewards/generated": -0.19049878418445587,
      "rewards/margins": 3.1443653106689453,
      "rewards/real": 2.953866958618164,
      "step": 510
    },
    {
      "epoch": 1.6640000000000001,
      "grad_norm": 51.7755109683276,
      "learning_rate": 2.4703087885985747e-07,
      "logits/generated": -2.6047139167785645,
      "logits/real": -2.575225353240967,
      "logps/generated": -125.86954498291016,
      "logps/real": -115.58686828613281,
      "loss": 0.3731,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": 0.010144609026610851,
      "rewards/margins": 2.9819352626800537,
      "rewards/real": 2.9920802116394043,
      "step": 520
    },
    {
      "epoch": 1.696,
      "grad_norm": 61.11300533777711,
      "learning_rate": 2.410926365795724e-07,
      "logits/generated": -2.598510503768921,
      "logits/real": -2.6134636402130127,
      "logps/generated": -130.40304565429688,
      "logps/real": -119.8432846069336,
      "loss": 0.4011,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": 0.22392193973064423,
      "rewards/margins": 3.1454458236694336,
      "rewards/real": 3.3693675994873047,
      "step": 530
    },
    {
      "epoch": 1.728,
      "grad_norm": 52.94216878715562,
      "learning_rate": 2.351543942992874e-07,
      "logits/generated": -2.54017972946167,
      "logits/real": -2.611607074737549,
      "logps/generated": -123.71136474609375,
      "logps/real": -117.9095687866211,
      "loss": 0.3699,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": 0.34511905908584595,
      "rewards/margins": 2.8268630504608154,
      "rewards/real": 3.1719822883605957,
      "step": 540
    },
    {
      "epoch": 1.76,
      "grad_norm": 34.291195649117704,
      "learning_rate": 2.2921615201900234e-07,
      "logits/generated": -2.530885696411133,
      "logits/real": -2.5769619941711426,
      "logps/generated": -129.57070922851562,
      "logps/real": -107.8799057006836,
      "loss": 0.3611,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": 0.188289612531662,
      "rewards/margins": 3.124110460281372,
      "rewards/real": 3.3124001026153564,
      "step": 550
    },
    {
      "epoch": 1.7856,
      "eval_logits/generated": -2.432302951812744,
      "eval_logits/real": -2.436417818069458,
      "eval_logps/generated": -104.27023315429688,
      "eval_logps/real": -115.58291625976562,
      "eval_loss": 0.7119041681289673,
      "eval_rewards/accuracies": 0.6730769276618958,
      "eval_rewards/generated": 1.704530119895935,
      "eval_rewards/margins": 0.6285931468009949,
      "eval_rewards/real": 2.333123207092285,
      "eval_runtime": 37.2029,
      "eval_samples_per_second": 5.376,
      "eval_steps_per_second": 0.349,
      "step": 558
    },
    {
      "epoch": 1.792,
      "grad_norm": 36.45272717171445,
      "learning_rate": 2.2327790973871732e-07,
      "logits/generated": -2.4495067596435547,
      "logits/real": -2.4794678688049316,
      "logps/generated": -132.83074951171875,
      "logps/real": -119.1983413696289,
      "loss": 0.3839,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": 0.6143507957458496,
      "rewards/margins": 2.7835047245025635,
      "rewards/real": 3.397855281829834,
      "step": 560
    },
    {
      "epoch": 1.8239999999999998,
      "grad_norm": 47.41784238396275,
      "learning_rate": 2.173396674584323e-07,
      "logits/generated": -2.371821641921997,
      "logits/real": -2.4102165699005127,
      "logps/generated": -118.35310363769531,
      "logps/real": -104.9133529663086,
      "loss": 0.3493,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": 0.35018137097358704,
      "rewards/margins": 2.667248487472534,
      "rewards/real": 3.017429828643799,
      "step": 570
    },
    {
      "epoch": 1.8559999999999999,
      "grad_norm": 49.63769156744519,
      "learning_rate": 2.1140142517814726e-07,
      "logits/generated": -2.4235947132110596,
      "logits/real": -2.425346851348877,
      "logps/generated": -111.2136001586914,
      "logps/real": -107.47050476074219,
      "loss": 0.3654,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": 0.31659263372421265,
      "rewards/margins": 2.903308153152466,
      "rewards/real": 3.219900608062744,
      "step": 580
    },
    {
      "epoch": 1.888,
      "grad_norm": 58.661910753155134,
      "learning_rate": 2.0546318289786222e-07,
      "logits/generated": -2.431994676589966,
      "logits/real": -2.488813638687134,
      "logps/generated": -129.04559326171875,
      "logps/real": -119.28230285644531,
      "loss": 0.4107,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": 0.3402375876903534,
      "rewards/margins": 2.8488047122955322,
      "rewards/real": 3.189042091369629,
      "step": 590
    },
    {
      "epoch": 1.92,
      "grad_norm": 21.619481492021077,
      "learning_rate": 1.9952494061757718e-07,
      "logits/generated": -2.436246871948242,
      "logits/real": -2.466956377029419,
      "logps/generated": -116.15250396728516,
      "logps/real": -107.74141693115234,
      "loss": 0.4577,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": 0.5984365344047546,
      "rewards/margins": 2.591343641281128,
      "rewards/real": 3.1897804737091064,
      "step": 600
    },
    {
      "epoch": 1.952,
      "grad_norm": 45.346853109328364,
      "learning_rate": 1.9358669833729216e-07,
      "logits/generated": -2.4542717933654785,
      "logits/real": -2.4808132648468018,
      "logps/generated": -101.80671691894531,
      "logps/real": -96.19842529296875,
      "loss": 0.4562,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": 0.5814850330352783,
      "rewards/margins": 2.2760722637176514,
      "rewards/real": 2.8575568199157715,
      "step": 610
    },
    {
      "epoch": 1.984,
      "grad_norm": 44.76458950174624,
      "learning_rate": 1.876484560570071e-07,
      "logits/generated": -2.506688356399536,
      "logits/real": -2.5051321983337402,
      "logps/generated": -109.52842712402344,
      "logps/real": -100.06086730957031,
      "loss": 0.3933,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": 0.37385421991348267,
      "rewards/margins": 2.584712266921997,
      "rewards/real": 2.958566665649414,
      "step": 620
    },
    {
      "epoch": 1.984,
      "eval_logits/generated": -2.432065725326538,
      "eval_logits/real": -2.426744222640991,
      "eval_logps/generated": -104.29174041748047,
      "eval_logps/real": -115.62232971191406,
      "eval_loss": 0.7168198227882385,
      "eval_rewards/accuracies": 0.6730769276618958,
      "eval_rewards/generated": 1.7023800611495972,
      "eval_rewards/margins": 0.6268020868301392,
      "eval_rewards/real": 2.3291821479797363,
      "eval_runtime": 36.5982,
      "eval_samples_per_second": 5.465,
      "eval_steps_per_second": 0.355,
      "step": 620
    },
    {
      "epoch": 2.016,
      "grad_norm": 20.538104998428924,
      "learning_rate": 1.8171021377672207e-07,
      "logits/generated": -2.5056118965148926,
      "logits/real": -2.5445072650909424,
      "logps/generated": -125.99369812011719,
      "logps/real": -116.36385345458984,
      "loss": 0.3077,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": 0.36697912216186523,
      "rewards/margins": 3.193668842315674,
      "rewards/real": 3.560647964477539,
      "step": 630
    },
    {
      "epoch": 2.048,
      "grad_norm": 26.08905065246513,
      "learning_rate": 1.7577197149643706e-07,
      "logits/generated": -2.5060019493103027,
      "logits/real": -2.5419182777404785,
      "logps/generated": -120.49100494384766,
      "logps/real": -89.79083251953125,
      "loss": 0.1648,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -0.3632515072822571,
      "rewards/margins": 3.968273639678955,
      "rewards/real": 3.6050219535827637,
      "step": 640
    },
    {
      "epoch": 2.08,
      "grad_norm": 23.70246936190852,
      "learning_rate": 1.6983372921615202e-07,
      "logits/generated": -2.3460607528686523,
      "logits/real": -2.47868013381958,
      "logps/generated": -118.75785827636719,
      "logps/real": -105.28114318847656,
      "loss": 0.2231,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -0.6100242137908936,
      "rewards/margins": 4.250189781188965,
      "rewards/real": 3.640165328979492,
      "step": 650
    },
    {
      "epoch": 2.112,
      "grad_norm": 52.82052578333479,
      "learning_rate": 1.6389548693586697e-07,
      "logits/generated": -2.4621224403381348,
      "logits/real": -2.535007953643799,
      "logps/generated": -125.203125,
      "logps/real": -101.14630126953125,
      "loss": 0.2196,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -0.19764788448810577,
      "rewards/margins": 4.143462181091309,
      "rewards/real": 3.945814847946167,
      "step": 660
    },
    {
      "epoch": 2.144,
      "grad_norm": 24.059115327582756,
      "learning_rate": 1.5795724465558193e-07,
      "logits/generated": -2.5252292156219482,
      "logits/real": -2.523108720779419,
      "logps/generated": -147.6621856689453,
      "logps/real": -110.50947570800781,
      "loss": 0.2099,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -0.5420292019844055,
      "rewards/margins": 4.283000946044922,
      "rewards/real": 3.7409720420837402,
      "step": 670
    },
    {
      "epoch": 2.176,
      "grad_norm": 31.606417791931438,
      "learning_rate": 1.520190023752969e-07,
      "logits/generated": -2.319981098175049,
      "logits/real": -2.3249740600585938,
      "logps/generated": -115.99867248535156,
      "logps/real": -91.61642456054688,
      "loss": 0.226,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -0.14518947899341583,
      "rewards/margins": 3.8571395874023438,
      "rewards/real": 3.7119498252868652,
      "step": 680
    },
    {
      "epoch": 2.1824,
      "eval_logits/generated": -2.3994381427764893,
      "eval_logits/real": -2.4211130142211914,
      "eval_logps/generated": -106.77970886230469,
      "eval_logps/real": -116.72004699707031,
      "eval_loss": 0.7430130839347839,
      "eval_rewards/accuracies": 0.6346153616905212,
      "eval_rewards/generated": 1.4535826444625854,
      "eval_rewards/margins": 0.765827476978302,
      "eval_rewards/real": 2.2194101810455322,
      "eval_runtime": 37.5985,
      "eval_samples_per_second": 5.319,
      "eval_steps_per_second": 0.346,
      "step": 682
    },
    {
      "epoch": 2.208,
      "grad_norm": 42.94228027224244,
      "learning_rate": 1.4608076009501184e-07,
      "logits/generated": -2.391913414001465,
      "logits/real": -2.399176597595215,
      "logps/generated": -138.39405822753906,
      "logps/real": -105.54881286621094,
      "loss": 0.3128,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": -0.44563937187194824,
      "rewards/margins": 3.831275224685669,
      "rewards/real": 3.3856358528137207,
      "step": 690
    },
    {
      "epoch": 2.24,
      "grad_norm": 21.835007321618892,
      "learning_rate": 1.4014251781472683e-07,
      "logits/generated": -2.358987331390381,
      "logits/real": -2.485370635986328,
      "logps/generated": -131.94631958007812,
      "logps/real": -101.49398803710938,
      "loss": 0.2099,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.540885329246521,
      "rewards/margins": 4.241226673126221,
      "rewards/real": 3.700340986251831,
      "step": 700
    },
    {
      "epoch": 2.2720000000000002,
      "grad_norm": 34.153099948226355,
      "learning_rate": 1.342042755344418e-07,
      "logits/generated": -2.4148426055908203,
      "logits/real": -2.470059394836426,
      "logps/generated": -109.29563903808594,
      "logps/real": -102.6874008178711,
      "loss": 0.2117,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -0.2213163673877716,
      "rewards/margins": 3.902988910675049,
      "rewards/real": 3.6816723346710205,
      "step": 710
    },
    {
      "epoch": 2.304,
      "grad_norm": 19.318708757059966,
      "learning_rate": 1.2826603325415677e-07,
      "logits/generated": -2.3977880477905273,
      "logits/real": -2.435490369796753,
      "logps/generated": -136.2637939453125,
      "logps/real": -103.02522277832031,
      "loss": 0.1954,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -0.8926679491996765,
      "rewards/margins": 4.538367748260498,
      "rewards/real": 3.645700454711914,
      "step": 720
    },
    {
      "epoch": 2.336,
      "grad_norm": 45.22290167254912,
      "learning_rate": 1.2232779097387173e-07,
      "logits/generated": -2.398287296295166,
      "logits/real": -2.4827542304992676,
      "logps/generated": -131.3472900390625,
      "logps/real": -111.76424407958984,
      "loss": 0.1815,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -0.5653759241104126,
      "rewards/margins": 4.332846641540527,
      "rewards/real": 3.7674708366394043,
      "step": 730
    },
    {
      "epoch": 2.368,
      "grad_norm": 13.254233051180913,
      "learning_rate": 1.163895486935867e-07,
      "logits/generated": -2.4649178981781006,
      "logits/real": -2.532130002975464,
      "logps/generated": -152.01437377929688,
      "logps/real": -120.23062896728516,
      "loss": 0.2117,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -0.7008813619613647,
      "rewards/margins": 4.570525169372559,
      "rewards/real": 3.8696446418762207,
      "step": 740
    },
    {
      "epoch": 2.3808,
      "eval_logits/generated": -2.4077060222625732,
      "eval_logits/real": -2.4527196884155273,
      "eval_logps/generated": -107.33969116210938,
      "eval_logps/real": -117.47949981689453,
      "eval_loss": 0.7449278235435486,
      "eval_rewards/accuracies": 0.5961538553237915,
      "eval_rewards/generated": 1.397584319114685,
      "eval_rewards/margins": 0.7458791732788086,
      "eval_rewards/real": 2.143463611602783,
      "eval_runtime": 36.1012,
      "eval_samples_per_second": 5.54,
      "eval_steps_per_second": 0.36,
      "step": 744
    },
    {
      "epoch": 2.4,
      "grad_norm": 10.788359913476551,
      "learning_rate": 1.1045130641330165e-07,
      "logits/generated": -2.490461826324463,
      "logits/real": -2.4492058753967285,
      "logps/generated": -134.56724548339844,
      "logps/real": -101.55000305175781,
      "loss": 0.2359,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -0.5838025808334351,
      "rewards/margins": 4.341630458831787,
      "rewards/real": 3.7578284740448,
      "step": 750
    },
    {
      "epoch": 2.432,
      "grad_norm": 10.902680751888331,
      "learning_rate": 1.0451306413301662e-07,
      "logits/generated": -2.4904537200927734,
      "logits/real": -2.522808313369751,
      "logps/generated": -128.68862915039062,
      "logps/real": -112.43693542480469,
      "loss": 0.2079,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -0.5499358773231506,
      "rewards/margins": 4.207226753234863,
      "rewards/real": 3.6572906970977783,
      "step": 760
    },
    {
      "epoch": 2.464,
      "grad_norm": 18.58168244237497,
      "learning_rate": 9.857482185273158e-08,
      "logits/generated": -2.447066068649292,
      "logits/real": -2.445065498352051,
      "logps/generated": -126.9102554321289,
      "logps/real": -96.48500061035156,
      "loss": 0.2136,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -0.6899724006652832,
      "rewards/margins": 4.371157646179199,
      "rewards/real": 3.6811859607696533,
      "step": 770
    },
    {
      "epoch": 2.496,
      "grad_norm": 62.64764523458125,
      "learning_rate": 9.263657957244655e-08,
      "logits/generated": -2.5049827098846436,
      "logits/real": -2.518681287765503,
      "logps/generated": -157.5089111328125,
      "logps/real": -117.27410888671875,
      "loss": 0.1927,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -1.1170618534088135,
      "rewards/margins": 4.752838134765625,
      "rewards/real": 3.6357765197753906,
      "step": 780
    },
    {
      "epoch": 2.528,
      "grad_norm": 57.60418622350474,
      "learning_rate": 8.669833729216151e-08,
      "logits/generated": -2.3157763481140137,
      "logits/real": -2.3824093341827393,
      "logps/generated": -126.97183990478516,
      "logps/real": -100.22572326660156,
      "loss": 0.2732,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -0.6774471402168274,
      "rewards/margins": 4.155969142913818,
      "rewards/real": 3.4785220623016357,
      "step": 790
    },
    {
      "epoch": 2.56,
      "grad_norm": 31.58369415511994,
      "learning_rate": 8.076009501187649e-08,
      "logits/generated": -2.4648053646087646,
      "logits/real": -2.4424023628234863,
      "logps/generated": -145.35653686523438,
      "logps/real": -111.73152160644531,
      "loss": 0.2304,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -0.7441765666007996,
      "rewards/margins": 4.259177207946777,
      "rewards/real": 3.515000104904175,
      "step": 800
    },
    {
      "epoch": 2.5792,
      "eval_logits/generated": -2.3410544395446777,
      "eval_logits/real": -2.392559051513672,
      "eval_logps/generated": -106.48121643066406,
      "eval_logps/real": -116.67195129394531,
      "eval_loss": 0.7552512884140015,
      "eval_rewards/accuracies": 0.5769230723381042,
      "eval_rewards/generated": 1.4834328889846802,
      "eval_rewards/margins": 0.7407881021499634,
      "eval_rewards/real": 2.2242212295532227,
      "eval_runtime": 36.8769,
      "eval_samples_per_second": 5.423,
      "eval_steps_per_second": 0.353,
      "step": 806
    },
    {
      "epoch": 2.592,
      "grad_norm": 49.93127642652704,
      "learning_rate": 7.482185273159145e-08,
      "logits/generated": -2.2843379974365234,
      "logits/real": -2.4402122497558594,
      "logps/generated": -136.36618041992188,
      "logps/real": -96.77020263671875,
      "loss": 0.2065,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -0.5222247838973999,
      "rewards/margins": 4.2587480545043945,
      "rewards/real": 3.736523151397705,
      "step": 810
    },
    {
      "epoch": 2.624,
      "grad_norm": 37.58250644609933,
      "learning_rate": 6.88836104513064e-08,
      "logits/generated": -2.3502609729766846,
      "logits/real": -2.4047064781188965,
      "logps/generated": -141.55758666992188,
      "logps/real": -115.49945068359375,
      "loss": 0.2793,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -0.4805728793144226,
      "rewards/margins": 4.331357002258301,
      "rewards/real": 3.850783586502075,
      "step": 820
    },
    {
      "epoch": 2.656,
      "grad_norm": 35.86590095950996,
      "learning_rate": 6.294536817102138e-08,
      "logits/generated": -2.4404706954956055,
      "logits/real": -2.3888516426086426,
      "logps/generated": -119.47889709472656,
      "logps/real": -102.79316711425781,
      "loss": 0.2456,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -0.43787437677383423,
      "rewards/margins": 4.165786266326904,
      "rewards/real": 3.7279114723205566,
      "step": 830
    },
    {
      "epoch": 2.6879999999999997,
      "grad_norm": 72.52101185428741,
      "learning_rate": 5.700712589073634e-08,
      "logits/generated": -2.3413405418395996,
      "logits/real": -2.4445858001708984,
      "logps/generated": -143.9036102294922,
      "logps/real": -120.9752197265625,
      "loss": 0.2809,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": 0.0636942982673645,
      "rewards/margins": 4.376380920410156,
      "rewards/real": 4.440074920654297,
      "step": 840
    },
    {
      "epoch": 2.7199999999999998,
      "grad_norm": 47.87386592895277,
      "learning_rate": 5.10688836104513e-08,
      "logits/generated": -2.459815740585327,
      "logits/real": -2.4833953380584717,
      "logps/generated": -145.05397033691406,
      "logps/real": -100.8636703491211,
      "loss": 0.1964,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -0.6957341432571411,
      "rewards/margins": 4.412557125091553,
      "rewards/real": 3.716822385787964,
      "step": 850
    },
    {
      "epoch": 2.752,
      "grad_norm": 14.759861860387986,
      "learning_rate": 4.5130641330166267e-08,
      "logits/generated": -2.402906656265259,
      "logits/real": -2.3498435020446777,
      "logps/generated": -135.8385467529297,
      "logps/real": -100.04483795166016,
      "loss": 0.2423,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -0.7252713441848755,
      "rewards/margins": 4.6403326988220215,
      "rewards/real": 3.9150619506835938,
      "step": 860
    },
    {
      "epoch": 2.7776,
      "eval_logits/generated": -2.3574001789093018,
      "eval_logits/real": -2.3973894119262695,
      "eval_logps/generated": -105.71865844726562,
      "eval_logps/real": -116.01789093017578,
      "eval_loss": 0.7526271343231201,
      "eval_rewards/accuracies": 0.5961538553237915,
      "eval_rewards/generated": 1.5596884489059448,
      "eval_rewards/margins": 0.7299386262893677,
      "eval_rewards/real": 2.2896268367767334,
      "eval_runtime": 37.3402,
      "eval_samples_per_second": 5.356,
      "eval_steps_per_second": 0.348,
      "step": 868
    },
    {
      "epoch": 2.784,
      "grad_norm": 39.33798985134705,
      "learning_rate": 3.919239904988123e-08,
      "logits/generated": -2.3367316722869873,
      "logits/real": -2.405890941619873,
      "logps/generated": -134.0688018798828,
      "logps/real": -103.1637191772461,
      "loss": 0.2581,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -0.24620242416858673,
      "rewards/margins": 4.148200035095215,
      "rewards/real": 3.9019978046417236,
      "step": 870
    },
    {
      "epoch": 2.816,
      "grad_norm": 19.24691526629623,
      "learning_rate": 3.32541567695962e-08,
      "logits/generated": -2.3322081565856934,
      "logits/real": -2.3643596172332764,
      "logps/generated": -131.92562866210938,
      "logps/real": -100.34136199951172,
      "loss": 0.2497,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -0.7228564620018005,
      "rewards/margins": 4.272365093231201,
      "rewards/real": 3.549508571624756,
      "step": 880
    },
    {
      "epoch": 2.848,
      "grad_norm": 74.32934828209018,
      "learning_rate": 2.7315914489311164e-08,
      "logits/generated": -2.411259174346924,
      "logits/real": -2.4403157234191895,
      "logps/generated": -153.52029418945312,
      "logps/real": -101.80616760253906,
      "loss": 0.2214,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -0.7813536524772644,
      "rewards/margins": 4.41187047958374,
      "rewards/real": 3.630516767501831,
      "step": 890
    },
    {
      "epoch": 2.88,
      "grad_norm": 17.231794888700385,
      "learning_rate": 2.1377672209026125e-08,
      "logits/generated": -2.440458297729492,
      "logits/real": -2.4329943656921387,
      "logps/generated": -131.6967010498047,
      "logps/real": -100.85757446289062,
      "loss": 0.1728,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -1.0743391513824463,
      "rewards/margins": 4.5941972732543945,
      "rewards/real": 3.5198585987091064,
      "step": 900
    },
    {
      "epoch": 2.912,
      "grad_norm": 31.31269522304366,
      "learning_rate": 1.5439429928741092e-08,
      "logits/generated": -2.288120985031128,
      "logits/real": -2.4024569988250732,
      "logps/generated": -126.20759582519531,
      "logps/real": -93.2852554321289,
      "loss": 0.2286,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -0.7462027072906494,
      "rewards/margins": 4.080456256866455,
      "rewards/real": 3.3342537879943848,
      "step": 910
    },
    {
      "epoch": 2.944,
      "grad_norm": 39.94010884942353,
      "learning_rate": 9.501187648456057e-09,
      "logits/generated": -2.390730381011963,
      "logits/real": -2.414212226867676,
      "logps/generated": -127.1091537475586,
      "logps/real": -113.78387451171875,
      "loss": 0.2001,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -0.6455323696136475,
      "rewards/margins": 4.345826148986816,
      "rewards/real": 3.700294017791748,
      "step": 920
    },
    {
      "epoch": 2.976,
      "grad_norm": 24.96632528956892,
      "learning_rate": 3.562945368171021e-09,
      "logits/generated": -2.4350810050964355,
      "logits/real": -2.3702778816223145,
      "logps/generated": -130.21060180664062,
      "logps/real": -103.36898040771484,
      "loss": 0.2881,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -0.7739312648773193,
      "rewards/margins": 4.571539878845215,
      "rewards/real": 3.7976088523864746,
      "step": 930
    },
    {
      "epoch": 2.976,
      "eval_logits/generated": -2.3562779426574707,
      "eval_logits/real": -2.3975794315338135,
      "eval_logps/generated": -106.50991821289062,
      "eval_logps/real": -116.46751403808594,
      "eval_loss": 0.7523060441017151,
      "eval_rewards/accuracies": 0.6153846383094788,
      "eval_rewards/generated": 1.4805629253387451,
      "eval_rewards/margins": 0.7641012668609619,
      "eval_rewards/real": 2.244664430618286,
      "eval_runtime": 37.8351,
      "eval_samples_per_second": 5.286,
      "eval_steps_per_second": 0.344,
      "step": 930
    }
  ],
  "logging_steps": 10,
  "max_steps": 936,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 62,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}