{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 15284,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 3.270111183780249e-09,
      "logits/chosen": -2.634561777114868,
      "logits/rejected": -2.673060417175293,
      "logps/chosen": -207.5323944091797,
      "logps/rejected": -286.9266052246094,
      "loss": 2500.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0,
      "learning_rate": 3.270111183780249e-08,
      "logits/chosen": -2.2174882888793945,
      "logits/rejected": -1.965146541595459,
      "logps/chosen": -185.96807861328125,
      "logps/rejected": -165.34136962890625,
      "loss": 2514.281,
      "rewards/accuracies": 0.3055555522441864,
      "rewards/chosen": -0.0004766077909152955,
      "rewards/margins": -0.0013973360182717443,
      "rewards/rejected": 0.0009207282564602792,
      "step": 10
    },
    {
      "epoch": 0.0,
      "learning_rate": 6.540222367560497e-08,
      "logits/chosen": -2.4321837425231934,
      "logits/rejected": -2.2228550910949707,
      "logps/chosen": -232.39230346679688,
      "logps/rejected": -231.34878540039062,
      "loss": 2502.0375,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0005583365564234555,
      "rewards/margins": -0.00018428356270305812,
      "rewards/rejected": 0.0007426199736073613,
      "step": 20
    },
    {
      "epoch": 0.0,
      "learning_rate": 9.810333551340746e-08,
      "logits/chosen": -2.257960796356201,
      "logits/rejected": -2.1621832847595215,
      "logps/chosen": -197.38980102539062,
      "logps/rejected": -219.08255004882812,
      "loss": 2497.7605,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00034341320861130953,
      "rewards/margins": 0.0002548896591179073,
      "rewards/rejected": 8.852362225297838e-05,
      "step": 30
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.3080444735120995e-07,
      "logits/chosen": -2.212952136993408,
      "logits/rejected": -2.2520880699157715,
      "logps/chosen": -275.9777526855469,
      "logps/rejected": -265.69256591796875,
      "loss": 2506.1715,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 9.906295599648729e-05,
      "rewards/margins": -0.0005692678969353437,
      "rewards/rejected": 0.0006683307001367211,
      "step": 40
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.6350555918901243e-07,
      "logits/chosen": -2.3477938175201416,
      "logits/rejected": -2.1409664154052734,
      "logps/chosen": -204.83493041992188,
      "logps/rejected": -184.77113342285156,
      "loss": 2495.4912,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0002936377131845802,
      "rewards/margins": 0.0004673409857787192,
      "rewards/rejected": -0.0007609786698594689,
      "step": 50
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.9620667102681492e-07,
      "logits/chosen": -2.3064382076263428,
      "logits/rejected": -2.067274570465088,
      "logps/chosen": -209.7997589111328,
      "logps/rejected": -185.90098571777344,
      "loss": 2507.9305,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 6.11677096458152e-05,
      "rewards/margins": -0.0007640757248736918,
      "rewards/rejected": 0.0008252434199675918,
      "step": 60
    },
    {
      "epoch": 0.0,
      "learning_rate": 2.289077828646174e-07,
      "logits/chosen": -2.2702558040618896,
      "logits/rejected": -2.1572697162628174,
      "logps/chosen": -217.9700927734375,
      "logps/rejected": -207.92623901367188,
      "loss": 2497.8785,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.00139452307485044,
      "rewards/margins": 0.00025031311088241637,
      "rewards/rejected": 0.001144210109487176,
      "step": 70
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.616088947024199e-07,
      "logits/chosen": -2.508427381515503,
      "logits/rejected": -2.2270090579986572,
      "logps/chosen": -258.7547912597656,
      "logps/rejected": -213.7139129638672,
      "loss": 2487.6809,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.001956066582351923,
      "rewards/margins": 0.0012630863348022103,
      "rewards/rejected": 0.0006929802475497127,
      "step": 80
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.943100065402224e-07,
      "logits/chosen": -2.258317708969116,
      "logits/rejected": -2.173069715499878,
      "logps/chosen": -184.67755126953125,
      "logps/rejected": -165.4068603515625,
      "loss": 2504.4389,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": 0.0010724717285484076,
      "rewards/margins": -0.00042183371260762215,
      "rewards/rejected": 0.0014943054411560297,
      "step": 90
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.2701111837802487e-07,
      "logits/chosen": -2.4314677715301514,
      "logits/rejected": -2.4265189170837402,
      "logps/chosen": -168.7687225341797,
      "logps/rejected": -183.88238525390625,
      "loss": 2504.3189,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.0016615685308352113,
      "rewards/margins": -0.00040531111881136894,
      "rewards/rejected": 0.0020668795332312584,
      "step": 100
    },
    {
      "epoch": 0.01,
      "eval_logits/chosen": -2.3479208946228027,
      "eval_logits/rejected": -2.1595327854156494,
      "eval_logps/chosen": -231.77784729003906,
      "eval_logps/rejected": -211.47471618652344,
      "eval_loss": 2491.39111328125,
      "eval_rewards/accuracies": 0.49149999022483826,
      "eval_rewards/chosen": 0.002271000761538744,
      "eval_rewards/margins": 0.000899210455827415,
      "eval_rewards/rejected": 0.0013717904221266508,
      "eval_runtime": 723.3073,
      "eval_samples_per_second": 2.765,
      "eval_steps_per_second": 1.383,
      "step": 100
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.5971223021582736e-07,
      "logits/chosen": -2.3472182750701904,
      "logits/rejected": -1.9991645812988281,
      "logps/chosen": -222.425048828125,
      "logps/rejected": -167.09478759765625,
      "loss": 2477.4027,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00370473088696599,
      "rewards/margins": 0.0023065346758812666,
      "rewards/rejected": 0.0013981962110847235,
      "step": 110
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.9241334205362984e-07,
      "logits/chosen": -2.3386950492858887,
      "logits/rejected": -2.244011402130127,
      "logps/chosen": -224.01455688476562,
      "logps/rejected": -234.2311553955078,
      "loss": 2486.7041,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.002775652799755335,
      "rewards/margins": 0.0013752636732533574,
      "rewards/rejected": 0.001400388777256012,
      "step": 120
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.251144538914324e-07,
      "logits/chosen": -2.261671543121338,
      "logits/rejected": -2.2174248695373535,
      "logps/chosen": -149.36570739746094,
      "logps/rejected": -148.32887268066406,
      "loss": 2481.2488,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0028014422859996557,
      "rewards/margins": 0.0019072892609983683,
      "rewards/rejected": 0.0008941531996242702,
      "step": 130
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.578155657292348e-07,
      "logits/chosen": -2.322067975997925,
      "logits/rejected": -2.2229113578796387,
      "logps/chosen": -225.69760131835938,
      "logps/rejected": -159.5988006591797,
      "loss": 2483.5568,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.002613655524328351,
      "rewards/margins": 0.0016905177617445588,
      "rewards/rejected": 0.0009231379954144359,
      "step": 140
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.905166775670374e-07,
      "logits/chosen": -2.3663547039031982,
      "logits/rejected": -2.157618999481201,
      "logps/chosen": -230.94027709960938,
      "logps/rejected": -229.13720703125,
      "loss": 2461.1934,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0041537596844136715,
      "rewards/margins": 0.003948424942791462,
      "rewards/rejected": 0.00020533411588985473,
      "step": 150
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.232177894048398e-07,
      "logits/chosen": -2.2142300605773926,
      "logits/rejected": -2.2270874977111816,
      "logps/chosen": -260.3035888671875,
      "logps/rejected": -224.7972412109375,
      "loss": 2465.09,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005649039521813393,
      "rewards/margins": 0.003626276273280382,
      "rewards/rejected": 0.002022763481363654,
      "step": 160
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.559189012426422e-07,
      "logits/chosen": -2.3138108253479004,
      "logits/rejected": -2.0362300872802734,
      "logps/chosen": -180.47308349609375,
      "logps/rejected": -156.69332885742188,
      "loss": 2483.0064,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0037951308768242598,
      "rewards/margins": 0.0018231167923659086,
      "rewards/rejected": 0.0019720138516277075,
      "step": 170
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.886200130804448e-07,
      "logits/chosen": -2.397017240524292,
      "logits/rejected": -2.3366546630859375,
      "logps/chosen": -217.3719940185547,
      "logps/rejected": -198.6235809326172,
      "loss": 2434.8789,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00856606476008892,
      "rewards/margins": 0.006757465191185474,
      "rewards/rejected": 0.0018085993360728025,
      "step": 180
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.213211249182473e-07,
      "logits/chosen": -2.068624258041382,
      "logits/rejected": -2.1739213466644287,
      "logps/chosen": -190.7734832763672,
      "logps/rejected": -208.3523406982422,
      "loss": 2460.0418,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.007227701134979725,
      "rewards/margins": 0.004224286414682865,
      "rewards/rejected": 0.0030034154187887907,
      "step": 190
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.540222367560497e-07,
      "logits/chosen": -2.275643825531006,
      "logits/rejected": -2.2396295070648193,
      "logps/chosen": -146.48544311523438,
      "logps/rejected": -177.35043334960938,
      "loss": 2448.3082,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.007266052067279816,
      "rewards/margins": 0.005484951194375753,
      "rewards/rejected": 0.001781100989319384,
      "step": 200
    },
    {
      "epoch": 0.01,
      "eval_logits/chosen": -2.346447467803955,
      "eval_logits/rejected": -2.158315420150757,
      "eval_logps/chosen": -231.08497619628906,
      "eval_logps/rejected": -211.30442810058594,
      "eval_loss": 2442.388427734375,
      "eval_rewards/accuracies": 0.5740000009536743,
      "eval_rewards/chosen": 0.009199734777212143,
      "eval_rewards/margins": 0.006124937906861305,
      "eval_rewards/rejected": 0.0030747964046895504,
      "eval_runtime": 722.3489,
      "eval_samples_per_second": 2.769,
      "eval_steps_per_second": 1.384,
      "step": 200
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.867233485938523e-07,
      "logits/chosen": -2.425363302230835,
      "logits/rejected": -2.2016208171844482,
      "logps/chosen": -218.32437133789062,
      "logps/rejected": -187.9815673828125,
      "loss": 2401.2883,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.012091477401554585,
      "rewards/margins": 0.010248173959553242,
      "rewards/rejected": 0.0018433047225698829,
      "step": 210
    },
    {
      "epoch": 0.01,
      "learning_rate": 7.194244604316547e-07,
      "logits/chosen": -2.2120730876922607,
      "logits/rejected": -2.03997802734375,
      "logps/chosen": -182.62161254882812,
      "logps/rejected": -174.7395782470703,
      "loss": 2451.2725,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0071476297453045845,
      "rewards/margins": 0.005110512487590313,
      "rewards/rejected": 0.0020371167920529842,
      "step": 220
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.521255722694571e-07,
      "logits/chosen": -2.4184441566467285,
      "logits/rejected": -2.023744583129883,
      "logps/chosen": -278.6354675292969,
      "logps/rejected": -183.7732696533203,
      "loss": 2448.2492,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.011438937857747078,
      "rewards/margins": 0.005931592546403408,
      "rewards/rejected": 0.005507343914359808,
      "step": 230
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.848266841072597e-07,
      "logits/chosen": -2.1973490715026855,
      "logits/rejected": -2.162393808364868,
      "logps/chosen": -214.4296112060547,
      "logps/rejected": -206.10311889648438,
      "loss": 2384.1496,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.014212613925337791,
      "rewards/margins": 0.012409132905304432,
      "rewards/rejected": 0.001803480088710785,
      "step": 240
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.175277959450622e-07,
      "logits/chosen": -2.1652462482452393,
      "logits/rejected": -2.323850154876709,
      "logps/chosen": -218.1637420654297,
      "logps/rejected": -220.7129364013672,
      "loss": 2410.3398,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.022624235600233078,
      "rewards/margins": 0.010092777200043201,
      "rewards/rejected": 0.012531456537544727,
      "step": 250
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.502289077828648e-07,
      "logits/chosen": -2.500765323638916,
      "logits/rejected": -2.1402554512023926,
      "logps/chosen": -253.96597290039062,
      "logps/rejected": -189.17242431640625,
      "loss": 2355.3189,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.031890373677015305,
      "rewards/margins": 0.01565035805106163,
      "rewards/rejected": 0.016240015625953674,
      "step": 260
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.829300196206672e-07,
      "logits/chosen": -2.418590784072876,
      "logits/rejected": -2.124760627746582,
      "logps/chosen": -246.11233520507812,
      "logps/rejected": -230.88671875,
      "loss": 2316.4652,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02698536589741707,
      "rewards/margins": 0.020038802176713943,
      "rewards/rejected": 0.0069465613923966885,
      "step": 270
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.156311314584696e-07,
      "logits/chosen": -2.3023593425750732,
      "logits/rejected": -2.1883482933044434,
      "logps/chosen": -159.69656372070312,
      "logps/rejected": -146.34173583984375,
      "loss": 2402.9592,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.024918871000409126,
      "rewards/margins": 0.011144409887492657,
      "rewards/rejected": 0.013774457387626171,
      "step": 280
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.483322432962722e-07,
      "logits/chosen": -2.548877716064453,
      "logits/rejected": -2.1554806232452393,
      "logps/chosen": -281.9593200683594,
      "logps/rejected": -225.94284057617188,
      "loss": 2370.7814,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.032194942235946655,
      "rewards/margins": 0.014771776273846626,
      "rewards/rejected": 0.01742316409945488,
      "step": 290
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.810333551340747e-07,
      "logits/chosen": -2.335076332092285,
      "logits/rejected": -2.1912741661071777,
      "logps/chosen": -265.0003356933594,
      "logps/rejected": -239.0242156982422,
      "loss": 2268.8053,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03659301623702049,
      "rewards/margins": 0.0267319492995739,
      "rewards/rejected": 0.009861066937446594,
      "step": 300
    },
    {
      "epoch": 0.02,
      "eval_logits/chosen": -2.3479902744293213,
      "eval_logits/rejected": -2.159848690032959,
      "eval_logps/chosen": -228.60084533691406,
      "eval_logps/rejected": -210.1324005126953,
      "eval_loss": 2338.618408203125,
      "eval_rewards/accuracies": 0.5855000019073486,
      "eval_rewards/chosen": 0.034041181206703186,
      "eval_rewards/margins": 0.019246207550168037,
      "eval_rewards/rejected": 0.014794973656535149,
      "eval_runtime": 721.764,
      "eval_samples_per_second": 2.771,
      "eval_steps_per_second": 1.385,
      "step": 300
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0137344669718771e-06,
      "logits/chosen": -2.3462746143341064,
      "logits/rejected": -2.360459804534912,
      "logps/chosen": -167.21005249023438,
      "logps/rejected": -156.17391967773438,
      "loss": 2456.4926,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.02280285768210888,
      "rewards/margins": 0.006144453771412373,
      "rewards/rejected": 0.01665840484201908,
      "step": 310
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0464355788096796e-06,
      "logits/chosen": -2.4511752128601074,
      "logits/rejected": -2.057394504547119,
      "logps/chosen": -221.5251007080078,
      "logps/rejected": -192.42202758789062,
      "loss": 2365.6414,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.036400049924850464,
      "rewards/margins": 0.016477955505251884,
      "rewards/rejected": 0.01992209441959858,
      "step": 320
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.079136690647482e-06,
      "logits/chosen": -2.431947708129883,
      "logits/rejected": -2.1998298168182373,
      "logps/chosen": -203.33926391601562,
      "logps/rejected": -176.35411071777344,
      "loss": 2227.7551,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.05072016641497612,
      "rewards/margins": 0.031620416790246964,
      "rewards/rejected": 0.019099745899438858,
      "step": 330
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.1118378024852844e-06,
      "logits/chosen": -2.1818463802337646,
      "logits/rejected": -2.3470516204833984,
      "logps/chosen": -150.8397216796875,
      "logps/rejected": -178.4027099609375,
      "loss": 2532.8238,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.02874670922756195,
      "rewards/margins": 0.0012015759712085128,
      "rewards/rejected": 0.027545129880309105,
      "step": 340
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.144538914323087e-06,
      "logits/chosen": -2.41914963722229,
      "logits/rejected": -1.9812551736831665,
      "logps/chosen": -317.7720031738281,
      "logps/rejected": -248.2475128173828,
      "loss": 2216.5986,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04338701814413071,
      "rewards/margins": 0.03322090953588486,
      "rewards/rejected": 0.010166105814278126,
      "step": 350
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.1772400261608895e-06,
      "logits/chosen": -2.4831156730651855,
      "logits/rejected": -2.1900362968444824,
      "logps/chosen": -219.9840850830078,
      "logps/rejected": -193.16220092773438,
      "loss": 2227.2625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.054959487169981,
      "rewards/margins": 0.03342561423778534,
      "rewards/rejected": 0.021533865481615067,
      "step": 360
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.2099411379986922e-06,
      "logits/chosen": -2.163857936859131,
      "logits/rejected": -2.239915609359741,
      "logps/chosen": -192.18533325195312,
      "logps/rejected": -206.8373260498047,
      "loss": 2365.8025,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.05311382934451103,
      "rewards/margins": 0.01884276606142521,
      "rewards/rejected": 0.03427106887102127,
      "step": 370
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.2426422498364946e-06,
      "logits/chosen": -2.323566436767578,
      "logits/rejected": -2.0360183715820312,
      "logps/chosen": -216.2499237060547,
      "logps/rejected": -162.70449829101562,
      "loss": 2159.9684,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.053983062505722046,
      "rewards/margins": 0.040319375693798065,
      "rewards/rejected": 0.013663689605891705,
      "step": 380
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2753433616742968e-06,
      "logits/chosen": -2.321364164352417,
      "logits/rejected": -2.241368293762207,
      "logps/chosen": -181.35208129882812,
      "logps/rejected": -245.04690551757812,
      "loss": 2274.409,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03948934003710747,
      "rewards/margins": 0.03012676164507866,
      "rewards/rejected": 0.009362581185996532,
      "step": 390
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3080444735120995e-06,
      "logits/chosen": -2.4815640449523926,
      "logits/rejected": -2.1130177974700928,
      "logps/chosen": -220.1060791015625,
      "logps/rejected": -181.07186889648438,
      "loss": 1992.3912,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.05386597663164139,
      "rewards/margins": 0.06216004490852356,
      "rewards/rejected": -0.008294071070849895,
      "step": 400
    },
    {
      "epoch": 0.03,
      "eval_logits/chosen": -2.347727060317993,
      "eval_logits/rejected": -2.159768581390381,
      "eval_logps/chosen": -227.7647247314453,
      "eval_logps/rejected": -211.16456604003906,
      "eval_loss": 2218.60302734375,
      "eval_rewards/accuracies": 0.6079999804496765,
      "eval_rewards/chosen": 0.04240221157670021,
      "eval_rewards/margins": 0.03792867437005043,
      "eval_rewards/rejected": 0.004473535809665918,
      "eval_runtime": 721.7154,
      "eval_samples_per_second": 2.771,
      "eval_steps_per_second": 1.386,
      "step": 400
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3407455853499021e-06,
      "logits/chosen": -2.464292049407959,
      "logits/rejected": -2.315389394760132,
      "logps/chosen": -257.93585205078125,
      "logps/rejected": -226.0089111328125,
      "loss": 2167.6412,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0441744327545166,
      "rewards/margins": 0.044694893062114716,
      "rewards/rejected": -0.0005204584449529648,
      "step": 410
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3734466971877046e-06,
      "logits/chosen": -2.2925400733947754,
      "logits/rejected": -2.19284725189209,
      "logps/chosen": -177.65432739257812,
      "logps/rejected": -174.91217041015625,
      "loss": 2143.5434,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04265324026346207,
      "rewards/margins": 0.047265905886888504,
      "rewards/rejected": -0.004612663760781288,
      "step": 420
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.406147809025507e-06,
      "logits/chosen": -2.2861475944519043,
      "logits/rejected": -2.099020004272461,
      "logps/chosen": -210.7884521484375,
      "logps/rejected": -183.57485961914062,
      "loss": 2120.6955,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04358814284205437,
      "rewards/margins": 0.052858226001262665,
      "rewards/rejected": -0.009270085021853447,
      "step": 430
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.4388489208633094e-06,
      "logits/chosen": -2.385740041732788,
      "logits/rejected": -2.1011345386505127,
      "logps/chosen": -250.75271606445312,
      "logps/rejected": -224.5830535888672,
      "loss": 2251.0816,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03744536638259888,
      "rewards/margins": 0.037420663982629776,
      "rewards/rejected": 2.4697743356227875e-05,
      "step": 440
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.471550032701112e-06,
      "logits/chosen": -2.4100568294525146,
      "logits/rejected": -2.207420825958252,
      "logps/chosen": -184.5848388671875,
      "logps/rejected": -194.29791259765625,
      "loss": 2247.6176,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.05063049867749214,
      "rewards/margins": 0.03911694884300232,
      "rewards/rejected": 0.011513547971844673,
      "step": 450
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5042511445389143e-06,
      "logits/chosen": -2.157378673553467,
      "logits/rejected": -2.204876184463501,
      "logps/chosen": -148.00125122070312,
      "logps/rejected": -214.1797332763672,
      "loss": 2323.2939,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.029387522488832474,
      "rewards/margins": 0.02752484381198883,
      "rewards/rejected": 0.0018626749515533447,
      "step": 460
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.536952256376717e-06,
      "logits/chosen": -2.049189329147339,
      "logits/rejected": -2.111467123031616,
      "logps/chosen": -200.1328582763672,
      "logps/rejected": -251.7501678466797,
      "loss": 2179.1457,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00797753781080246,
      "rewards/margins": 0.0605323500931263,
      "rewards/rejected": -0.052554816007614136,
      "step": 470
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5696533682145194e-06,
      "logits/chosen": -2.4478366374969482,
      "logits/rejected": -2.2150559425354004,
      "logps/chosen": -184.00650024414062,
      "logps/rejected": -158.0145263671875,
      "loss": 2159.2445,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.002402497921139002,
      "rewards/margins": 0.05066746473312378,
      "rewards/rejected": -0.04826496168971062,
      "step": 480
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.602354480052322e-06,
      "logits/chosen": -2.4213478565216064,
      "logits/rejected": -2.331252336502075,
      "logps/chosen": -264.4500732421875,
      "logps/rejected": -215.86386108398438,
      "loss": 2015.0939,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.021897155791521072,
      "rewards/margins": 0.06365485489368439,
      "rewards/rejected": -0.04175770282745361,
      "step": 490
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.6350555918901245e-06,
      "logits/chosen": -2.2143654823303223,
      "logits/rejected": -1.885271430015564,
      "logps/chosen": -216.16024780273438,
      "logps/rejected": -209.8026580810547,
      "loss": 1894.6148,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.019088638946413994,
      "rewards/margins": 0.08185932040214539,
      "rewards/rejected": -0.06277067959308624,
      "step": 500
    },
    {
      "epoch": 0.03,
      "eval_logits/chosen": -2.3530712127685547,
      "eval_logits/rejected": -2.1646721363067627,
      "eval_logps/chosen": -228.19320678710938,
      "eval_logps/rejected": -214.5511932373047,
      "eval_loss": 2099.700927734375,
      "eval_rewards/accuracies": 0.609000027179718,
      "eval_rewards/chosen": 0.0381174273788929,
      "eval_rewards/margins": 0.06751034408807755,
      "eval_rewards/rejected": -0.029392924159765244,
      "eval_runtime": 723.2269,
      "eval_samples_per_second": 2.765,
      "eval_steps_per_second": 1.383,
      "step": 500
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.6677567037279269e-06,
      "logits/chosen": -2.490424871444702,
      "logits/rejected": -2.140373706817627,
      "logps/chosen": -287.0405578613281,
      "logps/rejected": -258.70330810546875,
      "loss": 1731.6008,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.09223110973834991,
      "rewards/margins": 0.12047766149044037,
      "rewards/rejected": -0.028246542438864708,
      "step": 510
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.7004578155657295e-06,
      "logits/chosen": -2.293846845626831,
      "logits/rejected": -2.319566249847412,
      "logps/chosen": -200.75526428222656,
      "logps/rejected": -188.5205535888672,
      "loss": 1976.1541,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05964081361889839,
      "rewards/margins": 0.08185950666666031,
      "rewards/rejected": -0.022218704223632812,
      "step": 520
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.7331589274035318e-06,
      "logits/chosen": -2.174065113067627,
      "logits/rejected": -2.055107355117798,
      "logps/chosen": -181.2405548095703,
      "logps/rejected": -189.6827392578125,
      "loss": 2011.5094,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.026860039681196213,
      "rewards/margins": 0.08996396511793137,
      "rewards/rejected": -0.06310392916202545,
      "step": 530
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.7658600392413344e-06,
      "logits/chosen": -2.3945372104644775,
      "logits/rejected": -2.235696315765381,
      "logps/chosen": -202.3669891357422,
      "logps/rejected": -195.96456909179688,
      "loss": 1717.7557,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.004884188063442707,
      "rewards/margins": 0.10967157781124115,
      "rewards/rejected": -0.10478738695383072,
      "step": 540
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.7985611510791368e-06,
      "logits/chosen": -2.39052152633667,
      "logits/rejected": -1.9351263046264648,
      "logps/chosen": -267.9992980957031,
      "logps/rejected": -263.902099609375,
      "loss": 1842.549,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04449605196714401,
      "rewards/margins": 0.12291286140680313,
      "rewards/rejected": -0.16740891337394714,
      "step": 550
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8312622629169393e-06,
      "logits/chosen": -2.305115222930908,
      "logits/rejected": -2.1655917167663574,
      "logps/chosen": -278.2762145996094,
      "logps/rejected": -246.7991180419922,
      "loss": 2588.1338,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.1482178121805191,
      "rewards/margins": 0.030435502529144287,
      "rewards/rejected": -0.1786532998085022,
      "step": 560
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8639633747547417e-06,
      "logits/chosen": -2.3924612998962402,
      "logits/rejected": -2.2275023460388184,
      "logps/chosen": -246.13973999023438,
      "logps/rejected": -217.94775390625,
      "loss": 1937.843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1699463427066803,
      "rewards/margins": 0.07444851100444794,
      "rewards/rejected": -0.24439485371112823,
      "step": 570
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8966644865925443e-06,
      "logits/chosen": -2.2448031902313232,
      "logits/rejected": -2.270545482635498,
      "logps/chosen": -259.0118103027344,
      "logps/rejected": -274.3431701660156,
      "loss": 2362.1373,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.19803020358085632,
      "rewards/margins": 0.03936942294239998,
      "rewards/rejected": -0.237399622797966,
      "step": 580
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9293655984303466e-06,
      "logits/chosen": -2.6608781814575195,
      "logits/rejected": -2.240119457244873,
      "logps/chosen": -299.0372009277344,
      "logps/rejected": -222.8275604248047,
      "loss": 2179.9811,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.160709410905838,
      "rewards/margins": 0.06795064359903336,
      "rewards/rejected": -0.22866006195545197,
      "step": 590
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9620667102681494e-06,
      "logits/chosen": -2.405942916870117,
      "logits/rejected": -2.4169788360595703,
      "logps/chosen": -203.57545471191406,
      "logps/rejected": -193.13299560546875,
      "loss": 2250.0029,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1432301253080368,
      "rewards/margins": 0.03878119960427284,
      "rewards/rejected": -0.18201133608818054,
      "step": 600
    },
    {
      "epoch": 0.04,
      "eval_logits/chosen": -2.368892192840576,
      "eval_logits/rejected": -2.1789205074310303,
      "eval_logps/chosen": -246.8838653564453,
      "eval_logps/rejected": -232.47830200195312,
      "eval_loss": 2123.84716796875,
      "eval_rewards/accuracies": 0.6100000143051147,
      "eval_rewards/chosen": -0.148789182305336,
      "eval_rewards/margins": 0.0598747693002224,
      "eval_rewards/rejected": -0.2086639702320099,
      "eval_runtime": 721.6425,
      "eval_samples_per_second": 2.771,
      "eval_steps_per_second": 1.386,
      "step": 600
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.994767822105952e-06,
      "logits/chosen": -2.322364568710327,
      "logits/rejected": -2.221156597137451,
      "logps/chosen": -189.34378051757812,
      "logps/rejected": -200.86306762695312,
      "loss": 2275.9445,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.14140108227729797,
      "rewards/margins": 0.050842929631471634,
      "rewards/rejected": -0.19224399328231812,
      "step": 610
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0274689339437543e-06,
      "logits/chosen": -2.2486069202423096,
      "logits/rejected": -2.034440517425537,
      "logps/chosen": -289.38165283203125,
      "logps/rejected": -260.887939453125,
      "loss": 2170.768,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10691903531551361,
      "rewards/margins": 0.06860539317131042,
      "rewards/rejected": -0.17552444338798523,
      "step": 620
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0601700457815567e-06,
      "logits/chosen": -2.3933961391448975,
      "logits/rejected": -2.0247488021850586,
      "logps/chosen": -274.1460876464844,
      "logps/rejected": -234.80636596679688,
      "loss": 2300.2445,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14791390299797058,
      "rewards/margins": 0.04173479601740837,
      "rewards/rejected": -0.18964870274066925,
      "step": 630
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.092871157619359e-06,
      "logits/chosen": -2.4423108100891113,
      "logits/rejected": -2.2551608085632324,
      "logps/chosen": -188.18045043945312,
      "logps/rejected": -207.5229949951172,
      "loss": 2135.9152,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.11518283188343048,
      "rewards/margins": 0.05631045252084732,
      "rewards/rejected": -0.171493262052536,
      "step": 640
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.1255722694571616e-06,
      "logits/chosen": -2.459855318069458,
      "logits/rejected": -2.059840202331543,
      "logps/chosen": -287.207275390625,
      "logps/rejected": -203.07473754882812,
      "loss": 1968.3957,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08312606811523438,
      "rewards/margins": 0.0883508175611496,
      "rewards/rejected": -0.17147688567638397,
      "step": 650
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.158273381294964e-06,
      "logits/chosen": -2.366745710372925,
      "logits/rejected": -2.262253999710083,
      "logps/chosen": -221.7314910888672,
      "logps/rejected": -228.86129760742188,
      "loss": 2580.177,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.09304489940404892,
      "rewards/margins": 0.016183609142899513,
      "rewards/rejected": -0.10922850668430328,
      "step": 660
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.190974493132767e-06,
      "logits/chosen": -2.373584270477295,
      "logits/rejected": -2.0687379837036133,
      "logps/chosen": -247.98617553710938,
      "logps/rejected": -198.99374389648438,
      "loss": 1987.7182,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10307125747203827,
      "rewards/margins": 0.07264357805252075,
      "rewards/rejected": -0.17571485042572021,
      "step": 670
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.223675604970569e-06,
      "logits/chosen": -2.4493911266326904,
      "logits/rejected": -2.242863178253174,
      "logps/chosen": -219.24533081054688,
      "logps/rejected": -210.0750274658203,
      "loss": 2008.7635,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08511056005954742,
      "rewards/margins": 0.08159992098808289,
      "rewards/rejected": -0.16671046614646912,
      "step": 680
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.2563767168083718e-06,
      "logits/chosen": -2.531895637512207,
      "logits/rejected": -2.030471086502075,
      "logps/chosen": -260.61981201171875,
      "logps/rejected": -208.0525665283203,
      "loss": 1935.4762,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.033680204302072525,
      "rewards/margins": 0.10465041548013687,
      "rewards/rejected": -0.1383306086063385,
      "step": 690
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.289077828646174e-06,
      "logits/chosen": -2.364795684814453,
      "logits/rejected": -2.1295928955078125,
      "logps/chosen": -247.4524383544922,
      "logps/rejected": -228.5208740234375,
      "loss": 2168.8156,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08391450345516205,
      "rewards/margins": 0.05774035304784775,
      "rewards/rejected": -0.1416548639535904,
      "step": 700
    },
    {
      "epoch": 0.05,
      "eval_logits/chosen": -2.397294759750366,
      "eval_logits/rejected": -2.2053017616271973,
      "eval_logps/chosen": -235.78334045410156,
      "eval_logps/rejected": -222.8819122314453,
      "eval_loss": 2054.685546875,
      "eval_rewards/accuracies": 0.6205000281333923,
      "eval_rewards/chosen": -0.03778376802802086,
      "eval_rewards/margins": 0.07491618394851685,
      "eval_rewards/rejected": -0.112699955701828,
      "eval_runtime": 722.0761,
      "eval_samples_per_second": 2.77,
      "eval_steps_per_second": 1.385,
      "step": 700
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.3217789404839766e-06,
      "logits/chosen": -2.249499559402466,
      "logits/rejected": -2.334920883178711,
      "logps/chosen": -163.0601348876953,
      "logps/rejected": -213.415283203125,
      "loss": 2444.5898,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.06229011341929436,
      "rewards/margins": 0.03412432223558426,
      "rewards/rejected": -0.09641443192958832,
      "step": 710
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.354480052321779e-06,
      "logits/chosen": -2.586360454559326,
      "logits/rejected": -2.149442434310913,
      "logps/chosen": -260.3407287597656,
      "logps/rejected": -210.86669921875,
      "loss": 1814.7633,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.019512873142957687,
      "rewards/margins": 0.1016731709241867,
      "rewards/rejected": -0.12118605524301529,
      "step": 720
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.3871811641595815e-06,
      "logits/chosen": -2.410372257232666,
      "logits/rejected": -2.2249979972839355,
      "logps/chosen": -247.38467407226562,
      "logps/rejected": -198.99746704101562,
      "loss": 1741.6162,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.005922236945480108,
      "rewards/margins": 0.10268674045801163,
      "rewards/rejected": -0.10860898345708847,
      "step": 730
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4198822759973843e-06,
      "logits/chosen": -2.2442455291748047,
      "logits/rejected": -2.2593936920166016,
      "logps/chosen": -194.4607696533203,
      "logps/rejected": -212.80746459960938,
      "loss": 2339.5994,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.03854719549417496,
      "rewards/margins": 0.04414154589176178,
      "rewards/rejected": -0.08268874883651733,
      "step": 740
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4525833878351864e-06,
      "logits/chosen": -2.5059237480163574,
      "logits/rejected": -2.302354574203491,
      "logps/chosen": -254.85604858398438,
      "logps/rejected": -196.12966918945312,
      "loss": 2093.2186,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.020551379770040512,
      "rewards/margins": 0.07553622126579285,
      "rewards/rejected": -0.09608760476112366,
      "step": 750
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4852844996729892e-06,
      "logits/chosen": -2.2689011096954346,
      "logits/rejected": -2.174787998199463,
      "logps/chosen": -240.7899932861328,
      "logps/rejected": -254.06173706054688,
      "loss": 1940.8283,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.016656741499900818,
      "rewards/margins": 0.11813600361347198,
      "rewards/rejected": -0.1347927302122116,
      "step": 760
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5179856115107916e-06,
      "logits/chosen": -2.435136318206787,
      "logits/rejected": -2.0990397930145264,
      "logps/chosen": -273.72161865234375,
      "logps/rejected": -228.80355834960938,
      "loss": 1954.7613,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.002254198770970106,
      "rewards/margins": 0.09030239284038544,
      "rewards/rejected": -0.09255659580230713,
      "step": 770
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5506867233485937e-06,
      "logits/chosen": -2.5231940746307373,
      "logits/rejected": -2.090528964996338,
      "logps/chosen": -262.9032897949219,
      "logps/rejected": -220.0452880859375,
      "loss": 1513.9102,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.027926484122872353,
      "rewards/margins": 0.14640609920024872,
      "rewards/rejected": -0.11847962439060211,
      "step": 780
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5833878351863965e-06,
      "logits/chosen": -2.551626205444336,
      "logits/rejected": -2.480762004852295,
      "logps/chosen": -253.04141235351562,
      "logps/rejected": -260.40289306640625,
      "loss": 2008.6934,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.010434931144118309,
      "rewards/margins": 0.09981861710548401,
      "rewards/rejected": -0.11025355011224747,
      "step": 790
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.616088947024199e-06,
      "logits/chosen": -2.2253010272979736,
      "logits/rejected": -2.3965466022491455,
      "logps/chosen": -202.80874633789062,
      "logps/rejected": -232.88693237304688,
      "loss": 2370.3576,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06319282203912735,
      "rewards/margins": 0.055518168956041336,
      "rewards/rejected": -0.11871097981929779,
      "step": 800
    },
    {
      "epoch": 0.05,
      "eval_logits/chosen": -2.426872968673706,
      "eval_logits/rejected": -2.2322709560394287,
      "eval_logps/chosen": -237.03123474121094,
      "eval_logps/rejected": -226.0986785888672,
      "eval_loss": 1997.0155029296875,
      "eval_rewards/accuracies": 0.6140000224113464,
      "eval_rewards/chosen": -0.050262872129678726,
      "eval_rewards/margins": 0.0946047306060791,
      "eval_rewards/rejected": -0.14486758410930634,
      "eval_runtime": 722.8029,
      "eval_samples_per_second": 2.767,
      "eval_steps_per_second": 1.384,
      "step": 800
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6487900588620014e-06,
      "logits/chosen": -2.209472179412842,
      "logits/rejected": -1.9674603939056396,
      "logps/chosen": -203.85731506347656,
      "logps/rejected": -167.39962768554688,
      "loss": 2191.499,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03741743415594101,
      "rewards/margins": 0.05794162303209305,
      "rewards/rejected": -0.09535904973745346,
      "step": 810
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6814911706998042e-06,
      "logits/chosen": -2.3804078102111816,
      "logits/rejected": -2.196342945098877,
      "logps/chosen": -226.281494140625,
      "logps/rejected": -201.8038330078125,
      "loss": 2224.7926,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.009422911331057549,
      "rewards/margins": 0.06276295334100723,
      "rewards/rejected": -0.07218586653470993,
      "step": 820
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.7141922825376067e-06,
      "logits/chosen": -2.267695665359497,
      "logits/rejected": -2.2699947357177734,
      "logps/chosen": -261.9485168457031,
      "logps/rejected": -283.9317626953125,
      "loss": 1690.7559,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.040243808180093765,
      "rewards/margins": 0.12052266299724579,
      "rewards/rejected": -0.16076649725437164,
      "step": 830
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.746893394375409e-06,
      "logits/chosen": -2.3985633850097656,
      "logits/rejected": -2.2916035652160645,
      "logps/chosen": -238.35885620117188,
      "logps/rejected": -246.00900268554688,
      "loss": 2056.7525,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03964471071958542,
      "rewards/margins": 0.09431429952383041,
      "rewards/rejected": -0.13395901024341583,
      "step": 840
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.779594506213211e-06,
      "logits/chosen": -2.320394992828369,
      "logits/rejected": -2.0796637535095215,
      "logps/chosen": -197.7746124267578,
      "logps/rejected": -204.3472900390625,
      "loss": 2249.3328,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0791572779417038,
      "rewards/margins": 0.06885950267314911,
      "rewards/rejected": -0.1480167806148529,
      "step": 850
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.812295618051014e-06,
      "logits/chosen": -2.4867045879364014,
      "logits/rejected": -2.3241653442382812,
      "logps/chosen": -275.2378234863281,
      "logps/rejected": -225.60928344726562,
      "loss": 1811.7268,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.05723363161087036,
      "rewards/margins": 0.12098623812198639,
      "rewards/rejected": -0.06375260651111603,
      "step": 860
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.8449967298888164e-06,
      "logits/chosen": -2.4043803215026855,
      "logits/rejected": -2.1745409965515137,
      "logps/chosen": -176.84385681152344,
      "logps/rejected": -154.92471313476562,
      "loss": 2119.0828,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.00019865883223246783,
      "rewards/margins": 0.061392270028591156,
      "rewards/rejected": -0.061590928584337234,
      "step": 870
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.877697841726619e-06,
      "logits/chosen": -2.478651285171509,
      "logits/rejected": -2.372340679168701,
      "logps/chosen": -216.61239624023438,
      "logps/rejected": -206.92984008789062,
      "loss": 2108.6172,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.022234294563531876,
      "rewards/margins": 0.07132290303707123,
      "rewards/rejected": -0.049088604748249054,
      "step": 880
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.9103989535644217e-06,
      "logits/chosen": -2.387673854827881,
      "logits/rejected": -2.503478765487671,
      "logps/chosen": -216.41189575195312,
      "logps/rejected": -253.4788818359375,
      "loss": 2194.2061,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.00928253959864378,
      "rewards/margins": 0.06536583602428436,
      "rewards/rejected": -0.07464838773012161,
      "step": 890
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.943100065402224e-06,
      "logits/chosen": -2.428986072540283,
      "logits/rejected": -2.278474807739258,
      "logps/chosen": -293.5047302246094,
      "logps/rejected": -284.4921875,
      "loss": 2219.9355,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.008615334518253803,
      "rewards/margins": 0.0630418211221695,
      "rewards/rejected": -0.07165715098381042,
      "step": 900
    },
    {
      "epoch": 0.06,
      "eval_logits/chosen": -2.4120709896087646,
      "eval_logits/rejected": -2.220082998275757,
      "eval_logps/chosen": -229.2795867919922,
      "eval_logps/rejected": -217.4352569580078,
      "eval_loss": 2038.843994140625,
      "eval_rewards/accuracies": 0.6200000047683716,
      "eval_rewards/chosen": 0.02725347876548767,
      "eval_rewards/margins": 0.08548705279827118,
      "eval_rewards/rejected": -0.058233581483364105,
      "eval_runtime": 724.9078,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.379,
      "step": 900
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.9758011772400266e-06,
      "logits/chosen": -2.322211265563965,
      "logits/rejected": -2.34004282951355,
      "logps/chosen": -253.59512329101562,
      "logps/rejected": -267.9804382324219,
      "loss": 2021.3826,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03714027255773544,
      "rewards/margins": 0.07010693103075027,
      "rewards/rejected": -0.03296665847301483,
      "step": 910
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.0085022890778286e-06,
      "logits/chosen": -2.3901853561401367,
      "logits/rejected": -2.1437230110168457,
      "logps/chosen": -174.85421752929688,
      "logps/rejected": -155.4096221923828,
      "loss": 2090.9529,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.008853385224938393,
      "rewards/margins": 0.07478860765695572,
      "rewards/rejected": -0.06593521684408188,
      "step": 920
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.0412034009156314e-06,
      "logits/chosen": -2.3286807537078857,
      "logits/rejected": -2.497436285018921,
      "logps/chosen": -246.6853485107422,
      "logps/rejected": -249.7530975341797,
      "loss": 2147.3672,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.109990693628788,
      "rewards/margins": 0.0732945054769516,
      "rewards/rejected": -0.183285191655159,
      "step": 930
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.073904512753434e-06,
      "logits/chosen": -2.501011610031128,
      "logits/rejected": -2.1241161823272705,
      "logps/chosen": -247.611083984375,
      "logps/rejected": -225.5044708251953,
      "loss": 1653.3525,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17962466180324554,
      "rewards/margins": 0.12821651995182037,
      "rewards/rejected": -0.30784112215042114,
      "step": 940
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1066056245912363e-06,
      "logits/chosen": -2.373318672180176,
      "logits/rejected": -2.4368274211883545,
      "logps/chosen": -256.42279052734375,
      "logps/rejected": -234.74411010742188,
      "loss": 1920.7047,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.25438791513442993,
      "rewards/margins": 0.10280318558216095,
      "rewards/rejected": -0.3571911156177521,
      "step": 950
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1393067364290387e-06,
      "logits/chosen": -2.382197618484497,
      "logits/rejected": -2.1782593727111816,
      "logps/chosen": -240.29061889648438,
      "logps/rejected": -214.4322509765625,
      "loss": 1933.4619,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15163874626159668,
      "rewards/margins": 0.12727205455303192,
      "rewards/rejected": -0.2789107859134674,
      "step": 960
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1720078482668416e-06,
      "logits/chosen": -2.3237643241882324,
      "logits/rejected": -2.1025829315185547,
      "logps/chosen": -224.8642120361328,
      "logps/rejected": -191.6042938232422,
      "loss": 1851.0059,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13293561339378357,
      "rewards/margins": 0.1384698450565338,
      "rewards/rejected": -0.27140548825263977,
      "step": 970
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.204708960104644e-06,
      "logits/chosen": -2.436340808868408,
      "logits/rejected": -2.092273473739624,
      "logps/chosen": -223.2788848876953,
      "logps/rejected": -208.0653076171875,
      "loss": 1677.9363,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05876126140356064,
      "rewards/margins": 0.17374159395694733,
      "rewards/rejected": -0.23250284790992737,
      "step": 980
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.237410071942446e-06,
      "logits/chosen": -2.4342200756073,
      "logits/rejected": -2.1691014766693115,
      "logps/chosen": -215.2308349609375,
      "logps/rejected": -184.5440673828125,
      "loss": 2402.4279,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08869779109954834,
      "rewards/margins": 0.02877037599682808,
      "rewards/rejected": -0.11746816337108612,
      "step": 990
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.270111183780249e-06,
      "logits/chosen": -2.343592405319214,
      "logits/rejected": -2.1235079765319824,
      "logps/chosen": -205.7447509765625,
      "logps/rejected": -195.60147094726562,
      "loss": 2382.9531,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.05721202492713928,
      "rewards/margins": 0.07859710603952408,
      "rewards/rejected": -0.13580910861492157,
      "step": 1000
    },
    {
      "epoch": 0.07,
      "eval_logits/chosen": -2.370666027069092,
      "eval_logits/rejected": -2.1831679344177246,
      "eval_logps/chosen": -232.2192840576172,
      "eval_logps/rejected": -221.19912719726562,
      "eval_loss": 2008.1322021484375,
      "eval_rewards/accuracies": 0.6215000152587891,
      "eval_rewards/chosen": -0.0021434456575661898,
      "eval_rewards/margins": 0.09372860938310623,
      "eval_rewards/rejected": -0.09587204456329346,
      "eval_runtime": 725.0382,
      "eval_samples_per_second": 2.758,
      "eval_steps_per_second": 1.379,
      "step": 1000
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.3028122956180513e-06,
      "logits/chosen": -2.2390599250793457,
      "logits/rejected": -2.2753896713256836,
      "logps/chosen": -210.17648315429688,
      "logps/rejected": -244.66043090820312,
      "loss": 2124.2566,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.019214505329728127,
      "rewards/margins": 0.07015024125576019,
      "rewards/rejected": -0.05093574523925781,
      "step": 1010
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.3355134074558538e-06,
      "logits/chosen": -2.24173641204834,
      "logits/rejected": -2.1411194801330566,
      "logps/chosen": -221.272705078125,
      "logps/rejected": -204.73448181152344,
      "loss": 2000.4838,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.005399542860686779,
      "rewards/margins": 0.09049306809902191,
      "rewards/rejected": -0.09589260816574097,
      "step": 1020
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.368214519293656e-06,
      "logits/chosen": -2.191763162612915,
      "logits/rejected": -1.9984468221664429,
      "logps/chosen": -196.1796417236328,
      "logps/rejected": -185.42257690429688,
      "loss": 1851.1479,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.027906540781259537,
      "rewards/margins": 0.10141406208276749,
      "rewards/rejected": -0.07350752502679825,
      "step": 1030
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.400915631131459e-06,
      "logits/chosen": -2.1586947441101074,
      "logits/rejected": -2.190293788909912,
      "logps/chosen": -200.91342163085938,
      "logps/rejected": -240.82846069335938,
      "loss": 1875.4949,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.029409628361463547,
      "rewards/margins": 0.11263708025217056,
      "rewards/rejected": -0.142046719789505,
      "step": 1040
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.4336167429692615e-06,
      "logits/chosen": -2.405604839324951,
      "logits/rejected": -2.1920061111450195,
      "logps/chosen": -215.37515258789062,
      "logps/rejected": -208.7307891845703,
      "loss": 1940.4701,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.008247396908700466,
      "rewards/margins": 0.09609442949295044,
      "rewards/rejected": -0.10434182733297348,
      "step": 1050
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.4663178548070635e-06,
      "logits/chosen": -2.2052693367004395,
      "logits/rejected": -2.268381357192993,
      "logps/chosen": -213.3984375,
      "logps/rejected": -206.3799591064453,
      "loss": 1942.6387,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.015555182471871376,
      "rewards/margins": 0.09423165023326874,
      "rewards/rejected": -0.07867647707462311,
      "step": 1060
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.499018966644866e-06,
      "logits/chosen": -2.344458818435669,
      "logits/rejected": -2.147507905960083,
      "logps/chosen": -193.80804443359375,
      "logps/rejected": -196.10009765625,
      "loss": 2087.9314,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.019830048084259033,
      "rewards/margins": 0.09104426950216293,
      "rewards/rejected": -0.11087431758642197,
      "step": 1070
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.531720078482669e-06,
      "logits/chosen": -2.2609543800354004,
      "logits/rejected": -1.932922124862671,
      "logps/chosen": -232.9537353515625,
      "logps/rejected": -231.40316772460938,
      "loss": 2026.6357,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07084006816148758,
      "rewards/margins": 0.09680701792240143,
      "rewards/rejected": -0.1676470935344696,
      "step": 1080
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.5644211903204712e-06,
      "logits/chosen": -2.2446954250335693,
      "logits/rejected": -2.2965035438537598,
      "logps/chosen": -212.11300659179688,
      "logps/rejected": -223.2448272705078,
      "loss": 1987.2455,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13230548799037933,
      "rewards/margins": 0.08318330347537994,
      "rewards/rejected": -0.2154887616634369,
      "step": 1090
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.5971223021582737e-06,
      "logits/chosen": -2.4021565914154053,
      "logits/rejected": -2.26518177986145,
      "logps/chosen": -273.38848876953125,
      "logps/rejected": -225.50161743164062,
      "loss": 2155.5914,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.17587730288505554,
      "rewards/margins": 0.09228373318910599,
      "rewards/rejected": -0.26816102862358093,
      "step": 1100
    },
    {
      "epoch": 0.07,
      "eval_logits/chosen": -2.2892866134643555,
      "eval_logits/rejected": -2.106736898422241,
      "eval_logps/chosen": -252.53115844726562,
      "eval_logps/rejected": -241.62571716308594,
      "eval_loss": 2003.248046875,
      "eval_rewards/accuracies": 0.6244999766349792,
      "eval_rewards/chosen": -0.20526205003261566,
      "eval_rewards/margins": 0.09487606585025787,
      "eval_rewards/rejected": -0.30013811588287354,
      "eval_runtime": 725.6609,
      "eval_samples_per_second": 2.756,
      "eval_steps_per_second": 1.378,
      "step": 1100
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.6298234139960765e-06,
      "logits/chosen": -2.367889165878296,
      "logits/rejected": -2.0205140113830566,
      "logps/chosen": -238.40225219726562,
      "logps/rejected": -195.3968048095703,
      "loss": 2193.3691,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.20633168518543243,
      "rewards/margins": 0.06665893644094467,
      "rewards/rejected": -0.2729905843734741,
      "step": 1110
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.6625245258338785e-06,
      "logits/chosen": -2.1692986488342285,
      "logits/rejected": -2.098557949066162,
      "logps/chosen": -257.96173095703125,
      "logps/rejected": -344.1214294433594,
      "loss": 1914.0918,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.19183330237865448,
      "rewards/margins": 0.09699074178934097,
      "rewards/rejected": -0.28882405161857605,
      "step": 1120
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.695225637671681e-06,
      "logits/chosen": -2.4231133460998535,
      "logits/rejected": -2.2229857444763184,
      "logps/chosen": -227.0390625,
      "logps/rejected": -200.2311553955078,
      "loss": 2141.0023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19762437045574188,
      "rewards/margins": 0.06268110871315002,
      "rewards/rejected": -0.2603054940700531,
      "step": 1130
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.7279267495094834e-06,
      "logits/chosen": -2.34079909324646,
      "logits/rejected": -2.2142493724823,
      "logps/chosen": -173.04766845703125,
      "logps/rejected": -194.06484985351562,
      "loss": 1875.1215,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1262076050043106,
      "rewards/margins": 0.10891900211572647,
      "rewards/rejected": -0.2351265847682953,
      "step": 1140
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.7606278613472863e-06,
      "logits/chosen": -2.3947463035583496,
      "logits/rejected": -2.054185628890991,
      "logps/chosen": -288.93695068359375,
      "logps/rejected": -225.27359008789062,
      "loss": 1889.7932,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06665666401386261,
      "rewards/margins": 0.139047309756279,
      "rewards/rejected": -0.205704003572464,
      "step": 1150
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.7933289731850887e-06,
      "logits/chosen": -2.2144577503204346,
      "logits/rejected": -1.9980432987213135,
      "logps/chosen": -229.6927490234375,
      "logps/rejected": -215.7831573486328,
      "loss": 1700.6029,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.011321134865283966,
      "rewards/margins": 0.14348730444908142,
      "rewards/rejected": -0.1548084318637848,
      "step": 1160
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.826030085022891e-06,
      "logits/chosen": -2.375175952911377,
      "logits/rejected": -2.146986484527588,
      "logps/chosen": -247.9508056640625,
      "logps/rejected": -220.3432159423828,
      "loss": 2110.4133,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.003009948180988431,
      "rewards/margins": 0.08544488996267319,
      "rewards/rejected": -0.08845484256744385,
      "step": 1170
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.858731196860693e-06,
      "logits/chosen": -2.422227382659912,
      "logits/rejected": -1.972865343093872,
      "logps/chosen": -248.3476104736328,
      "logps/rejected": -211.3714599609375,
      "loss": 2249.0883,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.024858513846993446,
      "rewards/margins": 0.05259479209780693,
      "rewards/rejected": -0.02773628570139408,
      "step": 1180
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.891432308698496e-06,
      "logits/chosen": -2.1258654594421387,
      "logits/rejected": -2.0728182792663574,
      "logps/chosen": -191.03858947753906,
      "logps/rejected": -224.00015258789062,
      "loss": 2100.7717,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04191805049777031,
      "rewards/margins": 0.0771632045507431,
      "rewards/rejected": -0.03524515777826309,
      "step": 1190
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.924133420536299e-06,
      "logits/chosen": -2.238049030303955,
      "logits/rejected": -2.0524375438690186,
      "logps/chosen": -180.01011657714844,
      "logps/rejected": -153.81248474121094,
      "loss": 2408.5309,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.02134573832154274,
      "rewards/margins": 0.07205002009868622,
      "rewards/rejected": -0.050704289227724075,
      "step": 1200
    },
    {
      "epoch": 0.08,
      "eval_logits/chosen": -2.3281822204589844,
      "eval_logits/rejected": -2.142608880996704,
      "eval_logps/chosen": -228.13014221191406,
      "eval_logps/rejected": -217.15615844726562,
      "eval_loss": 1978.77734375,
      "eval_rewards/accuracies": 0.640500009059906,
      "eval_rewards/chosen": 0.038747940212488174,
      "eval_rewards/margins": 0.09419067949056625,
      "eval_rewards/rejected": -0.055442746728658676,
      "eval_runtime": 728.1646,
      "eval_samples_per_second": 2.747,
      "eval_steps_per_second": 1.373,
      "step": 1200
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.956834532374101e-06,
      "logits/chosen": -2.434722661972046,
      "logits/rejected": -2.054490566253662,
      "logps/chosen": -200.15614318847656,
      "logps/rejected": -170.80064392089844,
      "loss": 1546.1736,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.051065344363451004,
      "rewards/margins": 0.14595820009708405,
      "rewards/rejected": -0.09489286690950394,
      "step": 1210
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.989535644211904e-06,
      "logits/chosen": -2.2990026473999023,
      "logits/rejected": -2.0386624336242676,
      "logps/chosen": -221.647705078125,
      "logps/rejected": -209.01327514648438,
      "loss": 2074.093,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04520409181714058,
      "rewards/margins": 0.0998666062951088,
      "rewards/rejected": -0.14507070183753967,
      "step": 1220
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.022236756049706e-06,
      "logits/chosen": -2.4798145294189453,
      "logits/rejected": -2.0929503440856934,
      "logps/chosen": -281.10369873046875,
      "logps/rejected": -259.5100402832031,
      "loss": 2039.6855,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.039665382355451584,
      "rewards/margins": 0.10881809145212173,
      "rewards/rejected": -0.1484834849834442,
      "step": 1230
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.054937867887509e-06,
      "logits/chosen": -2.3605129718780518,
      "logits/rejected": -1.9098589420318604,
      "logps/chosen": -224.52041625976562,
      "logps/rejected": -230.14990234375,
      "loss": 1669.4992,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.002536333166062832,
      "rewards/margins": 0.16417892277240753,
      "rewards/rejected": -0.16671524941921234,
      "step": 1240
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.087638979725311e-06,
      "logits/chosen": -2.280735492706299,
      "logits/rejected": -2.3474297523498535,
      "logps/chosen": -223.4026336669922,
      "logps/rejected": -233.1063690185547,
      "loss": 2110.8816,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04437154531478882,
      "rewards/margins": 0.08248762786388397,
      "rewards/rejected": -0.03811608999967575,
      "step": 1250
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.1203400915631135e-06,
      "logits/chosen": -2.1865649223327637,
      "logits/rejected": -2.1238932609558105,
      "logps/chosen": -233.7169189453125,
      "logps/rejected": -214.2218017578125,
      "loss": 2116.418,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03614087775349617,
      "rewards/margins": 0.0878569632768631,
      "rewards/rejected": -0.05171608179807663,
      "step": 1260
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.153041203400916e-06,
      "logits/chosen": -2.4024250507354736,
      "logits/rejected": -2.246831178665161,
      "logps/chosen": -256.627685546875,
      "logps/rejected": -238.527099609375,
      "loss": 2156.5352,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0779462456703186,
      "rewards/margins": 0.06534029543399811,
      "rewards/rejected": 0.012605957686901093,
      "step": 1270
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.185742315238718e-06,
      "logits/chosen": -2.307811737060547,
      "logits/rejected": -2.0606985092163086,
      "logps/chosen": -194.02505493164062,
      "logps/rejected": -188.58154296875,
      "loss": 2189.1459,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.04076526314020157,
      "rewards/margins": 0.06423444300889969,
      "rewards/rejected": -0.02346917614340782,
      "step": 1280
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.218443427076521e-06,
      "logits/chosen": -2.2853074073791504,
      "logits/rejected": -2.1252546310424805,
      "logps/chosen": -181.89572143554688,
      "logps/rejected": -189.04368591308594,
      "loss": 2332.007,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.008628499694168568,
      "rewards/margins": 0.05676931142807007,
      "rewards/rejected": -0.048140816390514374,
      "step": 1290
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.251144538914323e-06,
      "logits/chosen": -2.245187997817993,
      "logits/rejected": -2.1047472953796387,
      "logps/chosen": -223.224853515625,
      "logps/rejected": -216.79763793945312,
      "loss": 2340.302,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.026345735415816307,
      "rewards/margins": 0.06768472492694855,
      "rewards/rejected": -0.0940304696559906,
      "step": 1300
    },
    {
      "epoch": 0.09,
      "eval_logits/chosen": -2.355750799179077,
      "eval_logits/rejected": -2.1680800914764404,
      "eval_logps/chosen": -228.8513641357422,
      "eval_logps/rejected": -217.4835205078125,
      "eval_loss": 1959.7281494140625,
      "eval_rewards/accuracies": 0.6445000171661377,
      "eval_rewards/chosen": 0.031535882502794266,
      "eval_rewards/margins": 0.09025205671787262,
      "eval_rewards/rejected": -0.058716174215078354,
      "eval_runtime": 725.6305,
      "eval_samples_per_second": 2.756,
      "eval_steps_per_second": 1.378,
      "step": 1300
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.283845650752126e-06,
      "logits/chosen": -2.388261079788208,
      "logits/rejected": -2.1793575286865234,
      "logps/chosen": -282.85009765625,
      "logps/rejected": -228.8882293701172,
      "loss": 1980.2102,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03361871466040611,
      "rewards/margins": 0.10229597240686417,
      "rewards/rejected": -0.06867726892232895,
      "step": 1310
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.316546762589928e-06,
      "logits/chosen": -2.3317654132843018,
      "logits/rejected": -2.24820876121521,
      "logps/chosen": -204.1890411376953,
      "logps/rejected": -178.5457763671875,
      "loss": 1923.8402,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03738489747047424,
      "rewards/margins": 0.0865490585565567,
      "rewards/rejected": -0.04916415363550186,
      "step": 1320
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.349247874427731e-06,
      "logits/chosen": -2.3867275714874268,
      "logits/rejected": -2.0918281078338623,
      "logps/chosen": -224.2469482421875,
      "logps/rejected": -274.74603271484375,
      "loss": 2061.5996,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.022870704531669617,
      "rewards/margins": 0.10416515171527863,
      "rewards/rejected": -0.08129443228244781,
      "step": 1330
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.381948986265534e-06,
      "logits/chosen": -2.535256862640381,
      "logits/rejected": -2.2696471214294434,
      "logps/chosen": -263.3232727050781,
      "logps/rejected": -261.365966796875,
      "loss": 1796.1584,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.004785762168467045,
      "rewards/margins": 0.11049681901931763,
      "rewards/rejected": -0.1152825802564621,
      "step": 1340
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.414650098103336e-06,
      "logits/chosen": -2.6866261959075928,
      "logits/rejected": -2.4202332496643066,
      "logps/chosen": -257.3060607910156,
      "logps/rejected": -244.3971405029297,
      "loss": 1999.8973,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.046464331448078156,
      "rewards/margins": 0.09515171498060226,
      "rewards/rejected": -0.04868737980723381,
      "step": 1350
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.447351209941138e-06,
      "logits/chosen": -2.4166512489318848,
      "logits/rejected": -2.1275861263275146,
      "logps/chosen": -217.0367431640625,
      "logps/rejected": -207.73403930664062,
      "loss": 2038.8555,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07372106611728668,
      "rewards/margins": 0.08838832378387451,
      "rewards/rejected": -0.162109375,
      "step": 1360
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.480052321778941e-06,
      "logits/chosen": -2.358241081237793,
      "logits/rejected": -2.3770785331726074,
      "logps/chosen": -231.0319061279297,
      "logps/rejected": -217.2124481201172,
      "loss": 2188.2721,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0412161760032177,
      "rewards/margins": 0.06876528263092041,
      "rewards/rejected": -0.10998145490884781,
      "step": 1370
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.5127534336167435e-06,
      "logits/chosen": -2.4255921840667725,
      "logits/rejected": -2.143770933151245,
      "logps/chosen": -252.64108276367188,
      "logps/rejected": -215.73745727539062,
      "loss": 2182.915,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05467110127210617,
      "rewards/margins": 0.06756436824798584,
      "rewards/rejected": -0.12223546206951141,
      "step": 1380
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.5454545454545455e-06,
      "logits/chosen": -2.3622450828552246,
      "logits/rejected": -2.3135733604431152,
      "logps/chosen": -173.19479370117188,
      "logps/rejected": -171.76528930664062,
      "loss": 2035.6379,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.037360869348049164,
      "rewards/margins": 0.08779537677764893,
      "rewards/rejected": -0.1251562535762787,
      "step": 1390
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.578155657292348e-06,
      "logits/chosen": -2.3347394466400146,
      "logits/rejected": -2.300281524658203,
      "logps/chosen": -255.4662322998047,
      "logps/rejected": -282.37481689453125,
      "loss": 1628.1867,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0006058543804101646,
      "rewards/margins": 0.17986665666103363,
      "rewards/rejected": -0.1804725080728531,
      "step": 1400
    },
    {
      "epoch": 0.09,
      "eval_logits/chosen": -2.3342952728271484,
      "eval_logits/rejected": -2.1490938663482666,
      "eval_logps/chosen": -234.87429809570312,
      "eval_logps/rejected": -225.39889526367188,
      "eval_loss": 1933.371337890625,
      "eval_rewards/accuracies": 0.6424999833106995,
      "eval_rewards/chosen": -0.0286934282630682,
      "eval_rewards/margins": 0.1091766282916069,
      "eval_rewards/rejected": -0.13787005841732025,
      "eval_runtime": 726.3679,
      "eval_samples_per_second": 2.753,
      "eval_steps_per_second": 1.377,
      "step": 1400
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.610856769130151e-06,
      "logits/chosen": -2.3903706073760986,
      "logits/rejected": -2.2447268962860107,
      "logps/chosen": -240.896240234375,
      "logps/rejected": -226.40408325195312,
      "loss": 1653.2947,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.017519865185022354,
      "rewards/margins": 0.15009410679340363,
      "rewards/rejected": -0.16761396825313568,
      "step": 1410
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.643557880967953e-06,
      "logits/chosen": -2.3760390281677246,
      "logits/rejected": -2.1798462867736816,
      "logps/chosen": -205.38589477539062,
      "logps/rejected": -211.92306518554688,
      "loss": 2097.3713,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.06587065756320953,
      "rewards/margins": 0.09036964178085327,
      "rewards/rejected": -0.156240314245224,
      "step": 1420
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.676258992805755e-06,
      "logits/chosen": -2.3314037322998047,
      "logits/rejected": -2.0588929653167725,
      "logps/chosen": -278.0765380859375,
      "logps/rejected": -250.62222290039062,
      "loss": 1547.0062,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08456582576036453,
      "rewards/margins": 0.1469404101371765,
      "rewards/rejected": -0.23150622844696045,
      "step": 1430
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.708960104643558e-06,
      "logits/chosen": -2.3256824016571045,
      "logits/rejected": -2.2896199226379395,
      "logps/chosen": -311.00970458984375,
      "logps/rejected": -285.14251708984375,
      "loss": 2028.9057,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04799982160329819,
      "rewards/margins": 0.10892969369888306,
      "rewards/rejected": -0.15692950785160065,
      "step": 1440
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.741661216481361e-06,
      "logits/chosen": -2.2723355293273926,
      "logits/rejected": -2.2319412231445312,
      "logps/chosen": -241.2588653564453,
      "logps/rejected": -266.7528076171875,
      "loss": 1714.3082,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.049133121967315674,
      "rewards/margins": 0.142476424574852,
      "rewards/rejected": -0.19160954654216766,
      "step": 1450
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.774362328319163e-06,
      "logits/chosen": -2.259704113006592,
      "logits/rejected": -2.1161341667175293,
      "logps/chosen": -218.36032104492188,
      "logps/rejected": -194.8245849609375,
      "loss": 2187.3797,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0257023423910141,
      "rewards/margins": 0.06413718312978745,
      "rewards/rejected": -0.08983953297138214,
      "step": 1460
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.807063440156966e-06,
      "logits/chosen": -2.2820794582366943,
      "logits/rejected": -2.0573856830596924,
      "logps/chosen": -260.0492858886719,
      "logps/rejected": -215.5932159423828,
      "loss": 1615.244,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03260207176208496,
      "rewards/margins": 0.1691434383392334,
      "rewards/rejected": -0.13654136657714844,
      "step": 1470
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.839764551994769e-06,
      "logits/chosen": -2.353740692138672,
      "logits/rejected": -2.0564048290252686,
      "logps/chosen": -251.4329376220703,
      "logps/rejected": -227.07162475585938,
      "loss": 2050.0922,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.027244482189416885,
      "rewards/margins": 0.13372252881526947,
      "rewards/rejected": -0.16096700727939606,
      "step": 1480
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.872465663832571e-06,
      "logits/chosen": -2.231527805328369,
      "logits/rejected": -2.1027674674987793,
      "logps/chosen": -209.59500122070312,
      "logps/rejected": -200.1400909423828,
      "loss": 1905.0725,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.018365005031228065,
      "rewards/margins": 0.11005227267742157,
      "rewards/rejected": -0.12841728329658508,
      "step": 1490
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.905166775670373e-06,
      "logits/chosen": -2.3522300720214844,
      "logits/rejected": -2.024580478668213,
      "logps/chosen": -217.8345184326172,
      "logps/rejected": -205.73214721679688,
      "loss": 1639.7521,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.03925449773669243,
      "rewards/margins": 0.15291200578212738,
      "rewards/rejected": -0.11365751922130585,
      "step": 1500
    },
    {
      "epoch": 0.1,
      "eval_logits/chosen": -2.3127949237823486,
      "eval_logits/rejected": -2.1304714679718018,
      "eval_logps/chosen": -231.580810546875,
      "eval_logps/rejected": -222.56143188476562,
      "eval_loss": 1932.8865966796875,
      "eval_rewards/accuracies": 0.6315000057220459,
      "eval_rewards/chosen": 0.004241608083248138,
      "eval_rewards/margins": 0.11373703181743622,
      "eval_rewards/rejected": -0.10949542373418808,
      "eval_runtime": 721.6104,
      "eval_samples_per_second": 2.772,
      "eval_steps_per_second": 1.386,
      "step": 1500
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.9378678875081756e-06,
      "logits/chosen": -2.4497547149658203,
      "logits/rejected": -2.1537137031555176,
      "logps/chosen": -213.77719116210938,
      "logps/rejected": -193.41302490234375,
      "loss": 1960.867,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.009664192795753479,
      "rewards/margins": 0.1064397469162941,
      "rewards/rejected": -0.09677554666996002,
      "step": 1510
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.9705689993459784e-06,
      "logits/chosen": -2.4343857765197754,
      "logits/rejected": -2.002175807952881,
      "logps/chosen": -208.59097290039062,
      "logps/rejected": -161.232177734375,
      "loss": 2100.7449,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.036656323820352554,
      "rewards/margins": 0.10354284942150116,
      "rewards/rejected": -0.14019916951656342,
      "step": 1520
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999999934793849e-06,
      "logits/chosen": -2.3356330394744873,
      "logits/rejected": -2.236433267593384,
      "logps/chosen": -248.06582641601562,
      "logps/rejected": -229.6756591796875,
      "loss": 2047.9803,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0011256359284743667,
      "rewards/margins": 0.11026228964328766,
      "rewards/rejected": -0.11138790845870972,
      "step": 1530
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999992110059814e-06,
      "logits/chosen": -2.315499782562256,
      "logits/rejected": -2.304673433303833,
      "logps/chosen": -274.79168701171875,
      "logps/rejected": -262.83050537109375,
      "loss": 2078.9848,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02895769476890564,
      "rewards/margins": 0.08152148872613907,
      "rewards/rejected": -0.05256379395723343,
      "step": 1540
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999971244142299e-06,
      "logits/chosen": -2.4524457454681396,
      "logits/rejected": -2.1760833263397217,
      "logps/chosen": -272.97418212890625,
      "logps/rejected": -243.3657989501953,
      "loss": 1978.4383,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.004789206199347973,
      "rewards/margins": 0.09637876600027084,
      "rewards/rejected": -0.0915895476937294,
      "step": 1550
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999937337150149e-06,
      "logits/chosen": -2.1443240642547607,
      "logits/rejected": -2.1270015239715576,
      "logps/chosen": -235.2775421142578,
      "logps/rejected": -230.3560333251953,
      "loss": 2022.7053,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0076267956756055355,
      "rewards/margins": 0.10459224879741669,
      "rewards/rejected": -0.11221903562545776,
      "step": 1560
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.99989038926024e-06,
      "logits/chosen": -2.1190567016601562,
      "logits/rejected": -2.264734983444214,
      "logps/chosen": -203.9278106689453,
      "logps/rejected": -216.89608764648438,
      "loss": 1955.8473,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05499737709760666,
      "rewards/margins": 0.10766267776489258,
      "rewards/rejected": -0.16266006231307983,
      "step": 1570
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999830400717476e-06,
      "logits/chosen": -2.313354015350342,
      "logits/rejected": -2.1768441200256348,
      "logps/chosen": -292.4227600097656,
      "logps/rejected": -290.5616149902344,
      "loss": 1720.1881,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.050356674939394,
      "rewards/margins": 0.1441434621810913,
      "rewards/rejected": -0.1945001184940338,
      "step": 1580
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999757371834787e-06,
      "logits/chosen": -2.1764461994171143,
      "logits/rejected": -2.073521614074707,
      "logps/chosen": -246.6184539794922,
      "logps/rejected": -245.967041015625,
      "loss": 1861.9547,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06799546629190445,
      "rewards/margins": 0.14864091575145721,
      "rewards/rejected": -0.21663637459278107,
      "step": 1590
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999671302993125e-06,
      "logits/chosen": -2.1240925788879395,
      "logits/rejected": -2.0615220069885254,
      "logps/chosen": -258.4682922363281,
      "logps/rejected": -285.40802001953125,
      "loss": 2100.5828,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09887874871492386,
      "rewards/margins": 0.11551608145236969,
      "rewards/rejected": -0.21439483761787415,
      "step": 1600
    },
    {
      "epoch": 0.1,
      "eval_logits/chosen": -2.305830240249634,
      "eval_logits/rejected": -2.1245832443237305,
      "eval_logps/chosen": -235.1657257080078,
      "eval_logps/rejected": -226.7130126953125,
      "eval_loss": 1981.93994140625,
      "eval_rewards/accuracies": 0.6244999766349792,
      "eval_rewards/chosen": -0.031607791781425476,
      "eval_rewards/margins": 0.11940329521894455,
      "eval_rewards/rejected": -0.15101107954978943,
      "eval_runtime": 723.4625,
      "eval_samples_per_second": 2.764,
      "eval_steps_per_second": 1.382,
      "step": 1600
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999572194641471e-06,
      "logits/chosen": -2.257770538330078,
      "logits/rejected": -2.1237246990203857,
      "logps/chosen": -278.04107666015625,
      "logps/rejected": -242.87435913085938,
      "loss": 1498.3113,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.025227898731827736,
      "rewards/margins": 0.17303243279457092,
      "rewards/rejected": -0.1982603222131729,
      "step": 1610
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999460047296819e-06,
      "logits/chosen": -2.2606070041656494,
      "logits/rejected": -2.1405534744262695,
      "logps/chosen": -221.9639129638672,
      "logps/rejected": -213.7935333251953,
      "loss": 1835.6604,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0635976493358612,
      "rewards/margins": 0.12738750874996185,
      "rewards/rejected": -0.19098515808582306,
      "step": 1620
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999334861544186e-06,
      "logits/chosen": -2.3754539489746094,
      "logits/rejected": -2.0342419147491455,
      "logps/chosen": -233.2405242919922,
      "logps/rejected": -194.00889587402344,
      "loss": 1724.1744,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03787320479750633,
      "rewards/margins": 0.16771630942821503,
      "rewards/rejected": -0.20558953285217285,
      "step": 1630
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999196638036604e-06,
      "logits/chosen": -2.4073004722595215,
      "logits/rejected": -2.219470500946045,
      "logps/chosen": -295.1899719238281,
      "logps/rejected": -269.0531921386719,
      "loss": 2039.366,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07774912565946579,
      "rewards/margins": 0.09099628031253815,
      "rewards/rejected": -0.16874538362026215,
      "step": 1640
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999045377495111e-06,
      "logits/chosen": -2.0742428302764893,
      "logits/rejected": -2.323902130126953,
      "logps/chosen": -191.41380310058594,
      "logps/rejected": -293.4927978515625,
      "loss": 1777.3875,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.17079134285449982,
      "rewards/margins": 0.14296479523181915,
      "rewards/rejected": -0.3137561082839966,
      "step": 1650
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998881080708759e-06,
      "logits/chosen": -2.1812198162078857,
      "logits/rejected": -2.102332830429077,
      "logps/chosen": -262.5244140625,
      "logps/rejected": -238.0368194580078,
      "loss": 2214.8494,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.2220289260149002,
      "rewards/margins": 0.06457245349884033,
      "rewards/rejected": -0.28660139441490173,
      "step": 1660
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998703748534599e-06,
      "logits/chosen": -2.0106849670410156,
      "logits/rejected": -1.764390230178833,
      "logps/chosen": -263.5819396972656,
      "logps/rejected": -224.62124633789062,
      "loss": 1848.1158,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2295217514038086,
      "rewards/margins": 0.11729858070611954,
      "rewards/rejected": -0.34682032465934753,
      "step": 1670
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998513381897683e-06,
      "logits/chosen": -2.2451679706573486,
      "logits/rejected": -2.031325340270996,
      "logps/chosen": -258.3676452636719,
      "logps/rejected": -207.52420043945312,
      "loss": 2428.216,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.19123883545398712,
      "rewards/margins": 0.06150858476758003,
      "rewards/rejected": -0.2527473568916321,
      "step": 1680
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.9983099817910565e-06,
      "logits/chosen": -2.2165026664733887,
      "logits/rejected": -2.032590389251709,
      "logps/chosen": -264.272216796875,
      "logps/rejected": -270.38641357421875,
      "loss": 2055.751,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.19010308384895325,
      "rewards/margins": 0.10592222213745117,
      "rewards/rejected": -0.2960253357887268,
      "step": 1690
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998093549275754e-06,
      "logits/chosen": -2.193469285964966,
      "logits/rejected": -2.1810290813446045,
      "logps/chosen": -273.475830078125,
      "logps/rejected": -305.5787048339844,
      "loss": 1859.6395,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13141794502735138,
      "rewards/margins": 0.10309936851263046,
      "rewards/rejected": -0.23451726138591766,
      "step": 1700
    },
    {
      "epoch": 0.11,
      "eval_logits/chosen": -2.2891504764556885,
      "eval_logits/rejected": -2.1078803539276123,
      "eval_logps/chosen": -243.98208618164062,
      "eval_logps/rejected": -234.24464416503906,
      "eval_loss": 1915.6973876953125,
      "eval_rewards/accuracies": 0.6345000267028809,
      "eval_rewards/chosen": -0.11977159231901169,
      "eval_rewards/margins": 0.10655605792999268,
      "eval_rewards/rejected": -0.22632767260074615,
      "eval_runtime": 723.2631,
      "eval_samples_per_second": 2.765,
      "eval_steps_per_second": 1.383,
      "step": 1700
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997864085480794e-06,
      "logits/chosen": -2.3092598915100098,
      "logits/rejected": -2.1490447521209717,
      "logps/chosen": -278.25634765625,
      "logps/rejected": -275.22210693359375,
      "loss": 2087.6984,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11910159885883331,
      "rewards/margins": 0.15502703189849854,
      "rewards/rejected": -0.27412861585617065,
      "step": 1710
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997621591603171e-06,
      "logits/chosen": -2.287717342376709,
      "logits/rejected": -2.093785047531128,
      "logps/chosen": -169.4284210205078,
      "logps/rejected": -178.6478729248047,
      "loss": 1925.2242,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11395607143640518,
      "rewards/margins": 0.09904253482818604,
      "rewards/rejected": -0.21299858391284943,
      "step": 1720
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997366068907853e-06,
      "logits/chosen": -2.2688260078430176,
      "logits/rejected": -2.216841459274292,
      "logps/chosen": -261.4203186035156,
      "logps/rejected": -248.23538208007812,
      "loss": 2182.6373,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03010564111173153,
      "rewards/margins": 0.06690201163291931,
      "rewards/rejected": -0.09700765460729599,
      "step": 1730
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997097518727771e-06,
      "logits/chosen": -2.319974422454834,
      "logits/rejected": -2.1052706241607666,
      "logps/chosen": -226.76400756835938,
      "logps/rejected": -204.2520751953125,
      "loss": 1985.3566,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03807578235864639,
      "rewards/margins": 0.10610507428646088,
      "rewards/rejected": -0.14418084919452667,
      "step": 1740
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.9968159424638155e-06,
      "logits/chosen": -2.1506412029266357,
      "logits/rejected": -2.3671443462371826,
      "logps/chosen": -224.4173126220703,
      "logps/rejected": -280.5165710449219,
      "loss": 2440.5133,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06810899078845978,
      "rewards/margins": 0.04254139959812164,
      "rewards/rejected": -0.11065038293600082,
      "step": 1750
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9965213415848235e-06,
      "logits/chosen": -2.2544147968292236,
      "logits/rejected": -1.8740804195404053,
      "logps/chosen": -235.32717895507812,
      "logps/rejected": -211.6268310546875,
      "loss": 1604.1649,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06922627985477448,
      "rewards/margins": 0.13159620761871338,
      "rewards/rejected": -0.20082247257232666,
      "step": 1760
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9962137176275805e-06,
      "logits/chosen": -2.3310704231262207,
      "logits/rejected": -2.1352953910827637,
      "logps/chosen": -243.7481231689453,
      "logps/rejected": -253.09689331054688,
      "loss": 1873.8672,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09452693909406662,
      "rewards/margins": 0.09906788170337677,
      "rewards/rejected": -0.1935948133468628,
      "step": 1770
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9958930721968015e-06,
      "logits/chosen": -2.2000842094421387,
      "logits/rejected": -2.2737386226654053,
      "logps/chosen": -235.41317749023438,
      "logps/rejected": -254.0403594970703,
      "loss": 2234.1324,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23378005623817444,
      "rewards/margins": 0.07125429809093475,
      "rewards/rejected": -0.305034339427948,
      "step": 1780
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995559406965132e-06,
      "logits/chosen": -2.4195902347564697,
      "logits/rejected": -2.056511402130127,
      "logps/chosen": -262.2850646972656,
      "logps/rejected": -238.8928680419922,
      "loss": 2395.4992,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3036973476409912,
      "rewards/margins": 0.06223466247320175,
      "rewards/rejected": -0.36593201756477356,
      "step": 1790
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995212723673131e-06,
      "logits/chosen": -2.3763651847839355,
      "logits/rejected": -2.149697780609131,
      "logps/chosen": -255.087158203125,
      "logps/rejected": -214.32907104492188,
      "loss": 1918.483,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2488408088684082,
      "rewards/margins": 0.09224793314933777,
      "rewards/rejected": -0.34108874201774597,
      "step": 1800
    },
    {
      "epoch": 0.12,
      "eval_logits/chosen": -2.3126766681671143,
      "eval_logits/rejected": -2.1281871795654297,
      "eval_logps/chosen": -253.7191925048828,
      "eval_logps/rejected": -242.60064697265625,
      "eval_loss": 1937.4407958984375,
      "eval_rewards/accuracies": 0.6330000162124634,
      "eval_rewards/chosen": -0.21714243292808533,
      "eval_rewards/margins": 0.09274499118328094,
      "eval_rewards/rejected": -0.30988743901252747,
      "eval_runtime": 723.7868,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.382,
      "step": 1800
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.99485302412927e-06,
      "logits/chosen": -2.073103189468384,
      "logits/rejected": -1.9980010986328125,
      "logps/chosen": -227.577880859375,
      "logps/rejected": -238.51278686523438,
      "loss": 1861.5789,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.19566836953163147,
      "rewards/margins": 0.1024223119020462,
      "rewards/rejected": -0.2980906367301941,
      "step": 1810
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.994480310209918e-06,
      "logits/chosen": -2.2701027393341064,
      "logits/rejected": -2.438636541366577,
      "logps/chosen": -256.5397033691406,
      "logps/rejected": -280.5426330566406,
      "loss": 1897.3301,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.14556670188903809,
      "rewards/margins": 0.10248645395040512,
      "rewards/rejected": -0.2480531483888626,
      "step": 1820
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.994094583859332e-06,
      "logits/chosen": -2.3042404651641846,
      "logits/rejected": -2.047922372817993,
      "logps/chosen": -178.28907775878906,
      "logps/rejected": -220.57376098632812,
      "loss": 1976.666,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13376356661319733,
      "rewards/margins": 0.1025514006614685,
      "rewards/rejected": -0.23631496727466583,
      "step": 1830
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9936958470896525e-06,
      "logits/chosen": -2.309569835662842,
      "logits/rejected": -2.0754079818725586,
      "logps/chosen": -232.97189331054688,
      "logps/rejected": -205.6602325439453,
      "loss": 1508.1504,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09226727485656738,
      "rewards/margins": 0.15591464936733246,
      "rewards/rejected": -0.24818190932273865,
      "step": 1840
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.993284101980883e-06,
      "logits/chosen": -2.250126361846924,
      "logits/rejected": -2.0782506465911865,
      "logps/chosen": -252.9325714111328,
      "logps/rejected": -232.00851440429688,
      "loss": 1648.8354,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10437178611755371,
      "rewards/margins": 0.18708911538124084,
      "rewards/rejected": -0.29146090149879456,
      "step": 1850
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9928593506808885e-06,
      "logits/chosen": -2.382662296295166,
      "logits/rejected": -2.1939454078674316,
      "logps/chosen": -268.0448913574219,
      "logps/rejected": -243.6344757080078,
      "loss": 2028.7369,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1123049259185791,
      "rewards/margins": 0.1256685107946396,
      "rewards/rejected": -0.2379734218120575,
      "step": 1860
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.992421595405381e-06,
      "logits/chosen": -2.3101463317871094,
      "logits/rejected": -2.073124408721924,
      "logps/chosen": -242.1526336669922,
      "logps/rejected": -184.484375,
      "loss": 2065.8262,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.139602392911911,
      "rewards/margins": 0.13178659975528717,
      "rewards/rejected": -0.2713889479637146,
      "step": 1870
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.991970838437905e-06,
      "logits/chosen": -2.208421468734741,
      "logits/rejected": -2.1142683029174805,
      "logps/chosen": -252.5650177001953,
      "logps/rejected": -292.11737060546875,
      "loss": 1759.9475,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.27235469222068787,
      "rewards/margins": 0.11447665840387344,
      "rewards/rejected": -0.3868313431739807,
      "step": 1880
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9915070821298294e-06,
      "logits/chosen": -2.3133842945098877,
      "logits/rejected": -1.9845050573349,
      "logps/chosen": -193.9105682373047,
      "logps/rejected": -192.04551696777344,
      "loss": 2260.5314,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.2694074511528015,
      "rewards/margins": 0.07223407179117203,
      "rewards/rejected": -0.34164154529571533,
      "step": 1890
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.991030328900336e-06,
      "logits/chosen": -2.240283250808716,
      "logits/rejected": -1.996715784072876,
      "logps/chosen": -303.8148498535156,
      "logps/rejected": -253.42446899414062,
      "loss": 1457.3569,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1924562156200409,
      "rewards/margins": 0.19645829498767853,
      "rewards/rejected": -0.3889145255088806,
      "step": 1900
    },
    {
      "epoch": 0.12,
      "eval_logits/chosen": -2.2870430946350098,
      "eval_logits/rejected": -2.105412483215332,
      "eval_logps/chosen": -249.21340942382812,
      "eval_logps/rejected": -242.13661193847656,
      "eval_loss": 1911.1392822265625,
      "eval_rewards/accuracies": 0.6430000066757202,
      "eval_rewards/chosen": -0.17208442091941833,
      "eval_rewards/margins": 0.13316264748573303,
      "eval_rewards/rejected": -0.30524706840515137,
      "eval_runtime": 723.5269,
      "eval_samples_per_second": 2.764,
      "eval_steps_per_second": 1.382,
      "step": 1900
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9905405812364014e-06,
      "logits/chosen": -2.252140522003174,
      "logits/rejected": -2.235658645629883,
      "logps/chosen": -216.7350616455078,
      "logps/rejected": -233.9397735595703,
      "loss": 1774.9133,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.14438828825950623,
      "rewards/margins": 0.14739413559436798,
      "rewards/rejected": -0.2917824387550354,
      "step": 1910
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990037841692791e-06,
      "logits/chosen": -2.2116873264312744,
      "logits/rejected": -1.971073865890503,
      "logps/chosen": -218.53347778320312,
      "logps/rejected": -189.02757263183594,
      "loss": 1868.9672,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12492616474628448,
      "rewards/margins": 0.13778650760650635,
      "rewards/rejected": -0.26271265745162964,
      "step": 1920
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.989522112892039e-06,
      "logits/chosen": -2.2761216163635254,
      "logits/rejected": -2.237509250640869,
      "logps/chosen": -214.1681365966797,
      "logps/rejected": -224.54296875,
      "loss": 1870.507,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.16538745164871216,
      "rewards/margins": 0.11264890432357788,
      "rewards/rejected": -0.27803635597229004,
      "step": 1930
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.98899339752444e-06,
      "logits/chosen": -2.3265023231506348,
      "logits/rejected": -2.0911898612976074,
      "logps/chosen": -232.5698699951172,
      "logps/rejected": -222.6390838623047,
      "loss": 1702.9227,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.029675597324967384,
      "rewards/margins": 0.19012561440467834,
      "rewards/rejected": -0.21980123221874237,
      "step": 1940
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.988451698348033e-06,
      "logits/chosen": -2.2512707710266113,
      "logits/rejected": -2.224365711212158,
      "logps/chosen": -186.9007110595703,
      "logps/rejected": -217.6002960205078,
      "loss": 2052.41,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08573143184185028,
      "rewards/margins": 0.10430196672677994,
      "rewards/rejected": -0.19003340601921082,
      "step": 1950
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987897018188585e-06,
      "logits/chosen": -2.209015130996704,
      "logits/rejected": -1.9794836044311523,
      "logps/chosen": -234.8341064453125,
      "logps/rejected": -187.73577880859375,
      "loss": 2258.7855,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12214773893356323,
      "rewards/margins": 0.09577733278274536,
      "rewards/rejected": -0.2179250717163086,
      "step": 1960
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9873293599395814e-06,
      "logits/chosen": -2.23533296585083,
      "logits/rejected": -2.1085968017578125,
      "logps/chosen": -210.78250122070312,
      "logps/rejected": -220.7260284423828,
      "loss": 1796.2572,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.17350736260414124,
      "rewards/margins": 0.13926991820335388,
      "rewards/rejected": -0.3127772808074951,
      "step": 1970
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.986748726562203e-06,
      "logits/chosen": -2.2655141353607178,
      "logits/rejected": -2.1425416469573975,
      "logps/chosen": -228.8271942138672,
      "logps/rejected": -219.7353973388672,
      "loss": 1605.6649,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.12972739338874817,
      "rewards/margins": 0.1580057144165039,
      "rewards/rejected": -0.2877331078052521,
      "step": 1980
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.98615512108532e-06,
      "logits/chosen": -2.3564047813415527,
      "logits/rejected": -2.2113535404205322,
      "logps/chosen": -228.1331329345703,
      "logps/rejected": -240.79299926757812,
      "loss": 1882.9188,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06622384488582611,
      "rewards/margins": 0.10876498371362686,
      "rewards/rejected": -0.17498883605003357,
      "step": 1990
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.985548546605469e-06,
      "logits/chosen": -2.1078901290893555,
      "logits/rejected": -2.234835147857666,
      "logps/chosen": -227.9741668701172,
      "logps/rejected": -251.87881469726562,
      "loss": 2512.6883,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.14868374168872833,
      "rewards/margins": 0.06264106929302216,
      "rewards/rejected": -0.2113248109817505,
      "step": 2000
    },
    {
      "epoch": 0.13,
      "eval_logits/chosen": -2.284355878829956,
      "eval_logits/rejected": -2.10317325592041,
      "eval_logps/chosen": -243.0243682861328,
      "eval_logps/rejected": -236.22425842285156,
      "eval_loss": 1904.934814453125,
      "eval_rewards/accuracies": 0.6395000219345093,
      "eval_rewards/chosen": -0.11019425839185715,
      "eval_rewards/margins": 0.13592927157878876,
      "eval_rewards/rejected": -0.2461235523223877,
      "eval_runtime": 724.8151,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.38,
      "step": 2000
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.984929006286838e-06,
      "logits/chosen": -2.135776996612549,
      "logits/rejected": -2.0879735946655273,
      "logps/chosen": -220.8446807861328,
      "logps/rejected": -230.2923583984375,
      "loss": 2360.9186,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11530672013759613,
      "rewards/margins": 0.05991410091519356,
      "rewards/rejected": -0.1752208173274994,
      "step": 2010
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.984296503361256e-06,
      "logits/chosen": -2.372276782989502,
      "logits/rejected": -2.01884126663208,
      "logps/chosen": -213.36813354492188,
      "logps/rejected": -185.35867309570312,
      "loss": 1618.9799,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11448262631893158,
      "rewards/margins": 0.13994377851486206,
      "rewards/rejected": -0.25442641973495483,
      "step": 2020
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9836510411281645e-06,
      "logits/chosen": -2.2159922122955322,
      "logits/rejected": -2.1070516109466553,
      "logps/chosen": -291.0960388183594,
      "logps/rejected": -272.9174499511719,
      "loss": 1484.0746,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.12564203143119812,
      "rewards/margins": 0.21223802864551544,
      "rewards/rejected": -0.33788007497787476,
      "step": 2030
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.982992622954613e-06,
      "logits/chosen": -2.3539059162139893,
      "logits/rejected": -2.06502628326416,
      "logps/chosen": -299.5829162597656,
      "logps/rejected": -200.73965454101562,
      "loss": 1749.1727,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.16054727137088776,
      "rewards/margins": 0.15325310826301575,
      "rewards/rejected": -0.3138003647327423,
      "step": 2040
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9823212522752325e-06,
      "logits/chosen": -2.445986032485962,
      "logits/rejected": -2.1705663204193115,
      "logps/chosen": -296.69390869140625,
      "logps/rejected": -279.83660888671875,
      "loss": 1693.1498,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1888018697500229,
      "rewards/margins": 0.17178688943386078,
      "rewards/rejected": -0.3605887293815613,
      "step": 2050
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.981636932592222e-06,
      "logits/chosen": -2.1977176666259766,
      "logits/rejected": -2.0966477394104004,
      "logps/chosen": -224.46115112304688,
      "logps/rejected": -235.4087677001953,
      "loss": 1721.5926,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11548157036304474,
      "rewards/margins": 0.1496855914592743,
      "rewards/rejected": -0.2651671767234802,
      "step": 2060
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.980939667475328e-06,
      "logits/chosen": -2.409071445465088,
      "logits/rejected": -2.061501979827881,
      "logps/chosen": -287.5926513671875,
      "logps/rejected": -238.5335693359375,
      "loss": 2094.627,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.134174644947052,
      "rewards/margins": 0.11532992124557495,
      "rewards/rejected": -0.24950456619262695,
      "step": 2070
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.980229460561826e-06,
      "logits/chosen": -2.2790915966033936,
      "logits/rejected": -2.2238266468048096,
      "logps/chosen": -231.64938354492188,
      "logps/rejected": -233.92288208007812,
      "loss": 1537.9895,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1356803923845291,
      "rewards/margins": 0.1757991760969162,
      "rewards/rejected": -0.3114795684814453,
      "step": 2080
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.979506315556503e-06,
      "logits/chosen": -2.238194465637207,
      "logits/rejected": -1.915668249130249,
      "logps/chosen": -303.0437316894531,
      "logps/rejected": -267.5552673339844,
      "loss": 1857.4201,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16551579535007477,
      "rewards/margins": 0.15499567985534668,
      "rewards/rejected": -0.32051146030426025,
      "step": 2090
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9787702362316395e-06,
      "logits/chosen": -2.307528018951416,
      "logits/rejected": -2.5269463062286377,
      "logps/chosen": -206.5220947265625,
      "logps/rejected": -241.37844848632812,
      "loss": 2032.659,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.15694910287857056,
      "rewards/margins": 0.09342299401760101,
      "rewards/rejected": -0.25037211179733276,
      "step": 2100
    },
    {
      "epoch": 0.14,
      "eval_logits/chosen": -2.30545711517334,
      "eval_logits/rejected": -2.1217041015625,
      "eval_logps/chosen": -243.19927978515625,
      "eval_logps/rejected": -235.20457458496094,
      "eval_loss": 1915.6287841796875,
      "eval_rewards/accuracies": 0.6359999775886536,
      "eval_rewards/chosen": -0.11194323003292084,
      "eval_rewards/margins": 0.12398341298103333,
      "eval_rewards/rejected": -0.23592665791511536,
      "eval_runtime": 725.809,
      "eval_samples_per_second": 2.756,
      "eval_steps_per_second": 1.378,
      "step": 2100
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9780212264269835e-06,
      "logits/chosen": -2.2523937225341797,
      "logits/rejected": -2.0116162300109863,
      "logps/chosen": -199.10507202148438,
      "logps/rejected": -190.06492614746094,
      "loss": 2266.8332,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.14285996556282043,
      "rewards/margins": 0.08095569163560867,
      "rewards/rejected": -0.2238156795501709,
      "step": 2110
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.977259290049739e-06,
      "logits/chosen": -2.4865453243255615,
      "logits/rejected": -1.9351905584335327,
      "logps/chosen": -288.5707092285156,
      "logps/rejected": -239.4718017578125,
      "loss": 1351.2916,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.01628245785832405,
      "rewards/margins": 0.20419542491436005,
      "rewards/rejected": -0.2204778641462326,
      "step": 2120
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.976484431074538e-06,
      "logits/chosen": -2.2364916801452637,
      "logits/rejected": -2.1809253692626953,
      "logps/chosen": -194.6975860595703,
      "logps/rejected": -187.02737426757812,
      "loss": 1983.0709,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008920894004404545,
      "rewards/margins": 0.09932135045528412,
      "rewards/rejected": -0.10824225097894669,
      "step": 2130
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.975696653543425e-06,
      "logits/chosen": -2.3318240642547607,
      "logits/rejected": -2.0875277519226074,
      "logps/chosen": -249.6975555419922,
      "logps/rejected": -251.0378875732422,
      "loss": 1705.1627,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0004876054881606251,
      "rewards/margins": 0.13909529149532318,
      "rewards/rejected": -0.13958290219306946,
      "step": 2140
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.974895961565835e-06,
      "logits/chosen": -2.257631301879883,
      "logits/rejected": -1.919825792312622,
      "logps/chosen": -178.15733337402344,
      "logps/rejected": -195.17355346679688,
      "loss": 2029.2096,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.029063880443572998,
      "rewards/margins": 0.11788249015808105,
      "rewards/rejected": -0.14694637060165405,
      "step": 2150
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.974082359318566e-06,
      "logits/chosen": -2.235304355621338,
      "logits/rejected": -2.070316791534424,
      "logps/chosen": -253.31259155273438,
      "logps/rejected": -223.68795776367188,
      "loss": 1586.7559,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.003374456660822034,
      "rewards/margins": 0.1585448533296585,
      "rewards/rejected": -0.15517041087150574,
      "step": 2160
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.973255851045769e-06,
      "logits/chosen": -2.2016162872314453,
      "logits/rejected": -2.2180228233337402,
      "logps/chosen": -219.61465454101562,
      "logps/rejected": -198.3446044921875,
      "loss": 1641.7762,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.022564545273780823,
      "rewards/margins": 0.1547742635011673,
      "rewards/rejected": -0.17733880877494812,
      "step": 2170
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.972416441058915e-06,
      "logits/chosen": -2.153087854385376,
      "logits/rejected": -2.004279613494873,
      "logps/chosen": -238.0662078857422,
      "logps/rejected": -230.45004272460938,
      "loss": 1542.9841,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.08310385793447495,
      "rewards/margins": 0.19025757908821106,
      "rewards/rejected": -0.273361474275589,
      "step": 2180
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.971564133736777e-06,
      "logits/chosen": -2.1095261573791504,
      "logits/rejected": -1.938579797744751,
      "logps/chosen": -184.16986083984375,
      "logps/rejected": -206.4436798095703,
      "loss": 1519.6011,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06169208139181137,
      "rewards/margins": 0.17292717099189758,
      "rewards/rejected": -0.23461923003196716,
      "step": 2190
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.970698933525409e-06,
      "logits/chosen": -2.3949289321899414,
      "logits/rejected": -2.128685712814331,
      "logps/chosen": -284.95123291015625,
      "logps/rejected": -259.23516845703125,
      "loss": 2207.576,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.056042980402708054,
      "rewards/margins": 0.1395549774169922,
      "rewards/rejected": -0.19559794664382935,
      "step": 2200
    },
    {
      "epoch": 0.14,
      "eval_logits/chosen": -2.284047842025757,
      "eval_logits/rejected": -2.1025538444519043,
      "eval_logps/chosen": -233.87107849121094,
      "eval_logps/rejected": -226.02322387695312,
      "eval_loss": 1899.6104736328125,
      "eval_rewards/accuracies": 0.6399999856948853,
      "eval_rewards/chosen": -0.018661215901374817,
      "eval_rewards/margins": 0.12545213103294373,
      "eval_rewards/rejected": -0.14411336183547974,
      "eval_runtime": 723.3244,
      "eval_samples_per_second": 2.765,
      "eval_steps_per_second": 1.383,
      "step": 2200
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.969820844938118e-06,
      "logits/chosen": -2.3908708095550537,
      "logits/rejected": -2.084207057952881,
      "logps/chosen": -223.781982421875,
      "logps/rejected": -184.0693359375,
      "loss": 1800.1578,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.011787386611104012,
      "rewards/margins": 0.1419001817703247,
      "rewards/rejected": -0.15368756651878357,
      "step": 2210
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.968929872555444e-06,
      "logits/chosen": -1.9293677806854248,
      "logits/rejected": -2.1174938678741455,
      "logps/chosen": -226.0688934326172,
      "logps/rejected": -267.40545654296875,
      "loss": 2410.917,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08934484422206879,
      "rewards/margins": 0.056735485792160034,
      "rewards/rejected": -0.14608034491539001,
      "step": 2220
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.968026021025137e-06,
      "logits/chosen": -2.3432908058166504,
      "logits/rejected": -2.1328892707824707,
      "logps/chosen": -205.4949951171875,
      "logps/rejected": -180.20901489257812,
      "loss": 1757.748,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03930085152387619,
      "rewards/margins": 0.1477939933538437,
      "rewards/rejected": -0.18709483742713928,
      "step": 2230
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.967109295062128e-06,
      "logits/chosen": -2.198868751525879,
      "logits/rejected": -1.9802119731903076,
      "logps/chosen": -233.25698852539062,
      "logps/rejected": -263.6954650878906,
      "loss": 2017.6684,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08255533128976822,
      "rewards/margins": 0.09517533332109451,
      "rewards/rejected": -0.17773064970970154,
      "step": 2240
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.966179699448509e-06,
      "logits/chosen": -2.1731762886047363,
      "logits/rejected": -1.978885293006897,
      "logps/chosen": -195.97317504882812,
      "logps/rejected": -187.7627716064453,
      "loss": 2002.8109,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05218205973505974,
      "rewards/margins": 0.09615719318389893,
      "rewards/rejected": -0.14833924174308777,
      "step": 2250
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.965237239033506e-06,
      "logits/chosen": -2.32716965675354,
      "logits/rejected": -2.152055263519287,
      "logps/chosen": -299.1788330078125,
      "logps/rejected": -276.8256530761719,
      "loss": 1632.5648,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.09520121663808823,
      "rewards/margins": 0.22428712248802185,
      "rewards/rejected": -0.3194883465766907,
      "step": 2260
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.964281918733453e-06,
      "logits/chosen": -2.2710909843444824,
      "logits/rejected": -2.0281014442443848,
      "logps/chosen": -193.1463623046875,
      "logps/rejected": -205.0344696044922,
      "loss": 1633.4338,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10031914710998535,
      "rewards/margins": 0.1634485125541687,
      "rewards/rejected": -0.26376765966415405,
      "step": 2270
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9633137435317715e-06,
      "logits/chosen": -2.289713144302368,
      "logits/rejected": -1.6606485843658447,
      "logps/chosen": -229.876953125,
      "logps/rejected": -179.68936157226562,
      "loss": 1529.5553,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06981930136680603,
      "rewards/margins": 0.17597553133964539,
      "rewards/rejected": -0.24579481780529022,
      "step": 2280
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9623327184789355e-06,
      "logits/chosen": -2.3933303356170654,
      "logits/rejected": -2.309675931930542,
      "logps/chosen": -221.02023315429688,
      "logps/rejected": -223.60763549804688,
      "loss": 2147.6383,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03279787302017212,
      "rewards/margins": 0.07413671165704727,
      "rewards/rejected": -0.10693458467721939,
      "step": 2290
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9613388486924525e-06,
      "logits/chosen": -1.9889920949935913,
      "logits/rejected": -2.12893009185791,
      "logps/chosen": -181.59829711914062,
      "logps/rejected": -211.2786865234375,
      "loss": 1761.0438,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.014642806723713875,
      "rewards/margins": 0.12300250679254532,
      "rewards/rejected": -0.13764533400535583,
      "step": 2300
    },
    {
      "epoch": 0.15,
      "eval_logits/chosen": -2.2852556705474854,
      "eval_logits/rejected": -2.1030173301696777,
      "eval_logps/chosen": -232.01632690429688,
      "eval_logps/rejected": -223.4245147705078,
      "eval_loss": 1910.914794921875,
      "eval_rewards/accuracies": 0.6365000009536743,
      "eval_rewards/chosen": -0.00011374091991456226,
      "eval_rewards/margins": 0.1180122047662735,
      "eval_rewards/rejected": -0.11812596023082733,
      "eval_runtime": 727.0474,
      "eval_samples_per_second": 2.751,
      "eval_steps_per_second": 1.375,
      "step": 2300
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.960332139356834e-06,
      "logits/chosen": -2.2433876991271973,
      "logits/rejected": -2.0597805976867676,
      "logps/chosen": -208.4698486328125,
      "logps/rejected": -196.3324432373047,
      "loss": 1667.8039,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0034699998795986176,
      "rewards/margins": 0.14302471280097961,
      "rewards/rejected": -0.1395547240972519,
      "step": 2310
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.95931259572357e-06,
      "logits/chosen": -2.329911947250366,
      "logits/rejected": -1.9796215295791626,
      "logps/chosen": -236.82339477539062,
      "logps/rejected": -278.1308288574219,
      "loss": 1809.4439,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.025833886116743088,
      "rewards/margins": 0.15633848309516907,
      "rewards/rejected": -0.18217237293720245,
      "step": 2320
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9582802231111e-06,
      "logits/chosen": -2.1584572792053223,
      "logits/rejected": -2.244807720184326,
      "logps/chosen": -221.17514038085938,
      "logps/rejected": -207.26791381835938,
      "loss": 1835.1227,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09301011264324188,
      "rewards/margins": 0.12971298396587372,
      "rewards/rejected": -0.2227230966091156,
      "step": 2330
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.957235026904782e-06,
      "logits/chosen": -2.3219685554504395,
      "logits/rejected": -2.0044803619384766,
      "logps/chosen": -267.11383056640625,
      "logps/rejected": -230.76583862304688,
      "loss": 1752.1791,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.049614351242780685,
      "rewards/margins": 0.14767198264598846,
      "rewards/rejected": -0.19728633761405945,
      "step": 2340
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.956177012556875e-06,
      "logits/chosen": -2.400453805923462,
      "logits/rejected": -2.1781132221221924,
      "logps/chosen": -258.6994934082031,
      "logps/rejected": -200.09947204589844,
      "loss": 2029.8385,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.14506658911705017,
      "rewards/margins": 0.12968036532402039,
      "rewards/rejected": -0.27474695444107056,
      "step": 2350
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9551061855864976e-06,
      "logits/chosen": -2.069888114929199,
      "logits/rejected": -2.1222586631774902,
      "logps/chosen": -208.03359985351562,
      "logps/rejected": -222.52658081054688,
      "loss": 1863.459,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15205471217632294,
      "rewards/margins": 0.10282406955957413,
      "rewards/rejected": -0.25487881898880005,
      "step": 2360
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.95402255157961e-06,
      "logits/chosen": -2.105889320373535,
      "logits/rejected": -2.1698150634765625,
      "logps/chosen": -196.15615844726562,
      "logps/rejected": -273.117431640625,
      "loss": 2058.5367,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11099891364574432,
      "rewards/margins": 0.12415103614330292,
      "rewards/rejected": -0.23514994978904724,
      "step": 2370
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.952926116188977e-06,
      "logits/chosen": -2.383444309234619,
      "logits/rejected": -2.316694974899292,
      "logps/chosen": -195.67141723632812,
      "logps/rejected": -240.1870880126953,
      "loss": 2270.6312,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.12294076383113861,
      "rewards/margins": 0.053111083805561066,
      "rewards/rejected": -0.17605182528495789,
      "step": 2380
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.951816885134143e-06,
      "logits/chosen": -2.2206435203552246,
      "logits/rejected": -2.2017269134521484,
      "logps/chosen": -210.9372100830078,
      "logps/rejected": -219.2127227783203,
      "loss": 2097.6289,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09384217113256454,
      "rewards/margins": 0.09136297553777695,
      "rewards/rejected": -0.1852051317691803,
      "step": 2390
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.950694864201399e-06,
      "logits/chosen": -2.2418875694274902,
      "logits/rejected": -2.162963390350342,
      "logps/chosen": -247.30673217773438,
      "logps/rejected": -265.2274475097656,
      "loss": 1459.2476,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0862477570772171,
      "rewards/margins": 0.1579146683216095,
      "rewards/rejected": -0.2441624402999878,
      "step": 2400
    },
    {
      "epoch": 0.16,
      "eval_logits/chosen": -2.260871171951294,
      "eval_logits/rejected": -2.0794146060943604,
      "eval_logps/chosen": -238.68450927734375,
      "eval_logps/rejected": -231.64358520507812,
      "eval_loss": 1888.1434326171875,
      "eval_rewards/accuracies": 0.6349999904632568,
      "eval_rewards/chosen": -0.06679563969373703,
      "eval_rewards/margins": 0.13352102041244507,
      "eval_rewards/rejected": -0.2003166675567627,
      "eval_runtime": 725.1376,
      "eval_samples_per_second": 2.758,
      "eval_steps_per_second": 1.379,
      "step": 2400
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9495600592437575e-06,
      "logits/chosen": -2.3008065223693848,
      "logits/rejected": -2.1299381256103516,
      "logps/chosen": -239.4790496826172,
      "logps/rejected": -255.85281372070312,
      "loss": 1879.6795,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1258460134267807,
      "rewards/margins": 0.1319214254617691,
      "rewards/rejected": -0.2577674388885498,
      "step": 2410
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.948412476180917e-06,
      "logits/chosen": -2.2109181880950928,
      "logits/rejected": -1.9310805797576904,
      "logps/chosen": -202.6303253173828,
      "logps/rejected": -194.5168914794922,
      "loss": 1935.1873,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18554428219795227,
      "rewards/margins": 0.14623305201530457,
      "rewards/rejected": -0.3317773640155792,
      "step": 2420
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.947252120999232e-06,
      "logits/chosen": -2.23991060256958,
      "logits/rejected": -1.9719324111938477,
      "logps/chosen": -291.8089904785156,
      "logps/rejected": -239.60586547851562,
      "loss": 2337.2834,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.24704165756702423,
      "rewards/margins": 0.06570660322904587,
      "rewards/rejected": -0.3127482533454895,
      "step": 2430
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.946078999751683e-06,
      "logits/chosen": -2.156996011734009,
      "logits/rejected": -2.0907959938049316,
      "logps/chosen": -193.2981719970703,
      "logps/rejected": -187.05856323242188,
      "loss": 1601.603,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13671033084392548,
      "rewards/margins": 0.15316611528396606,
      "rewards/rejected": -0.28987646102905273,
      "step": 2440
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.944893118557847e-06,
      "logits/chosen": -2.0801761150360107,
      "logits/rejected": -2.0442826747894287,
      "logps/chosen": -223.3832244873047,
      "logps/rejected": -181.52224731445312,
      "loss": 1935.543,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1689862757921219,
      "rewards/margins": 0.11829449236392975,
      "rewards/rejected": -0.28728073835372925,
      "step": 2450
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.943694483603861e-06,
      "logits/chosen": -2.3802292346954346,
      "logits/rejected": -1.9829076528549194,
      "logps/chosen": -230.80758666992188,
      "logps/rejected": -196.69564819335938,
      "loss": 1594.3304,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04803575947880745,
      "rewards/margins": 0.1686701476573944,
      "rewards/rejected": -0.21670588850975037,
      "step": 2460
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9424831011423914e-06,
      "logits/chosen": -2.3456430435180664,
      "logits/rejected": -2.257657527923584,
      "logps/chosen": -293.9297180175781,
      "logps/rejected": -263.3369140625,
      "loss": 2375.3443,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.05503131076693535,
      "rewards/margins": 0.06421846151351929,
      "rewards/rejected": -0.11924977600574493,
      "step": 2470
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9412589774926015e-06,
      "logits/chosen": -2.345982789993286,
      "logits/rejected": -2.0571885108947754,
      "logps/chosen": -275.6372985839844,
      "logps/rejected": -241.6995086669922,
      "loss": 1729.7469,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.014475801959633827,
      "rewards/margins": 0.16685692965984344,
      "rewards/rejected": -0.18133273720741272,
      "step": 2480
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.940022119040121e-06,
      "logits/chosen": -2.390468120574951,
      "logits/rejected": -2.0979933738708496,
      "logps/chosen": -295.2787170410156,
      "logps/rejected": -278.6435852050781,
      "loss": 1984.5342,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.033627700060606,
      "rewards/margins": 0.11183059215545654,
      "rewards/rejected": -0.14545831084251404,
      "step": 2490
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.93877253223701e-06,
      "logits/chosen": -2.327923536300659,
      "logits/rejected": -2.0985212326049805,
      "logps/chosen": -303.5312194824219,
      "logps/rejected": -283.330810546875,
      "loss": 1836.8891,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11259005963802338,
      "rewards/margins": 0.11372293531894684,
      "rewards/rejected": -0.22631299495697021,
      "step": 2500
    },
    {
      "epoch": 0.16,
      "eval_logits/chosen": -2.2696266174316406,
      "eval_logits/rejected": -2.087106466293335,
      "eval_logps/chosen": -244.68203735351562,
      "eval_logps/rejected": -237.92999267578125,
      "eval_loss": 1875.603759765625,
      "eval_rewards/accuracies": 0.637499988079071,
      "eval_rewards/chosen": -0.12677064538002014,
      "eval_rewards/margins": 0.13641037046909332,
      "eval_rewards/rejected": -0.26318103075027466,
      "eval_runtime": 722.4324,
      "eval_samples_per_second": 2.768,
      "eval_steps_per_second": 1.384,
      "step": 2500
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.937510223601725e-06,
      "logits/chosen": -2.4875149726867676,
      "logits/rejected": -2.3256750106811523,
      "logps/chosen": -269.61895751953125,
      "logps/rejected": -239.8365020751953,
      "loss": 1762.4992,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07093014568090439,
      "rewards/margins": 0.12428943067789078,
      "rewards/rejected": -0.19521956145763397,
      "step": 2510
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.936235199719085e-06,
      "logits/chosen": -2.302119016647339,
      "logits/rejected": -2.1802175045013428,
      "logps/chosen": -183.2852020263672,
      "logps/rejected": -172.84878540039062,
      "loss": 1602.4494,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10983192920684814,
      "rewards/margins": 0.19742779433727264,
      "rewards/rejected": -0.3072597086429596,
      "step": 2520
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.93494746724024e-06,
      "logits/chosen": -2.2991228103637695,
      "logits/rejected": -2.040062427520752,
      "logps/chosen": -238.6068878173828,
      "logps/rejected": -275.1326904296875,
      "loss": 1426.2948,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.14802417159080505,
      "rewards/margins": 0.19065825641155243,
      "rewards/rejected": -0.3386824429035187,
      "step": 2530
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.933647032882635e-06,
      "logits/chosen": -2.440565586090088,
      "logits/rejected": -2.1358885765075684,
      "logps/chosen": -258.40179443359375,
      "logps/rejected": -229.6172332763672,
      "loss": 1756.3088,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1353994458913803,
      "rewards/margins": 0.15634958446025848,
      "rewards/rejected": -0.2917490005493164,
      "step": 2540
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.932333903429969e-06,
      "logits/chosen": -2.1265618801116943,
      "logits/rejected": -1.9849107265472412,
      "logps/chosen": -208.16122436523438,
      "logps/rejected": -188.74383544921875,
      "loss": 2189.473,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.15078715980052948,
      "rewards/margins": 0.06388656795024872,
      "rewards/rejected": -0.2146737277507782,
      "step": 2550
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.931008085732172e-06,
      "logits/chosen": -2.3160247802734375,
      "logits/rejected": -1.9144550561904907,
      "logps/chosen": -227.34365844726562,
      "logps/rejected": -189.74508666992188,
      "loss": 1769.6924,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.245207279920578,
      "rewards/margins": 0.12402795255184174,
      "rewards/rejected": -0.36923524737358093,
      "step": 2560
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9296695867053565e-06,
      "logits/chosen": -2.2828354835510254,
      "logits/rejected": -2.066901683807373,
      "logps/chosen": -319.77569580078125,
      "logps/rejected": -278.95794677734375,
      "loss": 1323.6122,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.19174179434776306,
      "rewards/margins": 0.2514215111732483,
      "rewards/rejected": -0.44316330552101135,
      "step": 2570
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.928318413331791e-06,
      "logits/chosen": -2.3800201416015625,
      "logits/rejected": -2.16902494430542,
      "logps/chosen": -237.91647338867188,
      "logps/rejected": -230.12765502929688,
      "loss": 2151.4758,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2668939232826233,
      "rewards/margins": 0.08196422457695007,
      "rewards/rejected": -0.34885817766189575,
      "step": 2580
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.926954572659855e-06,
      "logits/chosen": -2.1593034267425537,
      "logits/rejected": -2.1724724769592285,
      "logps/chosen": -267.30853271484375,
      "logps/rejected": -300.624267578125,
      "loss": 1644.4615,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.25112849473953247,
      "rewards/margins": 0.16129925847053528,
      "rewards/rejected": -0.41242775321006775,
      "step": 2590
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.925578071804013e-06,
      "logits/chosen": -2.196808338165283,
      "logits/rejected": -2.143341064453125,
      "logps/chosen": -252.46435546875,
      "logps/rejected": -320.8917541503906,
      "loss": 1885.9926,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22758595645427704,
      "rewards/margins": 0.1101551502943039,
      "rewards/rejected": -0.33774110674858093,
      "step": 2600
    },
    {
      "epoch": 0.17,
      "eval_logits/chosen": -2.295515775680542,
      "eval_logits/rejected": -2.1104376316070557,
      "eval_logps/chosen": -250.35304260253906,
      "eval_logps/rejected": -244.417236328125,
      "eval_loss": 1840.02197265625,
      "eval_rewards/accuracies": 0.6570000052452087,
      "eval_rewards/chosen": -0.18348100781440735,
      "eval_rewards/margins": 0.14457230269908905,
      "eval_rewards/rejected": -0.3280532956123352,
      "eval_runtime": 725.3952,
      "eval_samples_per_second": 2.757,
      "eval_steps_per_second": 1.379,
      "step": 2600
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.924188917944763e-06,
      "logits/chosen": -2.357513904571533,
      "logits/rejected": -2.1812283992767334,
      "logps/chosen": -235.4732208251953,
      "logps/rejected": -219.39749145507812,
      "loss": 1659.5766,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15468116104602814,
      "rewards/margins": 0.17444713413715363,
      "rewards/rejected": -0.32912832498550415,
      "step": 2610
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.922787118328617e-06,
      "logits/chosen": -2.3777213096618652,
      "logits/rejected": -2.0662240982055664,
      "logps/chosen": -251.46151733398438,
      "logps/rejected": -181.6281280517578,
      "loss": 2529.3734,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.21443858742713928,
      "rewards/margins": 0.0647946149110794,
      "rewards/rejected": -0.2792331874370575,
      "step": 2620
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.921372680268045e-06,
      "logits/chosen": -2.359708309173584,
      "logits/rejected": -2.0272839069366455,
      "logps/chosen": -252.25289916992188,
      "logps/rejected": -227.28915405273438,
      "loss": 2616.8451,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.2224821299314499,
      "rewards/margins": 0.054050058126449585,
      "rewards/rejected": -0.2765321731567383,
      "step": 2630
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.919945611141451e-06,
      "logits/chosen": -2.4145865440368652,
      "logits/rejected": -2.044100284576416,
      "logps/chosen": -236.93545532226562,
      "logps/rejected": -190.10006713867188,
      "loss": 1913.1398,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11700806766748428,
      "rewards/margins": 0.11683978140354156,
      "rewards/rejected": -0.23384782671928406,
      "step": 2640
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.918505918393125e-06,
      "logits/chosen": -2.2917346954345703,
      "logits/rejected": -2.1400973796844482,
      "logps/chosen": -179.37405395507812,
      "logps/rejected": -213.5010223388672,
      "loss": 1871.7523,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10343358665704727,
      "rewards/margins": 0.10753600299358368,
      "rewards/rejected": -0.21096959710121155,
      "step": 2650
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.91705360953321e-06,
      "logits/chosen": -2.344794511795044,
      "logits/rejected": -2.1143381595611572,
      "logps/chosen": -260.46099853515625,
      "logps/rejected": -241.50204467773438,
      "loss": 1606.4104,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.14973367750644684,
      "rewards/margins": 0.13696415722370148,
      "rewards/rejected": -0.2866978049278259,
      "step": 2660
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9155886921376615e-06,
      "logits/chosen": -2.267577648162842,
      "logits/rejected": -2.222317934036255,
      "logps/chosen": -222.16390991210938,
      "logps/rejected": -253.34878540039062,
      "loss": 1950.7385,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.16195516288280487,
      "rewards/margins": 0.0951649397611618,
      "rewards/rejected": -0.25712013244628906,
      "step": 2670
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.914111173848205e-06,
      "logits/chosen": -2.316715717315674,
      "logits/rejected": -2.2697932720184326,
      "logps/chosen": -250.2243194580078,
      "logps/rejected": -240.92874145507812,
      "loss": 2477.1484,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.17192703485488892,
      "rewards/margins": 0.06122405454516411,
      "rewards/rejected": -0.2331511229276657,
      "step": 2680
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9126210623723e-06,
      "logits/chosen": -2.0935120582580566,
      "logits/rejected": -2.2726893424987793,
      "logps/chosen": -212.24560546875,
      "logps/rejected": -259.00762939453125,
      "loss": 1884.5598,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11872206628322601,
      "rewards/margins": 0.1299000084400177,
      "rewards/rejected": -0.24862205982208252,
      "step": 2690
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.911118365483098e-06,
      "logits/chosen": -2.190582752227783,
      "logits/rejected": -2.280961275100708,
      "logps/chosen": -216.3970947265625,
      "logps/rejected": -242.3699951171875,
      "loss": 1692.8492,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1464289426803589,
      "rewards/margins": 0.165466770529747,
      "rewards/rejected": -0.3118956685066223,
      "step": 2700
    },
    {
      "epoch": 0.18,
      "eval_logits/chosen": -2.3050317764282227,
      "eval_logits/rejected": -2.1195755004882812,
      "eval_logps/chosen": -247.22756958007812,
      "eval_logps/rejected": -238.92596435546875,
      "eval_loss": 1861.056640625,
      "eval_rewards/accuracies": 0.6549999713897705,
      "eval_rewards/chosen": -0.15222610533237457,
      "eval_rewards/margins": 0.12091454118490219,
      "eval_rewards/rejected": -0.27314069867134094,
      "eval_runtime": 724.7776,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.38,
      "step": 2700
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.909603091019403e-06,
      "logits/chosen": -2.4673802852630615,
      "logits/rejected": -2.0942001342773438,
      "logps/chosen": -246.7534942626953,
      "logps/rejected": -226.46493530273438,
      "loss": 1654.3146,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10376157611608505,
      "rewards/margins": 0.16331735253334045,
      "rewards/rejected": -0.2670789361000061,
      "step": 2710
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.908075246885626e-06,
      "logits/chosen": -2.241844892501831,
      "logits/rejected": -2.1684601306915283,
      "logps/chosen": -163.9541473388672,
      "logps/rejected": -142.56719970703125,
      "loss": 2271.8395,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.13226650655269623,
      "rewards/margins": 0.06004362553358078,
      "rewards/rejected": -0.1923101395368576,
      "step": 2720
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.906534841051755e-06,
      "logits/chosen": -2.1116034984588623,
      "logits/rejected": -2.1748390197753906,
      "logps/chosen": -256.3478088378906,
      "logps/rejected": -277.2479248046875,
      "loss": 2043.9254,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12944331765174866,
      "rewards/margins": 0.09864501655101776,
      "rewards/rejected": -0.2280883491039276,
      "step": 2730
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.904981881553297e-06,
      "logits/chosen": -2.37188982963562,
      "logits/rejected": -2.04917311668396,
      "logps/chosen": -238.0791015625,
      "logps/rejected": -181.63294982910156,
      "loss": 2043.5838,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1388280689716339,
      "rewards/margins": 0.07672025263309479,
      "rewards/rejected": -0.2155483067035675,
      "step": 2740
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.903416376491252e-06,
      "logits/chosen": -2.3755104541778564,
      "logits/rejected": -1.9843056201934814,
      "logps/chosen": -289.67669677734375,
      "logps/rejected": -264.0829772949219,
      "loss": 1753.6057,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11384458839893341,
      "rewards/margins": 0.14998391270637512,
      "rewards/rejected": -0.26382848620414734,
      "step": 2750
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.90183833403206e-06,
      "logits/chosen": -2.438305377960205,
      "logits/rejected": -2.282226800918579,
      "logps/chosen": -271.61114501953125,
      "logps/rejected": -249.8431396484375,
      "loss": 1752.6217,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07398039102554321,
      "rewards/margins": 0.14484994113445282,
      "rewards/rejected": -0.21883034706115723,
      "step": 2760
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.900247762407564e-06,
      "logits/chosen": -2.2102179527282715,
      "logits/rejected": -1.9937061071395874,
      "logps/chosen": -181.43264770507812,
      "logps/rejected": -217.74343872070312,
      "loss": 1837.5686,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.09140662103891373,
      "rewards/margins": 0.14268136024475098,
      "rewards/rejected": -0.2340880185365677,
      "step": 2770
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.898644669914965e-06,
      "logits/chosen": -2.2669076919555664,
      "logits/rejected": -2.18135929107666,
      "logps/chosen": -231.2422332763672,
      "logps/rejected": -231.5166015625,
      "loss": 1757.7465,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07169830054044724,
      "rewards/margins": 0.14100399613380432,
      "rewards/rejected": -0.21270231902599335,
      "step": 2780
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.897029064916778e-06,
      "logits/chosen": -2.0898585319519043,
      "logits/rejected": -1.890570044517517,
      "logps/chosen": -217.30859375,
      "logps/rejected": -213.62460327148438,
      "loss": 2316.4094,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.09158669412136078,
      "rewards/margins": 0.08704870194196701,
      "rewards/rejected": -0.1786354035139084,
      "step": 2790
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.895400955840791e-06,
      "logits/chosen": -2.440882444381714,
      "logits/rejected": -1.7953065633773804,
      "logps/chosen": -222.1812286376953,
      "logps/rejected": -193.8184356689453,
      "loss": 1884.091,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01829930767416954,
      "rewards/margins": 0.13334982097148895,
      "rewards/rejected": -0.15164914727210999,
      "step": 2800
    },
    {
      "epoch": 0.18,
      "eval_logits/chosen": -2.2834653854370117,
      "eval_logits/rejected": -2.0998635292053223,
      "eval_logps/chosen": -236.10433959960938,
      "eval_logps/rejected": -228.92465209960938,
      "eval_loss": 1863.062255859375,
      "eval_rewards/accuracies": 0.6439999938011169,
      "eval_rewards/chosen": -0.04099392145872116,
      "eval_rewards/margins": 0.13213366270065308,
      "eval_rewards/rejected": -0.17312759160995483,
      "eval_runtime": 727.0631,
      "eval_samples_per_second": 2.751,
      "eval_steps_per_second": 1.375,
      "step": 2800
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.893760351180018e-06,
      "logits/chosen": -2.280066967010498,
      "logits/rejected": -2.216137170791626,
      "logps/chosen": -204.33236694335938,
      "logps/rejected": -221.7998504638672,
      "loss": 1961.8484,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05791081115603447,
      "rewards/margins": 0.12366513907909393,
      "rewards/rejected": -0.1815759390592575,
      "step": 2810
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.892107259492657e-06,
      "logits/chosen": -2.232470750808716,
      "logits/rejected": -1.9912874698638916,
      "logps/chosen": -243.64651489257812,
      "logps/rejected": -255.9471893310547,
      "loss": 2238.5156,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03787205368280411,
      "rewards/margins": 0.06281791627407074,
      "rewards/rejected": -0.10068996995687485,
      "step": 2820
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.890441689402042e-06,
      "logits/chosen": -2.360664129257202,
      "logits/rejected": -2.188075542449951,
      "logps/chosen": -341.1266174316406,
      "logps/rejected": -309.93060302734375,
      "loss": 1642.7727,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05621671676635742,
      "rewards/margins": 0.168039932847023,
      "rewards/rejected": -0.22425666451454163,
      "step": 2830
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.888763649596606e-06,
      "logits/chosen": -2.3540730476379395,
      "logits/rejected": -2.131410598754883,
      "logps/chosen": -217.951171875,
      "logps/rejected": -220.7685089111328,
      "loss": 2136.8172,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07789482176303864,
      "rewards/margins": 0.11618749797344208,
      "rewards/rejected": -0.1940823197364807,
      "step": 2840
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.887073148829824e-06,
      "logits/chosen": -2.2597062587738037,
      "logits/rejected": -2.1080522537231445,
      "logps/chosen": -274.22418212890625,
      "logps/rejected": -267.17230224609375,
      "loss": 1813.1502,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07320614904165268,
      "rewards/margins": 0.15883690118789673,
      "rewards/rejected": -0.23204305768013,
      "step": 2850
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.885370195920177e-06,
      "logits/chosen": -2.1275761127471924,
      "logits/rejected": -2.072493314743042,
      "logps/chosen": -201.2732391357422,
      "logps/rejected": -205.1205291748047,
      "loss": 1935.8369,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1594954878091812,
      "rewards/margins": 0.12420445680618286,
      "rewards/rejected": -0.2836999297142029,
      "step": 2860
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.883654799751101e-06,
      "logits/chosen": -2.0364058017730713,
      "logits/rejected": -2.2626266479492188,
      "logps/chosen": -233.87698364257812,
      "logps/rejected": -280.73248291015625,
      "loss": 1835.7615,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1816781461238861,
      "rewards/margins": 0.12115390598773956,
      "rewards/rejected": -0.3028320372104645,
      "step": 2870
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8819269692709435e-06,
      "logits/chosen": -2.3218634128570557,
      "logits/rejected": -2.165917158126831,
      "logps/chosen": -288.1819152832031,
      "logps/rejected": -229.6846160888672,
      "loss": 1725.4633,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.21135666966438293,
      "rewards/margins": 0.13864858448505402,
      "rewards/rejected": -0.35000520944595337,
      "step": 2880
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.880186713492915e-06,
      "logits/chosen": -2.1620290279388428,
      "logits/rejected": -1.9891254901885986,
      "logps/chosen": -253.9564971923828,
      "logps/rejected": -210.16915893554688,
      "loss": 2131.4555,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.26239141821861267,
      "rewards/margins": 0.09834585338830948,
      "rewards/rejected": -0.36073732376098633,
      "step": 2890
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.878434041495041e-06,
      "logits/chosen": -2.218656063079834,
      "logits/rejected": -2.3371100425720215,
      "logps/chosen": -256.22857666015625,
      "logps/rejected": -279.4388427734375,
      "loss": 1507.8515,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.19062820076942444,
      "rewards/margins": 0.21348030865192413,
      "rewards/rejected": -0.40410852432250977,
      "step": 2900
    },
    {
      "epoch": 0.19,
      "eval_logits/chosen": -2.2201428413391113,
      "eval_logits/rejected": -2.0429089069366455,
      "eval_logps/chosen": -251.0389404296875,
      "eval_logps/rejected": -243.269775390625,
      "eval_loss": 1893.5802001953125,
      "eval_rewards/accuracies": 0.6434999704360962,
      "eval_rewards/chosen": -0.19034002721309662,
      "eval_rewards/margins": 0.12623876333236694,
      "eval_rewards/rejected": -0.31657880544662476,
      "eval_runtime": 724.4377,
      "eval_samples_per_second": 2.761,
      "eval_steps_per_second": 1.38,
      "step": 2900
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.876668962420117e-06,
      "logits/chosen": -2.2052314281463623,
      "logits/rejected": -1.9575411081314087,
      "logps/chosen": -305.30145263671875,
      "logps/rejected": -259.78326416015625,
      "loss": 1727.6314,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1565035879611969,
      "rewards/margins": 0.14097970724105835,
      "rewards/rejected": -0.29748329520225525,
      "step": 2910
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.87489148547566e-06,
      "logits/chosen": -2.212196111679077,
      "logits/rejected": -2.0677270889282227,
      "logps/chosen": -280.8736572265625,
      "logps/rejected": -257.7037048339844,
      "loss": 2261.3482,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.21015194058418274,
      "rewards/margins": 0.10002975165843964,
      "rewards/rejected": -0.31018170714378357,
      "step": 2920
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.873101619933862e-06,
      "logits/chosen": -2.4206960201263428,
      "logits/rejected": -2.0775089263916016,
      "logps/chosen": -279.95599365234375,
      "logps/rejected": -240.8511962890625,
      "loss": 2119.6105,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.15428827702999115,
      "rewards/margins": 0.10210802406072617,
      "rewards/rejected": -0.2563963234424591,
      "step": 2930
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8712993751315385e-06,
      "logits/chosen": -2.2349343299865723,
      "logits/rejected": -2.1785027980804443,
      "logps/chosen": -132.24380493164062,
      "logps/rejected": -144.46116638183594,
      "loss": 1970.4197,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.11585874855518341,
      "rewards/margins": 0.08315489441156387,
      "rewards/rejected": -0.1990136355161667,
      "step": 2940
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.869484760470079e-06,
      "logits/chosen": -2.2715892791748047,
      "logits/rejected": -2.0801520347595215,
      "logps/chosen": -198.9450225830078,
      "logps/rejected": -173.6745147705078,
      "loss": 1885.7426,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10284195095300674,
      "rewards/margins": 0.1102866679430008,
      "rewards/rejected": -0.21312859654426575,
      "step": 2950
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.867657785415404e-06,
      "logits/chosen": -2.2036194801330566,
      "logits/rejected": -1.9305412769317627,
      "logps/chosen": -263.3036193847656,
      "logps/rejected": -236.1821746826172,
      "loss": 1663.068,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1420459896326065,
      "rewards/margins": 0.14623163640499115,
      "rewards/rejected": -0.28827762603759766,
      "step": 2960
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.865818459497911e-06,
      "logits/chosen": -2.438999652862549,
      "logits/rejected": -1.9822498559951782,
      "logps/chosen": -299.4256591796875,
      "logps/rejected": -227.34402465820312,
      "loss": 1838.8684,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1524694859981537,
      "rewards/margins": 0.12715639173984528,
      "rewards/rejected": -0.27962589263916016,
      "step": 2970
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.863966792312423e-06,
      "logits/chosen": -2.2882912158966064,
      "logits/rejected": -2.063962459564209,
      "logps/chosen": -253.9712677001953,
      "logps/rejected": -231.25228881835938,
      "loss": 1536.9635,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12512370944023132,
      "rewards/margins": 0.18332403898239136,
      "rewards/rejected": -0.30844777822494507,
      "step": 2980
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.862102793518145e-06,
      "logits/chosen": -2.148599147796631,
      "logits/rejected": -2.2063260078430176,
      "logps/chosen": -214.335693359375,
      "logps/rejected": -231.567138671875,
      "loss": 2156.0416,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.20053806900978088,
      "rewards/margins": 0.10306447744369507,
      "rewards/rejected": -0.30360254645347595,
      "step": 2990
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8602264728386075e-06,
      "logits/chosen": -2.243011951446533,
      "logits/rejected": -2.100947618484497,
      "logps/chosen": -262.6037902832031,
      "logps/rejected": -279.33447265625,
      "loss": 1346.5843,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10104241222143173,
      "rewards/margins": 0.20149335265159607,
      "rewards/rejected": -0.3025357723236084,
      "step": 3000
    },
    {
      "epoch": 0.2,
      "eval_logits/chosen": -2.2279601097106934,
      "eval_logits/rejected": -2.049701452255249,
      "eval_logps/chosen": -240.255126953125,
      "eval_logps/rejected": -234.92210388183594,
      "eval_loss": 1877.9161376953125,
      "eval_rewards/accuracies": 0.6504999995231628,
      "eval_rewards/chosen": -0.08250155299901962,
      "eval_rewards/margins": 0.15060047805309296,
      "eval_rewards/rejected": -0.23310202360153198,
      "eval_runtime": 725.2528,
      "eval_samples_per_second": 2.758,
      "eval_steps_per_second": 1.379,
      "step": 3000
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.858337840061616e-06,
      "logits/chosen": -2.2241852283477783,
      "logits/rejected": -2.142771005630493,
      "logps/chosen": -188.62991333007812,
      "logps/rejected": -255.5631561279297,
      "loss": 2052.9611,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.05549541115760803,
      "rewards/margins": 0.13284072279930115,
      "rewards/rejected": -0.18833611905574799,
      "step": 3010
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.856436905039208e-06,
      "logits/chosen": -2.2276177406311035,
      "logits/rejected": -2.082777738571167,
      "logps/chosen": -218.30136108398438,
      "logps/rejected": -198.83096313476562,
      "loss": 1747.2271,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07096312940120697,
      "rewards/margins": 0.14916923642158508,
      "rewards/rejected": -0.22013239562511444,
      "step": 3020
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.854523677687588e-06,
      "logits/chosen": -2.1049065589904785,
      "logits/rejected": -2.1721158027648926,
      "logps/chosen": -184.58395385742188,
      "logps/rejected": -217.59536743164062,
      "loss": 1815.2102,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03338410705327988,
      "rewards/margins": 0.14614860713481903,
      "rewards/rejected": -0.1795327216386795,
      "step": 3030
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.85259816798709e-06,
      "logits/chosen": -2.30084490776062,
      "logits/rejected": -1.7834408283233643,
      "logps/chosen": -287.6846618652344,
      "logps/rejected": -224.79928588867188,
      "loss": 1338.8376,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.0007740993169136345,
      "rewards/margins": 0.17670534551143646,
      "rewards/rejected": -0.17593125998973846,
      "step": 3040
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.850660385982114e-06,
      "logits/chosen": -2.304020404815674,
      "logits/rejected": -2.145203113555908,
      "logps/chosen": -250.6403045654297,
      "logps/rejected": -208.04324340820312,
      "loss": 1921.6279,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02921169437468052,
      "rewards/margins": 0.1268424391746521,
      "rewards/rejected": -0.15605410933494568,
      "step": 3050
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.848710341781081e-06,
      "logits/chosen": -2.060889720916748,
      "logits/rejected": -2.165678024291992,
      "logps/chosen": -180.858642578125,
      "logps/rejected": -178.9075164794922,
      "loss": 2665.2871,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.06957096606492996,
      "rewards/margins": 0.037756115198135376,
      "rewards/rejected": -0.10732706636190414,
      "step": 3060
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.846748045556377e-06,
      "logits/chosen": -2.2592568397521973,
      "logits/rejected": -1.953974723815918,
      "logps/chosen": -240.12161254882812,
      "logps/rejected": -192.3448944091797,
      "loss": 1955.2693,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.006345144007354975,
      "rewards/margins": 0.1273423135280609,
      "rewards/rejected": -0.13368746638298035,
      "step": 3070
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8447735075442995e-06,
      "logits/chosen": -2.1320343017578125,
      "logits/rejected": -2.200254201889038,
      "logps/chosen": -198.74644470214844,
      "logps/rejected": -212.51657104492188,
      "loss": 1535.8741,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.011842348612844944,
      "rewards/margins": 0.1717064082622528,
      "rewards/rejected": -0.18354876339435577,
      "step": 3080
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8427867380450075e-06,
      "logits/chosen": -2.319826602935791,
      "logits/rejected": -1.9436454772949219,
      "logps/chosen": -234.8804473876953,
      "logps/rejected": -204.9860382080078,
      "loss": 1767.59,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04541159048676491,
      "rewards/margins": 0.16355356574058533,
      "rewards/rejected": -0.20896515250205994,
      "step": 3090
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.840787747422462e-06,
      "logits/chosen": -2.2996203899383545,
      "logits/rejected": -2.048703193664551,
      "logps/chosen": -199.39035034179688,
      "logps/rejected": -182.53619384765625,
      "loss": 1925.325,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03734218329191208,
      "rewards/margins": 0.13403446972370148,
      "rewards/rejected": -0.17137663066387177,
      "step": 3100
    },
    {
      "epoch": 0.2,
      "eval_logits/chosen": -2.252258777618408,
      "eval_logits/rejected": -2.0728886127471924,
      "eval_logps/chosen": -237.34962463378906,
      "eval_logps/rejected": -231.37498474121094,
      "eval_loss": 1869.25146484375,
      "eval_rewards/accuracies": 0.6434999704360962,
      "eval_rewards/chosen": -0.05344657227396965,
      "eval_rewards/margins": 0.1441843956708908,
      "eval_rewards/rejected": -0.19763098657131195,
      "eval_runtime": 724.8757,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.38,
      "step": 3100
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.838776546104378e-06,
      "logits/chosen": -2.225860118865967,
      "logits/rejected": -2.2323122024536133,
      "logps/chosen": -283.26953125,
      "logps/rejected": -259.8985595703125,
      "loss": 1587.6234,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.023162994533777237,
      "rewards/margins": 0.17030958831310272,
      "rewards/rejected": -0.19347259402275085,
      "step": 3110
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.836753144582168e-06,
      "logits/chosen": -2.1938886642456055,
      "logits/rejected": -1.9944331645965576,
      "logps/chosen": -249.22616577148438,
      "logps/rejected": -242.0547332763672,
      "loss": 1700.248,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0699460357427597,
      "rewards/margins": 0.16941139101982117,
      "rewards/rejected": -0.23935742676258087,
      "step": 3120
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.834717553410884e-06,
      "logits/chosen": -2.2574543952941895,
      "logits/rejected": -2.0224785804748535,
      "logps/chosen": -193.8590850830078,
      "logps/rejected": -225.0542449951172,
      "loss": 1770.31,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.045830778777599335,
      "rewards/margins": 0.17525990307331085,
      "rewards/rejected": -0.2210906744003296,
      "step": 3130
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.832669783209167e-06,
      "logits/chosen": -2.1316919326782227,
      "logits/rejected": -2.170029401779175,
      "logps/chosen": -252.727294921875,
      "logps/rejected": -261.55657958984375,
      "loss": 2120.9629,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08301271498203278,
      "rewards/margins": 0.0828041061758995,
      "rewards/rejected": -0.16581682860851288,
      "step": 3140
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.8306098446591895e-06,
      "logits/chosen": -1.8441492319107056,
      "logits/rejected": -2.0120224952697754,
      "logps/chosen": -182.72091674804688,
      "logps/rejected": -216.39108276367188,
      "loss": 2055.383,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06452050805091858,
      "rewards/margins": 0.08738856017589569,
      "rewards/rejected": -0.15190906822681427,
      "step": 3150
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828537748506601e-06,
      "logits/chosen": -2.341561794281006,
      "logits/rejected": -2.0833611488342285,
      "logps/chosen": -279.31024169921875,
      "logps/rejected": -233.48934936523438,
      "loss": 2085.3748,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05911997705698013,
      "rewards/margins": 0.10425326973199844,
      "rewards/rejected": -0.16337324678897858,
      "step": 3160
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.826453505560469e-06,
      "logits/chosen": -2.0433287620544434,
      "logits/rejected": -2.0206546783447266,
      "logps/chosen": -195.5936279296875,
      "logps/rejected": -193.37220764160156,
      "loss": 2035.9855,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.04614509269595146,
      "rewards/margins": 0.11475135385990143,
      "rewards/rejected": -0.1608964502811432,
      "step": 3170
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.824357126693226e-06,
      "logits/chosen": -2.1624441146850586,
      "logits/rejected": -1.7851953506469727,
      "logps/chosen": -264.33087158203125,
      "logps/rejected": -229.04415893554688,
      "loss": 1915.7865,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05587694048881531,
      "rewards/margins": 0.1344069540500641,
      "rewards/rejected": -0.1902839094400406,
      "step": 3180
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.8222486228406105e-06,
      "logits/chosen": -2.3245861530303955,
      "logits/rejected": -2.0476653575897217,
      "logps/chosen": -221.2161102294922,
      "logps/rejected": -202.4019012451172,
      "loss": 1605.2804,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09975741058588028,
      "rewards/margins": 0.15866848826408386,
      "rewards/rejected": -0.25842589139938354,
      "step": 3190
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.820128005001612e-06,
      "logits/chosen": -2.0240657329559326,
      "logits/rejected": -1.9451675415039062,
      "logps/chosen": -227.9483184814453,
      "logps/rejected": -223.92037963867188,
      "loss": 1560.9117,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08954762667417526,
      "rewards/margins": 0.17514896392822266,
      "rewards/rejected": -0.2646965980529785,
      "step": 3200
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": -2.276754379272461,
      "eval_logits/rejected": -2.0942800045013428,
      "eval_logps/chosen": -241.58523559570312,
      "eval_logps/rejected": -235.07289123535156,
      "eval_loss": 1867.82958984375,
      "eval_rewards/accuracies": 0.6434999704360962,
      "eval_rewards/chosen": -0.09580282866954803,
      "eval_rewards/margins": 0.13880720734596252,
      "eval_rewards/rejected": -0.23461003601551056,
      "eval_runtime": 723.6984,
      "eval_samples_per_second": 2.764,
      "eval_steps_per_second": 1.382,
      "step": 3200
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.817995284238412e-06,
      "logits/chosen": -2.069889783859253,
      "logits/rejected": -2.154778480529785,
      "logps/chosen": -202.47483825683594,
      "logps/rejected": -244.8623046875,
      "loss": 1725.6744,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.059504859149456024,
      "rewards/margins": 0.14902649819850922,
      "rewards/rejected": -0.20853137969970703,
      "step": 3210
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.815850471676327e-06,
      "logits/chosen": -2.1980929374694824,
      "logits/rejected": -2.0959055423736572,
      "logps/chosen": -249.5984649658203,
      "logps/rejected": -258.99560546875,
      "loss": 1917.734,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11184433847665787,
      "rewards/margins": 0.13644441962242126,
      "rewards/rejected": -0.24828878045082092,
      "step": 3220
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.813693578503751e-06,
      "logits/chosen": -2.2432820796966553,
      "logits/rejected": -2.066694736480713,
      "logps/chosen": -302.91668701171875,
      "logps/rejected": -265.0500793457031,
      "loss": 1498.3356,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06409894675016403,
      "rewards/margins": 0.17591038346290588,
      "rewards/rejected": -0.2400093525648117,
      "step": 3230
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.811524615972093e-06,
      "logits/chosen": -2.2296462059020996,
      "logits/rejected": -2.0955986976623535,
      "logps/chosen": -240.7917938232422,
      "logps/rejected": -265.7709045410156,
      "loss": 1962.7566,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10856205224990845,
      "rewards/margins": 0.16776728630065918,
      "rewards/rejected": -0.2763293385505676,
      "step": 3240
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.809343595395724e-06,
      "logits/chosen": -2.424299716949463,
      "logits/rejected": -2.241176128387451,
      "logps/chosen": -200.36248779296875,
      "logps/rejected": -184.1348419189453,
      "loss": 1954.6293,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1130683422088623,
      "rewards/margins": 0.1436108499765396,
      "rewards/rejected": -0.2566792070865631,
      "step": 3250
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.807150528151918e-06,
      "logits/chosen": -2.257136344909668,
      "logits/rejected": -2.0939416885375977,
      "logps/chosen": -171.06979370117188,
      "logps/rejected": -207.30239868164062,
      "loss": 1552.4317,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04377254471182823,
      "rewards/margins": 0.18365688621997833,
      "rewards/rejected": -0.22742946445941925,
      "step": 3260
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.804945425680787e-06,
      "logits/chosen": -2.2189509868621826,
      "logits/rejected": -2.2934834957122803,
      "logps/chosen": -195.434326171875,
      "logps/rejected": -185.47146606445312,
      "loss": 1900.6648,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06991083920001984,
      "rewards/margins": 0.10115430504083633,
      "rewards/rejected": -0.17106513679027557,
      "step": 3270
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.802728299485225e-06,
      "logits/chosen": -2.082670211791992,
      "logits/rejected": -2.0714988708496094,
      "logps/chosen": -166.86862182617188,
      "logps/rejected": -196.05703735351562,
      "loss": 2415.0586,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.16264189779758453,
      "rewards/margins": 0.07656769454479218,
      "rewards/rejected": -0.2392096072435379,
      "step": 3280
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.8004991611308495e-06,
      "logits/chosen": -2.3655083179473877,
      "logits/rejected": -2.0764927864074707,
      "logps/chosen": -250.82284545898438,
      "logps/rejected": -250.86312866210938,
      "loss": 1605.4111,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09269125759601593,
      "rewards/margins": 0.16090475022792816,
      "rewards/rejected": -0.2535960078239441,
      "step": 3290
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.798258022245937e-06,
      "logits/chosen": -2.2947518825531006,
      "logits/rejected": -1.8997490406036377,
      "logps/chosen": -232.28384399414062,
      "logps/rejected": -207.6372833251953,
      "loss": 1823.3514,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1467735767364502,
      "rewards/margins": 0.13818533718585968,
      "rewards/rejected": -0.28495892882347107,
      "step": 3300
    },
    {
      "epoch": 0.22,
      "eval_logits/chosen": -2.2288150787353516,
      "eval_logits/rejected": -2.0495386123657227,
      "eval_logps/chosen": -245.45086669921875,
      "eval_logps/rejected": -238.97447204589844,
      "eval_loss": 1848.311279296875,
      "eval_rewards/accuracies": 0.6430000066757202,
      "eval_rewards/chosen": -0.13445906341075897,
      "eval_rewards/margins": 0.13916657865047455,
      "eval_rewards/rejected": -0.2736256420612335,
      "eval_runtime": 723.963,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.381,
      "step": 3300
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.796004894521365e-06,
      "logits/chosen": -2.217214345932007,
      "logits/rejected": -2.036797285079956,
      "logps/chosen": -241.6744842529297,
      "logps/rejected": -277.825439453125,
      "loss": 1920.133,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.13104744255542755,
      "rewards/margins": 0.1489441692829132,
      "rewards/rejected": -0.27999162673950195,
      "step": 3310
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7937397897105545e-06,
      "logits/chosen": -2.206418514251709,
      "logits/rejected": -2.125432014465332,
      "logps/chosen": -219.0151824951172,
      "logps/rejected": -203.31431579589844,
      "loss": 2123.0996,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13782355189323425,
      "rewards/margins": 0.09512574225664139,
      "rewards/rejected": -0.23294928669929504,
      "step": 3320
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.791462719629399e-06,
      "logits/chosen": -2.217872142791748,
      "logits/rejected": -2.0785470008850098,
      "logps/chosen": -197.8411865234375,
      "logps/rejected": -189.520263671875,
      "loss": 1871.5398,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12116026878356934,
      "rewards/margins": 0.1409054398536682,
      "rewards/rejected": -0.26206570863723755,
      "step": 3330
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.789173696156212e-06,
      "logits/chosen": -2.224691390991211,
      "logits/rejected": -1.8815933465957642,
      "logps/chosen": -290.6009826660156,
      "logps/rejected": -292.05194091796875,
      "loss": 1376.5012,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1416037529706955,
      "rewards/margins": 0.201004296541214,
      "rewards/rejected": -0.3426080346107483,
      "step": 3340
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.786872731231662e-06,
      "logits/chosen": -2.2597310543060303,
      "logits/rejected": -2.1441400051116943,
      "logps/chosen": -237.48989868164062,
      "logps/rejected": -237.34518432617188,
      "loss": 1916.399,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20598940551280975,
      "rewards/margins": 0.11224385350942612,
      "rewards/rejected": -0.31823328137397766,
      "step": 3350
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.784559836858709e-06,
      "logits/chosen": -2.252401351928711,
      "logits/rejected": -1.7922512292861938,
      "logps/chosen": -249.43563842773438,
      "logps/rejected": -234.86087036132812,
      "loss": 1483.0201,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.15518534183502197,
      "rewards/margins": 0.1584618091583252,
      "rewards/rejected": -0.3136471211910248,
      "step": 3360
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.782235025102542e-06,
      "logits/chosen": -2.2455966472625732,
      "logits/rejected": -2.182466983795166,
      "logps/chosen": -240.5248260498047,
      "logps/rejected": -238.02346801757812,
      "loss": 1967.6418,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.14205512404441833,
      "rewards/margins": 0.14979180693626404,
      "rewards/rejected": -0.2918469309806824,
      "step": 3370
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.779898308090519e-06,
      "logits/chosen": -2.2079274654388428,
      "logits/rejected": -1.9989955425262451,
      "logps/chosen": -286.9621276855469,
      "logps/rejected": -267.46014404296875,
      "loss": 1900.592,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12284872680902481,
      "rewards/margins": 0.17328739166259766,
      "rewards/rejected": -0.2961360812187195,
      "step": 3380
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.777549698012101e-06,
      "logits/chosen": -2.1511409282684326,
      "logits/rejected": -1.9988940954208374,
      "logps/chosen": -259.1100158691406,
      "logps/rejected": -258.4521179199219,
      "loss": 1776.992,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15893355011940002,
      "rewards/margins": 0.1850178986787796,
      "rewards/rejected": -0.34395143389701843,
      "step": 3390
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.775189207118787e-06,
      "logits/chosen": -2.18037748336792,
      "logits/rejected": -1.9939063787460327,
      "logps/chosen": -288.7840270996094,
      "logps/rejected": -281.2822265625,
      "loss": 1617.63,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1696634292602539,
      "rewards/margins": 0.14835859835147858,
      "rewards/rejected": -0.3180220425128937,
      "step": 3400
    },
    {
      "epoch": 0.22,
      "eval_logits/chosen": -2.2290878295898438,
      "eval_logits/rejected": -2.049717903137207,
      "eval_logps/chosen": -249.02952575683594,
      "eval_logps/rejected": -242.2179718017578,
      "eval_loss": 1866.488525390625,
      "eval_rewards/accuracies": 0.6414999961853027,
      "eval_rewards/chosen": -0.1702459454536438,
      "eval_rewards/margins": 0.1358148157596588,
      "eval_rewards/rejected": -0.3060607314109802,
      "eval_runtime": 723.452,
      "eval_samples_per_second": 2.765,
      "eval_steps_per_second": 1.382,
      "step": 3400
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.772816847724054e-06,
      "logits/chosen": -2.3193163871765137,
      "logits/rejected": -2.0606117248535156,
      "logps/chosen": -241.829833984375,
      "logps/rejected": -251.56503295898438,
      "loss": 2431.6721,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1927526593208313,
      "rewards/margins": 0.10237501561641693,
      "rewards/rejected": -0.2951276898384094,
      "step": 3410
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.770432632203294e-06,
      "logits/chosen": -2.0573058128356934,
      "logits/rejected": -1.9737217426300049,
      "logps/chosen": -270.8164978027344,
      "logps/rejected": -226.3074493408203,
      "loss": 2336.4207,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.27169421315193176,
      "rewards/margins": 0.05809188634157181,
      "rewards/rejected": -0.3297860622406006,
      "step": 3420
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.768036572993738e-06,
      "logits/chosen": -2.1174683570861816,
      "logits/rejected": -2.2044284343719482,
      "logps/chosen": -303.7652282714844,
      "logps/rejected": -298.9791259765625,
      "loss": 2009.1703,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23746299743652344,
      "rewards/margins": 0.10813780128955841,
      "rewards/rejected": -0.34560078382492065,
      "step": 3430
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.765628682594409e-06,
      "logits/chosen": -2.297766923904419,
      "logits/rejected": -2.1239116191864014,
      "logps/chosen": -262.238037109375,
      "logps/rejected": -249.93252563476562,
      "loss": 1776.9062,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1848095953464508,
      "rewards/margins": 0.11596927791833878,
      "rewards/rejected": -0.3007788360118866,
      "step": 3440
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.763208973566041e-06,
      "logits/chosen": -2.104485034942627,
      "logits/rejected": -2.1253323554992676,
      "logps/chosen": -205.01254272460938,
      "logps/rejected": -235.3104248046875,
      "loss": 1871.548,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.20882877707481384,
      "rewards/margins": 0.11582591384649277,
      "rewards/rejected": -0.3246546685695648,
      "step": 3450
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.76077745853102e-06,
      "logits/chosen": -2.296905755996704,
      "logits/rejected": -2.159364700317383,
      "logps/chosen": -283.6266174316406,
      "logps/rejected": -297.8143615722656,
      "loss": 2050.5668,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.33572444319725037,
      "rewards/margins": 0.13062633574008942,
      "rewards/rejected": -0.4663507342338562,
      "step": 3460
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.758334150173322e-06,
      "logits/chosen": -2.139819622039795,
      "logits/rejected": -1.965150237083435,
      "logps/chosen": -287.6877746582031,
      "logps/rejected": -279.01116943359375,
      "loss": 1550.6515,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.24603800475597382,
      "rewards/margins": 0.16377019882202148,
      "rewards/rejected": -0.4098082184791565,
      "step": 3470
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.755879061238439e-06,
      "logits/chosen": -2.2042808532714844,
      "logits/rejected": -1.9885804653167725,
      "logps/chosen": -289.332275390625,
      "logps/rejected": -292.26824951171875,
      "loss": 2013.2613,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3315892219543457,
      "rewards/margins": 0.15573135018348694,
      "rewards/rejected": -0.48732057213783264,
      "step": 3480
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.753412204533317e-06,
      "logits/chosen": -2.364527702331543,
      "logits/rejected": -1.8919954299926758,
      "logps/chosen": -292.3900451660156,
      "logps/rejected": -264.13616943359375,
      "loss": 1665.785,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.29531973600387573,
      "rewards/margins": 0.17137546837329865,
      "rewards/rejected": -0.4666951596736908,
      "step": 3490
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.750933592926292e-06,
      "logits/chosen": -2.2476608753204346,
      "logits/rejected": -1.9270604848861694,
      "logps/chosen": -255.2253875732422,
      "logps/rejected": -246.07723999023438,
      "loss": 1438.7666,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.36497727036476135,
      "rewards/margins": 0.18068274855613708,
      "rewards/rejected": -0.545660138130188,
      "step": 3500
    },
    {
      "epoch": 0.23,
      "eval_logits/chosen": -2.1436102390289307,
      "eval_logits/rejected": -1.970076322555542,
      "eval_logps/chosen": -263.0986022949219,
      "eval_logps/rejected": -256.9674072265625,
      "eval_loss": 1877.9581298828125,
      "eval_rewards/accuracies": 0.6470000147819519,
      "eval_rewards/chosen": -0.31093651056289673,
      "eval_rewards/margins": 0.14261873066425323,
      "eval_rewards/rejected": -0.45355525612831116,
      "eval_runtime": 724.8549,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.38,
      "step": 3500
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7484432393470124e-06,
      "logits/chosen": -2.3309848308563232,
      "logits/rejected": -1.824968695640564,
      "logps/chosen": -228.2218475341797,
      "logps/rejected": -190.5856475830078,
      "loss": 1673.3477,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.26139897108078003,
      "rewards/margins": 0.17760643362998962,
      "rewards/rejected": -0.43900543451309204,
      "step": 3510
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.745941156786385e-06,
      "logits/chosen": -1.9236522912979126,
      "logits/rejected": -1.9156990051269531,
      "logps/chosen": -181.29397583007812,
      "logps/rejected": -226.3966522216797,
      "loss": 1797.2092,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.28254860639572144,
      "rewards/margins": 0.14261487126350403,
      "rewards/rejected": -0.42516350746154785,
      "step": 3520
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.743427358296497e-06,
      "logits/chosen": -2.053074359893799,
      "logits/rejected": -1.8683373928070068,
      "logps/chosen": -209.05892944335938,
      "logps/rejected": -242.22012329101562,
      "loss": 1428.9707,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1935068666934967,
      "rewards/margins": 0.21853800117969513,
      "rewards/rejected": -0.41204482316970825,
      "step": 3530
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.740901856990553e-06,
      "logits/chosen": -1.9476228952407837,
      "logits/rejected": -1.7711150646209717,
      "logps/chosen": -279.8335266113281,
      "logps/rejected": -244.2646026611328,
      "loss": 2220.4473,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.23141586780548096,
      "rewards/margins": 0.08503684401512146,
      "rewards/rejected": -0.31645268201828003,
      "step": 3540
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.738364666042804e-06,
      "logits/chosen": -2.169304132461548,
      "logits/rejected": -1.770307183265686,
      "logps/chosen": -307.50030517578125,
      "logps/rejected": -260.1822814941406,
      "loss": 1543.7248,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15755341947078705,
      "rewards/margins": 0.16321787238121033,
      "rewards/rejected": -0.32077130675315857,
      "step": 3550
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.735815798688483e-06,
      "logits/chosen": -2.104562282562256,
      "logits/rejected": -1.8917967081069946,
      "logps/chosen": -219.07351684570312,
      "logps/rejected": -255.24197387695312,
      "loss": 1848.7328,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2074153870344162,
      "rewards/margins": 0.1602933704853058,
      "rewards/rejected": -0.36770880222320557,
      "step": 3560
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7332552682237285e-06,
      "logits/chosen": -2.1062798500061035,
      "logits/rejected": -1.6887365579605103,
      "logps/chosen": -188.10646057128906,
      "logps/rejected": -194.13392639160156,
      "loss": 1267.9649,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.14346513152122498,
      "rewards/margins": 0.21799930930137634,
      "rewards/rejected": -0.3614644408226013,
      "step": 3570
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7306830880055234e-06,
      "logits/chosen": -2.1045773029327393,
      "logits/rejected": -2.0392093658447266,
      "logps/chosen": -217.57101440429688,
      "logps/rejected": -236.7305908203125,
      "loss": 2163.6355,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2683719992637634,
      "rewards/margins": 0.10822489112615585,
      "rewards/rejected": -0.3765968680381775,
      "step": 3580
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.728099271451619e-06,
      "logits/chosen": -2.129319190979004,
      "logits/rejected": -2.060136318206787,
      "logps/chosen": -212.7565155029297,
      "logps/rejected": -218.15103149414062,
      "loss": 1891.9111,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.21748504042625427,
      "rewards/margins": 0.12347017228603363,
      "rewards/rejected": -0.3409552276134491,
      "step": 3590
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.725503832040466e-06,
      "logits/chosen": -1.9385948181152344,
      "logits/rejected": -1.9002023935317993,
      "logps/chosen": -169.0155487060547,
      "logps/rejected": -205.9110565185547,
      "loss": 2120.1508,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.198576882481575,
      "rewards/margins": 0.10808098316192627,
      "rewards/rejected": -0.3066578805446625,
      "step": 3600
    },
    {
      "epoch": 0.24,
      "eval_logits/chosen": -2.0810132026672363,
      "eval_logits/rejected": -1.910104513168335,
      "eval_logps/chosen": -255.11273193359375,
      "eval_logps/rejected": -248.61317443847656,
      "eval_loss": 1857.2381591796875,
      "eval_rewards/accuracies": 0.6504999995231628,
      "eval_rewards/chosen": -0.23107774555683136,
      "eval_rewards/margins": 0.1389349102973938,
      "eval_rewards/rejected": -0.37001264095306396,
      "eval_runtime": 726.2968,
      "eval_samples_per_second": 2.754,
      "eval_steps_per_second": 1.377,
      "step": 3600
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.722896783311152e-06,
      "logits/chosen": -2.076775074005127,
      "logits/rejected": -1.9659488201141357,
      "logps/chosen": -285.59515380859375,
      "logps/rejected": -348.5102233886719,
      "loss": 2152.6373,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2654615044593811,
      "rewards/margins": 0.11781106889247894,
      "rewards/rejected": -0.38327255845069885,
      "step": 3610
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.720278138863318e-06,
      "logits/chosen": -2.2312514781951904,
      "logits/rejected": -2.0606610774993896,
      "logps/chosen": -212.52163696289062,
      "logps/rejected": -191.35154724121094,
      "loss": 2192.3848,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.22399914264678955,
      "rewards/margins": 0.106370709836483,
      "rewards/rejected": -0.33036983013153076,
      "step": 3620
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.717647912357095e-06,
      "logits/chosen": -2.2318344116210938,
      "logits/rejected": -2.3128485679626465,
      "logps/chosen": -297.67083740234375,
      "logps/rejected": -313.06500244140625,
      "loss": 2577.318,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.25193700194358826,
      "rewards/margins": 0.024617036804556847,
      "rewards/rejected": -0.27655404806137085,
      "step": 3630
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.715006117513035e-06,
      "logits/chosen": -2.3442459106445312,
      "logits/rejected": -2.1202805042266846,
      "logps/chosen": -340.41632080078125,
      "logps/rejected": -299.45916748046875,
      "loss": 1648.0377,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1629071980714798,
      "rewards/margins": 0.14222896099090576,
      "rewards/rejected": -0.30513614416122437,
      "step": 3640
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.7123527681120326e-06,
      "logits/chosen": -2.16243052482605,
      "logits/rejected": -2.0179338455200195,
      "logps/chosen": -266.94195556640625,
      "logps/rejected": -250.9875946044922,
      "loss": 1774.2609,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.197707861661911,
      "rewards/margins": 0.12197317183017731,
      "rewards/rejected": -0.3196810781955719,
      "step": 3650
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.7096878779952594e-06,
      "logits/chosen": -2.2487692832946777,
      "logits/rejected": -2.1916444301605225,
      "logps/chosen": -296.69451904296875,
      "logps/rejected": -311.2198791503906,
      "loss": 1489.8128,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.21281103789806366,
      "rewards/margins": 0.17108117043972015,
      "rewards/rejected": -0.3838922083377838,
      "step": 3660
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.707011461064086e-06,
      "logits/chosen": -2.036893844604492,
      "logits/rejected": -1.7952464818954468,
      "logps/chosen": -326.61944580078125,
      "logps/rejected": -300.3973083496094,
      "loss": 1669.2322,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.17760398983955383,
      "rewards/margins": 0.17968544363975525,
      "rewards/rejected": -0.35728946328163147,
      "step": 3670
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.704323531280016e-06,
      "logits/chosen": -2.0617146492004395,
      "logits/rejected": -1.9116485118865967,
      "logps/chosen": -344.8534240722656,
      "logps/rejected": -277.119140625,
      "loss": 1866.4686,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.17641881108283997,
      "rewards/margins": 0.1546126902103424,
      "rewards/rejected": -0.3310315012931824,
      "step": 3680
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.701624102664606e-06,
      "logits/chosen": -2.2474734783172607,
      "logits/rejected": -1.929483413696289,
      "logps/chosen": -283.1015625,
      "logps/rejected": -239.931640625,
      "loss": 2034.3928,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.21985697746276855,
      "rewards/margins": 0.10930268466472626,
      "rewards/rejected": -0.329159677028656,
      "step": 3690
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.698913189299399e-06,
      "logits/chosen": -2.0997631549835205,
      "logits/rejected": -2.1936910152435303,
      "logps/chosen": -202.63076782226562,
      "logps/rejected": -245.99765014648438,
      "loss": 2384.8625,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.16478955745697021,
      "rewards/margins": 0.10860179364681244,
      "rewards/rejected": -0.27339136600494385,
      "step": 3700
    },
    {
      "epoch": 0.24,
      "eval_logits/chosen": -2.1779725551605225,
      "eval_logits/rejected": -2.0020358562469482,
      "eval_logps/chosen": -247.79246520996094,
      "eval_logps/rejected": -241.4032745361328,
      "eval_loss": 1852.2376708984375,
      "eval_rewards/accuracies": 0.6499999761581421,
      "eval_rewards/chosen": -0.15787512063980103,
      "eval_rewards/margins": 0.1400386244058609,
      "eval_rewards/rejected": -0.29791373014450073,
      "eval_runtime": 724.8438,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.38,
      "step": 3700
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.696190805325847e-06,
      "logits/chosen": -2.2065725326538086,
      "logits/rejected": -2.0684335231781006,
      "logps/chosen": -220.91055297851562,
      "logps/rejected": -209.87887573242188,
      "loss": 1673.4158,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14531712234020233,
      "rewards/margins": 0.16087010502815247,
      "rewards/rejected": -0.306187242269516,
      "step": 3710
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.693456964945239e-06,
      "logits/chosen": -2.320401191711426,
      "logits/rejected": -1.8658478260040283,
      "logps/chosen": -307.8102722167969,
      "logps/rejected": -225.3717041015625,
      "loss": 1202.8632,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0970473513007164,
      "rewards/margins": 0.18653647601604462,
      "rewards/rejected": -0.2835838198661804,
      "step": 3720
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.6907116824186245e-06,
      "logits/chosen": -2.2743570804595947,
      "logits/rejected": -2.231112241744995,
      "logps/chosen": -236.314697265625,
      "logps/rejected": -245.04940795898438,
      "loss": 2315.3547,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08987172693014145,
      "rewards/margins": 0.08930940926074982,
      "rewards/rejected": -0.17918114364147186,
      "step": 3730
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.687954972066742e-06,
      "logits/chosen": -2.1849050521850586,
      "logits/rejected": -1.9289398193359375,
      "logps/chosen": -239.7222137451172,
      "logps/rejected": -239.80740356445312,
      "loss": 1672.2711,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09905945509672165,
      "rewards/margins": 0.2095617949962616,
      "rewards/rejected": -0.30862125754356384,
      "step": 3740
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.685186848269944e-06,
      "logits/chosen": -2.1947669982910156,
      "logits/rejected": -2.0647358894348145,
      "logps/chosen": -222.42724609375,
      "logps/rejected": -196.56259155273438,
      "loss": 1854.6092,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10554896295070648,
      "rewards/margins": 0.1151428073644638,
      "rewards/rejected": -0.2206917703151703,
      "step": 3750
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.682407325468119e-06,
      "logits/chosen": -2.277951717376709,
      "logits/rejected": -1.9266493320465088,
      "logps/chosen": -225.47329711914062,
      "logps/rejected": -207.5655517578125,
      "loss": 1687.2664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08054288476705551,
      "rewards/margins": 0.15721990168094635,
      "rewards/rejected": -0.23776277899742126,
      "step": 3760
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.67961641816062e-06,
      "logits/chosen": -2.248936176300049,
      "logits/rejected": -2.052278757095337,
      "logps/chosen": -284.053955078125,
      "logps/rejected": -254.1475067138672,
      "loss": 2078.1754,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08257193863391876,
      "rewards/margins": 0.1142364963889122,
      "rewards/rejected": -0.19680842757225037,
      "step": 3770
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.676814140906188e-06,
      "logits/chosen": -2.1284399032592773,
      "logits/rejected": -2.001819133758545,
      "logps/chosen": -254.8821258544922,
      "logps/rejected": -239.061279296875,
      "loss": 1785.2656,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15345919132232666,
      "rewards/margins": 0.14784851670265198,
      "rewards/rejected": -0.301307737827301,
      "step": 3780
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.674000508322872e-06,
      "logits/chosen": -1.996980905532837,
      "logits/rejected": -2.069845199584961,
      "logps/chosen": -233.3638458251953,
      "logps/rejected": -250.4616241455078,
      "loss": 2388.3703,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15828964114189148,
      "rewards/margins": 0.07914143055677414,
      "rewards/rejected": -0.23743107914924622,
      "step": 3790
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.671175535087959e-06,
      "logits/chosen": -2.1575369834899902,
      "logits/rejected": -2.1205859184265137,
      "logps/chosen": -308.8890075683594,
      "logps/rejected": -327.61541748046875,
      "loss": 1321.1524,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.17561353743076324,
      "rewards/margins": 0.21958065032958984,
      "rewards/rejected": -0.3951941728591919,
      "step": 3800
    },
    {
      "epoch": 0.25,
      "eval_logits/chosen": -2.2452280521392822,
      "eval_logits/rejected": -2.065730571746826,
      "eval_logps/chosen": -247.39849853515625,
      "eval_logps/rejected": -239.1979217529297,
      "eval_loss": 1872.46875,
      "eval_rewards/accuracies": 0.6470000147819519,
      "eval_rewards/chosen": -0.15393544733524323,
      "eval_rewards/margins": 0.12192466855049133,
      "eval_rewards/rejected": -0.27586010098457336,
      "eval_runtime": 724.0232,
      "eval_samples_per_second": 2.762,
      "eval_steps_per_second": 1.381,
      "step": 3800
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6683392359378924e-06,
      "logits/chosen": -2.1318156719207764,
      "logits/rejected": -1.9592097997665405,
      "logps/chosen": -247.49972534179688,
      "logps/rejected": -237.6021270751953,
      "loss": 1829.1955,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12350733578205109,
      "rewards/margins": 0.15618854761123657,
      "rewards/rejected": -0.27969586849212646,
      "step": 3810
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.665491625668198e-06,
      "logits/chosen": -2.059387683868408,
      "logits/rejected": -2.105896472930908,
      "logps/chosen": -172.2931671142578,
      "logps/rejected": -207.222412109375,
      "loss": 2150.1033,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.16045895218849182,
      "rewards/margins": 0.11374203115701675,
      "rewards/rejected": -0.274200975894928,
      "step": 3820
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.662632719133407e-06,
      "logits/chosen": -2.342655658721924,
      "logits/rejected": -2.0594210624694824,
      "logps/chosen": -238.0370635986328,
      "logps/rejected": -183.14178466796875,
      "loss": 1867.0152,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07239507883787155,
      "rewards/margins": 0.12112174183130264,
      "rewards/rejected": -0.1935168206691742,
      "step": 3830
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.659762531246974e-06,
      "logits/chosen": -2.2439637184143066,
      "logits/rejected": -2.091200590133667,
      "logps/chosen": -231.6832275390625,
      "logps/rejected": -206.22177124023438,
      "loss": 2122.8434,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17104485630989075,
      "rewards/margins": 0.08495251089334488,
      "rewards/rejected": -0.25599735975265503,
      "step": 3840
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.656881076981207e-06,
      "logits/chosen": -2.2985904216766357,
      "logits/rejected": -2.1653382778167725,
      "logps/chosen": -224.47488403320312,
      "logps/rejected": -211.1822052001953,
      "loss": 2243.6316,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.11291512101888657,
      "rewards/margins": 0.046060457825660706,
      "rewards/rejected": -0.15897558629512787,
      "step": 3850
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.653988371367183e-06,
      "logits/chosen": -2.231370449066162,
      "logits/rejected": -1.9748144149780273,
      "logps/chosen": -251.9241485595703,
      "logps/rejected": -203.20687866210938,
      "loss": 1686.8398,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11120152473449707,
      "rewards/margins": 0.1249074935913086,
      "rewards/rejected": -0.23610901832580566,
      "step": 3860
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.651084429494671e-06,
      "logits/chosen": -2.2892768383026123,
      "logits/rejected": -2.0120041370391846,
      "logps/chosen": -288.3143005371094,
      "logps/rejected": -224.89505004882812,
      "loss": 1673.5797,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.12963344156742096,
      "rewards/margins": 0.1610449105501175,
      "rewards/rejected": -0.29067832231521606,
      "step": 3870
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.648169266512053e-06,
      "logits/chosen": -2.3402390480041504,
      "logits/rejected": -2.069182872772217,
      "logps/chosen": -234.9876251220703,
      "logps/rejected": -200.89300537109375,
      "loss": 1949.3049,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09659501165151596,
      "rewards/margins": 0.10859502851963043,
      "rewards/rejected": -0.2051900327205658,
      "step": 3880
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6452428976262505e-06,
      "logits/chosen": -2.149710178375244,
      "logits/rejected": -1.9293615818023682,
      "logps/chosen": -209.88943481445312,
      "logps/rejected": -186.5958709716797,
      "loss": 1336.7575,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06097991019487381,
      "rewards/margins": 0.21029214560985565,
      "rewards/rejected": -0.27127203345298767,
      "step": 3890
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.642305338102633e-06,
      "logits/chosen": -2.1881473064422607,
      "logits/rejected": -2.272202968597412,
      "logps/chosen": -169.59375,
      "logps/rejected": -199.1464385986328,
      "loss": 1975.567,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07844208925962448,
      "rewards/margins": 0.11406981945037842,
      "rewards/rejected": -0.1925118863582611,
      "step": 3900
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.211078643798828,
      "eval_logits/rejected": -2.0324509143829346,
      "eval_logps/chosen": -237.15188598632812,
      "eval_logps/rejected": -232.16534423828125,
      "eval_loss": 1848.4305419921875,
      "eval_rewards/accuracies": 0.6554999947547913,
      "eval_rewards/chosen": -0.05146953463554382,
      "eval_rewards/margins": 0.15406478941440582,
      "eval_rewards/rejected": -0.20553433895111084,
      "eval_runtime": 725.9243,
      "eval_samples_per_second": 2.755,
      "eval_steps_per_second": 1.378,
      "step": 3900
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.639356603264953e-06,
      "logits/chosen": -2.2523646354675293,
      "logits/rejected": -2.032790184020996,
      "logps/chosen": -248.07278442382812,
      "logps/rejected": -236.7919921875,
      "loss": 2005.1246,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.062409840524196625,
      "rewards/margins": 0.14450041949748993,
      "rewards/rejected": -0.20691028237342834,
      "step": 3910
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.636396708495255e-06,
      "logits/chosen": -2.09509015083313,
      "logits/rejected": -2.0792410373687744,
      "logps/chosen": -234.39501953125,
      "logps/rejected": -227.2949676513672,
      "loss": 1513.6564,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05199925974011421,
      "rewards/margins": 0.17743878066539764,
      "rewards/rejected": -0.22943802177906036,
      "step": 3920
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.633425669233799e-06,
      "logits/chosen": -2.2118465900421143,
      "logits/rejected": -2.2278549671173096,
      "logps/chosen": -244.0120086669922,
      "logps/rejected": -249.1499786376953,
      "loss": 2088.2393,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09185406565666199,
      "rewards/margins": 0.09585288166999817,
      "rewards/rejected": -0.18770697712898254,
      "step": 3930
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6304435009789825e-06,
      "logits/chosen": -2.2175028324127197,
      "logits/rejected": -1.9971988201141357,
      "logps/chosen": -243.27426147460938,
      "logps/rejected": -186.73477172851562,
      "loss": 1765.7463,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05767058581113815,
      "rewards/margins": 0.14370371401309967,
      "rewards/rejected": -0.20137432217597961,
      "step": 3940
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.627450219287256e-06,
      "logits/chosen": -2.2618629932403564,
      "logits/rejected": -2.1243340969085693,
      "logps/chosen": -192.6259765625,
      "logps/rejected": -181.14503479003906,
      "loss": 2074.3313,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11066880077123642,
      "rewards/margins": 0.1103011816740036,
      "rewards/rejected": -0.22096994519233704,
      "step": 3950
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.624445839773042e-06,
      "logits/chosen": -2.2068986892700195,
      "logits/rejected": -2.1380820274353027,
      "logps/chosen": -184.40818786621094,
      "logps/rejected": -188.77137756347656,
      "loss": 2496.201,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.1439688503742218,
      "rewards/margins": 0.05427010729908943,
      "rewards/rejected": -0.19823895394802094,
      "step": 3960
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.621430378108656e-06,
      "logits/chosen": -2.197033405303955,
      "logits/rejected": -2.040515184402466,
      "logps/chosen": -273.7949523925781,
      "logps/rejected": -280.93048095703125,
      "loss": 1702.4613,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1603659689426422,
      "rewards/margins": 0.13899901509284973,
      "rewards/rejected": -0.29936498403549194,
      "step": 3970
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.618403850024223e-06,
      "logits/chosen": -2.104780435562134,
      "logits/rejected": -1.8865197896957397,
      "logps/chosen": -268.41644287109375,
      "logps/rejected": -232.78341674804688,
      "loss": 1980.4014,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.129262313246727,
      "rewards/margins": 0.11029335111379623,
      "rewards/rejected": -0.23955564200878143,
      "step": 3980
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.615366271307598e-06,
      "logits/chosen": -2.266728162765503,
      "logits/rejected": -2.113433361053467,
      "logps/chosen": -208.0624542236328,
      "logps/rejected": -208.528076171875,
      "loss": 1785.8047,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16119475662708282,
      "rewards/margins": 0.12357983738183975,
      "rewards/rejected": -0.2847746014595032,
      "step": 3990
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.612317657804277e-06,
      "logits/chosen": -2.1275062561035156,
      "logits/rejected": -2.176178455352783,
      "logps/chosen": -163.8437042236328,
      "logps/rejected": -229.2069854736328,
      "loss": 1914.4258,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.16610661149024963,
      "rewards/margins": 0.14015412330627441,
      "rewards/rejected": -0.30626070499420166,
      "step": 4000
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.2187206745147705,
      "eval_logits/rejected": -2.0401241779327393,
      "eval_logps/chosen": -245.95481872558594,
      "eval_logps/rejected": -239.28158569335938,
      "eval_loss": 1862.8907470703125,
      "eval_rewards/accuracies": 0.6535000205039978,
      "eval_rewards/chosen": -0.13949863612651825,
      "eval_rewards/margins": 0.13719816505908966,
      "eval_rewards/rejected": -0.2766967713832855,
      "eval_runtime": 727.2291,
      "eval_samples_per_second": 2.75,
      "eval_steps_per_second": 1.375,
      "step": 4000
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6092580254173236e-06,
      "logits/chosen": -2.1136960983276367,
      "logits/rejected": -1.892932653427124,
      "logps/chosen": -269.98236083984375,
      "logps/rejected": -268.93701171875,
      "loss": 1639.9719,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12659600377082825,
      "rewards/margins": 0.17622780799865723,
      "rewards/rejected": -0.3028238117694855,
      "step": 4010
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.606187390107277e-06,
      "logits/chosen": -2.0953426361083984,
      "logits/rejected": -1.9211162328720093,
      "logps/chosen": -241.5983428955078,
      "logps/rejected": -212.2313232421875,
      "loss": 2033.3469,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.167229562997818,
      "rewards/margins": 0.09625671803951263,
      "rewards/rejected": -0.26348626613616943,
      "step": 4020
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.603105767892077e-06,
      "logits/chosen": -2.2119300365448,
      "logits/rejected": -2.1371030807495117,
      "logps/chosen": -207.75601196289062,
      "logps/rejected": -240.72824096679688,
      "loss": 1638.3612,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.13065990805625916,
      "rewards/margins": 0.15018993616104126,
      "rewards/rejected": -0.2808498442173004,
      "step": 4030
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6000131748469725e-06,
      "logits/chosen": -2.27934193611145,
      "logits/rejected": -1.9166730642318726,
      "logps/chosen": -256.64068603515625,
      "logps/rejected": -193.72409057617188,
      "loss": 2078.6699,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.06500424444675446,
      "rewards/margins": 0.11401742696762085,
      "rewards/rejected": -0.1790216565132141,
      "step": 4040
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.596909627104445e-06,
      "logits/chosen": -2.3300695419311523,
      "logits/rejected": -2.251364231109619,
      "logps/chosen": -260.0342102050781,
      "logps/rejected": -241.5802764892578,
      "loss": 2057.6164,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11295942217111588,
      "rewards/margins": 0.1418023258447647,
      "rewards/rejected": -0.2547617554664612,
      "step": 4050
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5937951408541215e-06,
      "logits/chosen": -2.369083881378174,
      "logits/rejected": -1.8532183170318604,
      "logps/chosen": -262.75360107421875,
      "logps/rejected": -235.9652099609375,
      "loss": 1622.623,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10115152597427368,
      "rewards/margins": 0.185886412858963,
      "rewards/rejected": -0.2870379388332367,
      "step": 4060
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.590669732342685e-06,
      "logits/chosen": -2.0819499492645264,
      "logits/rejected": -1.943646788597107,
      "logps/chosen": -220.8902130126953,
      "logps/rejected": -239.7845001220703,
      "loss": 1625.7482,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08742596954107285,
      "rewards/margins": 0.1700166016817093,
      "rewards/rejected": -0.25744256377220154,
      "step": 4070
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.587533417873799e-06,
      "logits/chosen": -2.1543188095092773,
      "logits/rejected": -2.2358295917510986,
      "logps/chosen": -207.8849334716797,
      "logps/rejected": -284.0242614746094,
      "loss": 1614.4966,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14720898866653442,
      "rewards/margins": 0.16435673832893372,
      "rewards/rejected": -0.31156572699546814,
      "step": 4080
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.584386213808016e-06,
      "logits/chosen": -2.1320371627807617,
      "logits/rejected": -1.815946340560913,
      "logps/chosen": -235.2030792236328,
      "logps/rejected": -201.69064331054688,
      "loss": 1804.5004,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13524064421653748,
      "rewards/margins": 0.12263710796833038,
      "rewards/rejected": -0.25787776708602905,
      "step": 4090
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.581228136562693e-06,
      "logits/chosen": -2.03398060798645,
      "logits/rejected": -2.144202947616577,
      "logps/chosen": -250.9056854248047,
      "logps/rejected": -231.795654296875,
      "loss": 2663.7783,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1272524893283844,
      "rewards/margins": 0.06574568152427673,
      "rewards/rejected": -0.19299815595149994,
      "step": 4100
    },
    {
      "epoch": 0.27,
      "eval_logits/chosen": -2.215273141860962,
      "eval_logits/rejected": -2.0376996994018555,
      "eval_logps/chosen": -247.2678680419922,
      "eval_logps/rejected": -240.6035614013672,
      "eval_loss": 1853.19189453125,
      "eval_rewards/accuracies": 0.6535000205039978,
      "eval_rewards/chosen": -0.15262925624847412,
      "eval_rewards/margins": 0.13728740811347961,
      "eval_rewards/rejected": -0.28991666436195374,
      "eval_runtime": 724.4867,
      "eval_samples_per_second": 2.761,
      "eval_steps_per_second": 1.38,
      "step": 4100
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.578059202611909e-06,
      "logits/chosen": -2.2305524349212646,
      "logits/rejected": -2.020010471343994,
      "logps/chosen": -270.2746276855469,
      "logps/rejected": -265.7089538574219,
      "loss": 2003.9635,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13115721940994263,
      "rewards/margins": 0.10190287977457047,
      "rewards/rejected": -0.2330600768327713,
      "step": 4110
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.574879428486376e-06,
      "logits/chosen": -2.208253860473633,
      "logits/rejected": -1.9521793127059937,
      "logps/chosen": -231.3865203857422,
      "logps/rejected": -243.73385620117188,
      "loss": 1639.358,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.18469981849193573,
      "rewards/margins": 0.15269534289836884,
      "rewards/rejected": -0.33739519119262695,
      "step": 4120
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.571688830773352e-06,
      "logits/chosen": -2.236798048019409,
      "logits/rejected": -2.132211208343506,
      "logps/chosen": -235.32858276367188,
      "logps/rejected": -223.46475219726562,
      "loss": 2082.0211,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1326039731502533,
      "rewards/margins": 0.08015134930610657,
      "rewards/rejected": -0.21275529265403748,
      "step": 4130
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.568487426116559e-06,
      "logits/chosen": -2.1624183654785156,
      "logits/rejected": -2.169710636138916,
      "logps/chosen": -180.14920043945312,
      "logps/rejected": -184.7618865966797,
      "loss": 1673.3771,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08177350461483002,
      "rewards/margins": 0.12146095186471939,
      "rewards/rejected": -0.20323446393013,
      "step": 4140
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.565275231216092e-06,
      "logits/chosen": -2.066892385482788,
      "logits/rejected": -2.044159412384033,
      "logps/chosen": -160.7727813720703,
      "logps/rejected": -217.7138671875,
      "loss": 1986.8971,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09429313987493515,
      "rewards/margins": 0.1095115914940834,
      "rewards/rejected": -0.20380473136901855,
      "step": 4150
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.562052262828331e-06,
      "logits/chosen": -2.1357436180114746,
      "logits/rejected": -2.003145456314087,
      "logps/chosen": -213.689453125,
      "logps/rejected": -223.7344207763672,
      "loss": 1702.7379,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.14878423511981964,
      "rewards/margins": 0.15320990979671478,
      "rewards/rejected": -0.3019941449165344,
      "step": 4160
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.558818537765861e-06,
      "logits/chosen": -2.3047287464141846,
      "logits/rejected": -2.0756564140319824,
      "logps/chosen": -252.1100616455078,
      "logps/rejected": -233.3250274658203,
      "loss": 1776.3279,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16095681488513947,
      "rewards/margins": 0.1608300507068634,
      "rewards/rejected": -0.3217868506908417,
      "step": 4170
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.555574072897374e-06,
      "logits/chosen": -2.1982951164245605,
      "logits/rejected": -2.209923267364502,
      "logps/chosen": -221.6260528564453,
      "logps/rejected": -228.4404754638672,
      "loss": 2324.7164,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.19664804637432098,
      "rewards/margins": 0.09086047857999802,
      "rewards/rejected": -0.2875085175037384,
      "step": 4180
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.552318885147589e-06,
      "logits/chosen": -2.3210973739624023,
      "logits/rejected": -1.977421760559082,
      "logps/chosen": -255.15255737304688,
      "logps/rejected": -209.5719451904297,
      "loss": 1803.0857,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1423492282629013,
      "rewards/margins": 0.13808949291706085,
      "rewards/rejected": -0.28043872117996216,
      "step": 4190
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.549052991497159e-06,
      "logits/chosen": -2.192139148712158,
      "logits/rejected": -2.164456844329834,
      "logps/chosen": -194.3700714111328,
      "logps/rejected": -207.3929443359375,
      "loss": 1816.1697,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.14202898740768433,
      "rewards/margins": 0.1260402500629425,
      "rewards/rejected": -0.26806920766830444,
      "step": 4200
    },
    {
      "epoch": 0.27,
      "eval_logits/chosen": -2.2034807205200195,
      "eval_logits/rejected": -2.0271365642547607,
      "eval_logps/chosen": -244.57276916503906,
      "eval_logps/rejected": -238.06834411621094,
      "eval_loss": 1847.70458984375,
      "eval_rewards/accuracies": 0.656000018119812,
      "eval_rewards/chosen": -0.1256781369447708,
      "eval_rewards/margins": 0.13888615369796753,
      "eval_rewards/rejected": -0.26456430554389954,
      "eval_runtime": 725.6716,
      "eval_samples_per_second": 2.756,
      "eval_steps_per_second": 1.378,
      "step": 4200
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.545776408982585e-06,
      "logits/chosen": -2.132502794265747,
      "logits/rejected": -2.126312255859375,
      "logps/chosen": -243.8193817138672,
      "logps/rejected": -246.9946746826172,
      "loss": 2001.8141,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12075313180685043,
      "rewards/margins": 0.12188541889190674,
      "rewards/rejected": -0.2426385134458542,
      "step": 4210
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.542489154696128e-06,
      "logits/chosen": -2.3441829681396484,
      "logits/rejected": -1.9969489574432373,
      "logps/chosen": -274.8600158691406,
      "logps/rejected": -226.11526489257812,
      "loss": 1901.0047,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07139638811349869,
      "rewards/margins": 0.12505993247032166,
      "rewards/rejected": -0.19645631313323975,
      "step": 4220
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.5391912457857145e-06,
      "logits/chosen": -2.2296595573425293,
      "logits/rejected": -1.9845161437988281,
      "logps/chosen": -274.591064453125,
      "logps/rejected": -239.41983032226562,
      "loss": 1906.3684,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09621410816907883,
      "rewards/margins": 0.1193847805261612,
      "rewards/rejected": -0.21559889614582062,
      "step": 4230
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.535882699454854e-06,
      "logits/chosen": -2.233431339263916,
      "logits/rejected": -2.115579128265381,
      "logps/chosen": -284.89764404296875,
      "logps/rejected": -325.75762939453125,
      "loss": 1728.6193,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1384657323360443,
      "rewards/margins": 0.191973477602005,
      "rewards/rejected": -0.3304392099380493,
      "step": 4240
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.532563532962546e-06,
      "logits/chosen": -2.297442674636841,
      "logits/rejected": -2.3749196529388428,
      "logps/chosen": -208.76651000976562,
      "logps/rejected": -241.6005859375,
      "loss": 2318.2254,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.188722625374794,
      "rewards/margins": 0.11239129304885864,
      "rewards/rejected": -0.30111393332481384,
      "step": 4250
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.529233763623187e-06,
      "logits/chosen": -2.2471325397491455,
      "logits/rejected": -1.9625120162963867,
      "logps/chosen": -219.39297485351562,
      "logps/rejected": -182.62008666992188,
      "loss": 2061.3088,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17292466759681702,
      "rewards/margins": 0.11186778545379639,
      "rewards/rejected": -0.2847924828529358,
      "step": 4260
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.5258934088064854e-06,
      "logits/chosen": -2.1585116386413574,
      "logits/rejected": -1.7696845531463623,
      "logps/chosen": -240.76730346679688,
      "logps/rejected": -201.3708038330078,
      "loss": 1758.8008,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.20786544680595398,
      "rewards/margins": 0.15300999581813812,
      "rewards/rejected": -0.3608754575252533,
      "step": 4270
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.522542485937369e-06,
      "logits/chosen": -2.262988567352295,
      "logits/rejected": -2.0149128437042236,
      "logps/chosen": -307.56085205078125,
      "logps/rejected": -229.0597381591797,
      "loss": 1402.5003,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.1690836250782013,
      "rewards/margins": 0.18139702081680298,
      "rewards/rejected": -0.3504806458950043,
      "step": 4280
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.519181012495892e-06,
      "logits/chosen": -2.273116111755371,
      "logits/rejected": -2.140737295150757,
      "logps/chosen": -256.5496826171875,
      "logps/rejected": -243.6362762451172,
      "loss": 1785.6244,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14330413937568665,
      "rewards/margins": 0.1616070568561554,
      "rewards/rejected": -0.30491116642951965,
      "step": 4290
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.515809006017147e-06,
      "logits/chosen": -2.191171646118164,
      "logits/rejected": -1.8993133306503296,
      "logps/chosen": -248.03738403320312,
      "logps/rejected": -223.3999481201172,
      "loss": 2080.9717,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.14627774059772491,
      "rewards/margins": 0.1087186336517334,
      "rewards/rejected": -0.2549963891506195,
      "step": 4300
    },
    {
      "epoch": 0.28,
      "eval_logits/chosen": -2.2131552696228027,
      "eval_logits/rejected": -2.0350379943847656,
      "eval_logps/chosen": -246.4134979248047,
      "eval_logps/rejected": -239.87435913085938,
      "eval_loss": 1834.1217041015625,
      "eval_rewards/accuracies": 0.6549999713897705,
      "eval_rewards/chosen": -0.14408540725708008,
      "eval_rewards/margins": 0.1385391652584076,
      "eval_rewards/rejected": -0.28262457251548767,
      "eval_runtime": 726.458,
      "eval_samples_per_second": 2.753,
      "eval_steps_per_second": 1.377,
      "step": 4300
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.512426484091171e-06,
      "logits/chosen": -2.3461813926696777,
      "logits/rejected": -2.020876169204712,
      "logps/chosen": -290.72125244140625,
      "logps/rejected": -267.656005859375,
      "loss": 1668.8262,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13140076398849487,
      "rewards/margins": 0.1322779804468155,
      "rewards/rejected": -0.26367875933647156,
      "step": 4310
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.509033464362858e-06,
      "logits/chosen": -2.053769111633301,
      "logits/rejected": -2.112048864364624,
      "logps/chosen": -252.5277099609375,
      "logps/rejected": -284.105224609375,
      "loss": 1808.401,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11261830478906631,
      "rewards/margins": 0.17346562445163727,
      "rewards/rejected": -0.2860839068889618,
      "step": 4320
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.505629964531857e-06,
      "logits/chosen": -2.32875919342041,
      "logits/rejected": -2.133932590484619,
      "logps/chosen": -235.81735229492188,
      "logps/rejected": -220.7907257080078,
      "loss": 1688.3691,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1261296570301056,
      "rewards/margins": 0.15786603093147278,
      "rewards/rejected": -0.28399568796157837,
      "step": 4330
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.502216002352492e-06,
      "logits/chosen": -2.3252675533294678,
      "logits/rejected": -2.10353422164917,
      "logps/chosen": -176.7300567626953,
      "logps/rejected": -171.5137176513672,
      "loss": 1939.9209,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1368914544582367,
      "rewards/margins": 0.14197859168052673,
      "rewards/rejected": -0.2788700461387634,
      "step": 4340
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.498791595633663e-06,
      "logits/chosen": -2.1548573970794678,
      "logits/rejected": -1.7888492345809937,
      "logps/chosen": -275.2837829589844,
      "logps/rejected": -200.15524291992188,
      "loss": 1920.0037,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12899497151374817,
      "rewards/margins": 0.12071913480758667,
      "rewards/rejected": -0.24971409142017365,
      "step": 4350
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.495356762238751e-06,
      "logits/chosen": -2.402125597000122,
      "logits/rejected": -1.935086965560913,
      "logps/chosen": -294.13031005859375,
      "logps/rejected": -214.6305694580078,
      "loss": 1564.3077,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1093236431479454,
      "rewards/margins": 0.17952266335487366,
      "rewards/rejected": -0.28884631395339966,
      "step": 4360
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.491911520085532e-06,
      "logits/chosen": -1.9718379974365234,
      "logits/rejected": -1.875333547592163,
      "logps/chosen": -212.18783569335938,
      "logps/rejected": -235.5560302734375,
      "loss": 1678.2053,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11780087649822235,
      "rewards/margins": 0.12595582008361816,
      "rewards/rejected": -0.24375669658184052,
      "step": 4370
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.488455887146075e-06,
      "logits/chosen": -2.093561887741089,
      "logits/rejected": -2.0697121620178223,
      "logps/chosen": -184.47760009765625,
      "logps/rejected": -218.77206420898438,
      "loss": 1343.2975,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1436549723148346,
      "rewards/margins": 0.21130797266960144,
      "rewards/rejected": -0.35496294498443604,
      "step": 4380
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.484989881446654e-06,
      "logits/chosen": -2.343677520751953,
      "logits/rejected": -2.1531260013580322,
      "logps/chosen": -215.5881805419922,
      "logps/rejected": -207.2852325439453,
      "loss": 2147.8211,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1311216652393341,
      "rewards/margins": 0.09641782194375992,
      "rewards/rejected": -0.22753946483135223,
      "step": 4390
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.481513521067654e-06,
      "logits/chosen": -2.3038599491119385,
      "logits/rejected": -1.9719699621200562,
      "logps/chosen": -241.0018768310547,
      "logps/rejected": -224.58663940429688,
      "loss": 1718.8639,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.16227325797080994,
      "rewards/margins": 0.1900053471326828,
      "rewards/rejected": -0.35227862000465393,
      "step": 4400
    },
    {
      "epoch": 0.29,
      "eval_logits/chosen": -2.201906681060791,
      "eval_logits/rejected": -2.0237557888031006,
      "eval_logps/chosen": -250.08668518066406,
      "eval_logps/rejected": -245.08804321289062,
      "eval_loss": 1853.157470703125,
      "eval_rewards/accuracies": 0.6579999923706055,
      "eval_rewards/chosen": -0.18081730604171753,
      "eval_rewards/margins": 0.15394414961338043,
      "eval_rewards/rejected": -0.33476147055625916,
      "eval_runtime": 723.7701,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.382,
      "step": 4400
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.478026824143473e-06,
      "logits/chosen": -2.2210612297058105,
      "logits/rejected": -2.091966152191162,
      "logps/chosen": -287.1651306152344,
      "logps/rejected": -248.45028686523438,
      "loss": 1764.9635,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.21103541553020477,
      "rewards/margins": 0.17852582037448883,
      "rewards/rejected": -0.3895612359046936,
      "step": 4410
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.474529808862429e-06,
      "logits/chosen": -2.106393814086914,
      "logits/rejected": -2.0460286140441895,
      "logps/chosen": -207.8838653564453,
      "logps/rejected": -240.6083984375,
      "loss": 1983.9123,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1906355768442154,
      "rewards/margins": 0.15100248157978058,
      "rewards/rejected": -0.3416380286216736,
      "step": 4420
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.471022493466669e-06,
      "logits/chosen": -2.2233099937438965,
      "logits/rejected": -1.9008467197418213,
      "logps/chosen": -322.4415588378906,
      "logps/rejected": -255.49288940429688,
      "loss": 1897.2184,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2050246298313141,
      "rewards/margins": 0.10062649101018906,
      "rewards/rejected": -0.30565109848976135,
      "step": 4430
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.467504896252066e-06,
      "logits/chosen": -2.2593376636505127,
      "logits/rejected": -2.158722400665283,
      "logps/chosen": -271.5922546386719,
      "logps/rejected": -254.9861602783203,
      "loss": 1963.3658,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.21890942752361298,
      "rewards/margins": 0.1213611364364624,
      "rewards/rejected": -0.3402705788612366,
      "step": 4440
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.463977035568132e-06,
      "logits/chosen": -2.1187386512756348,
      "logits/rejected": -2.3420820236206055,
      "logps/chosen": -232.56396484375,
      "logps/rejected": -291.1766052246094,
      "loss": 2271.5484,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.20181193947792053,
      "rewards/margins": 0.0690707340836525,
      "rewards/rejected": -0.27088266611099243,
      "step": 4450
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.460438929817914e-06,
      "logits/chosen": -2.1885595321655273,
      "logits/rejected": -2.0391077995300293,
      "logps/chosen": -223.7105712890625,
      "logps/rejected": -232.64151000976562,
      "loss": 1891.498,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.17290803790092468,
      "rewards/margins": 0.13219548761844635,
      "rewards/rejected": -0.30510348081588745,
      "step": 4460
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.456890597457907e-06,
      "logits/chosen": -2.036794662475586,
      "logits/rejected": -2.0769336223602295,
      "logps/chosen": -233.7548370361328,
      "logps/rejected": -265.60992431640625,
      "loss": 1752.2451,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21619176864624023,
      "rewards/margins": 0.13650615513324738,
      "rewards/rejected": -0.3526979088783264,
      "step": 4470
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.453332056997951e-06,
      "logits/chosen": -2.153390407562256,
      "logits/rejected": -2.204169750213623,
      "logps/chosen": -192.40567016601562,
      "logps/rejected": -205.3812713623047,
      "loss": 1493.2564,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12446187436580658,
      "rewards/margins": 0.1730065643787384,
      "rewards/rejected": -0.2974683940410614,
      "step": 4480
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.449763327001134e-06,
      "logits/chosen": -2.1880500316619873,
      "logits/rejected": -2.103924512863159,
      "logps/chosen": -206.2200164794922,
      "logps/rejected": -247.2500457763672,
      "loss": 1935.248,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1745779663324356,
      "rewards/margins": 0.12614604830741882,
      "rewards/rejected": -0.3007240295410156,
      "step": 4490
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.446184426083702e-06,
      "logits/chosen": -2.176182985305786,
      "logits/rejected": -1.961561918258667,
      "logps/chosen": -213.0888214111328,
      "logps/rejected": -240.0130615234375,
      "loss": 1744.0963,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.19524452090263367,
      "rewards/margins": 0.18556657433509827,
      "rewards/rejected": -0.3808111250400543,
      "step": 4500
    },
    {
      "epoch": 0.29,
      "eval_logits/chosen": -2.2259559631347656,
      "eval_logits/rejected": -2.0475778579711914,
      "eval_logps/chosen": -252.73110961914062,
      "eval_logps/rejected": -246.5689239501953,
      "eval_loss": 1841.0155029296875,
      "eval_rewards/accuracies": 0.6554999947547913,
      "eval_rewards/chosen": -0.2072615772485733,
      "eval_rewards/margins": 0.14230865240097046,
      "eval_rewards/rejected": -0.3495701849460602,
      "eval_runtime": 723.0161,
      "eval_samples_per_second": 2.766,
      "eval_steps_per_second": 1.383,
      "step": 4500
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.442595372914954e-06,
      "logits/chosen": -2.2738208770751953,
      "logits/rejected": -2.0263562202453613,
      "logps/chosen": -253.7425994873047,
      "logps/rejected": -182.83026123046875,
      "loss": 1683.9623,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1676226258277893,
      "rewards/margins": 0.14361922442913055,
      "rewards/rejected": -0.3112418055534363,
      "step": 4510
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.43899618621715e-06,
      "logits/chosen": -2.2293381690979004,
      "logits/rejected": -2.0211691856384277,
      "logps/chosen": -275.49951171875,
      "logps/rejected": -293.45526123046875,
      "loss": 2163.3051,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24445602297782898,
      "rewards/margins": 0.1475904881954193,
      "rewards/rejected": -0.3920465111732483,
      "step": 4520
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.4353868847654105e-06,
      "logits/chosen": -2.3481974601745605,
      "logits/rejected": -2.113680601119995,
      "logps/chosen": -262.9469909667969,
      "logps/rejected": -245.699462890625,
      "loss": 1954.6063,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17088761925697327,
      "rewards/margins": 0.12820035219192505,
      "rewards/rejected": -0.2990879416465759,
      "step": 4530
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.43176748738762e-06,
      "logits/chosen": -2.254246711730957,
      "logits/rejected": -2.0310864448547363,
      "logps/chosen": -250.73941040039062,
      "logps/rejected": -270.41595458984375,
      "loss": 1790.5406,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.18791458010673523,
      "rewards/margins": 0.15288159251213074,
      "rewards/rejected": -0.34079617261886597,
      "step": 4540
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.4281380129643295e-06,
      "logits/chosen": -2.1591944694519043,
      "logits/rejected": -1.999447226524353,
      "logps/chosen": -241.9261016845703,
      "logps/rejected": -247.62173461914062,
      "loss": 1913.1277,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12175941467285156,
      "rewards/margins": 0.16169685125350952,
      "rewards/rejected": -0.2834562659263611,
      "step": 4550
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.424498480428654e-06,
      "logits/chosen": -2.1831679344177246,
      "logits/rejected": -2.078092098236084,
      "logps/chosen": -261.2657775878906,
      "logps/rejected": -229.1588897705078,
      "loss": 2331.508,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.13993816077709198,
      "rewards/margins": 0.0791858583688736,
      "rewards/rejected": -0.21912403404712677,
      "step": 4560
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.420848908766178e-06,
      "logits/chosen": -2.304107189178467,
      "logits/rejected": -2.232893943786621,
      "logps/chosen": -217.4763946533203,
      "logps/rejected": -234.603759765625,
      "loss": 2148.2904,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.10249540954828262,
      "rewards/margins": 0.08247154206037521,
      "rewards/rejected": -0.18496695160865784,
      "step": 4570
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.417189317014855e-06,
      "logits/chosen": -2.1350772380828857,
      "logits/rejected": -2.364804983139038,
      "logps/chosen": -214.90951538085938,
      "logps/rejected": -251.42098999023438,
      "loss": 2562.5131,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.14113643765449524,
      "rewards/margins": 0.058145761489868164,
      "rewards/rejected": -0.1992821991443634,
      "step": 4580
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.41351972426491e-06,
      "logits/chosen": -2.0422589778900146,
      "logits/rejected": -2.074331760406494,
      "logps/chosen": -263.0894470214844,
      "logps/rejected": -328.100830078125,
      "loss": 1863.9609,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1635008156299591,
      "rewards/margins": 0.10958373546600342,
      "rewards/rejected": -0.2730845510959625,
      "step": 4590
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.409840149658735e-06,
      "logits/chosen": -2.1350038051605225,
      "logits/rejected": -1.894366979598999,
      "logps/chosen": -295.7662353515625,
      "logps/rejected": -260.21661376953125,
      "loss": 1741.5717,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11248985677957535,
      "rewards/margins": 0.14602139592170715,
      "rewards/rejected": -0.2585112452507019,
      "step": 4600
    },
    {
      "epoch": 0.3,
      "eval_logits/chosen": -2.2153773307800293,
      "eval_logits/rejected": -2.0375046730041504,
      "eval_logps/chosen": -245.6457061767578,
      "eval_logps/rejected": -236.9356231689453,
      "eval_loss": 1874.8072509765625,
      "eval_rewards/accuracies": 0.6439999938011169,
      "eval_rewards/chosen": -0.13640740513801575,
      "eval_rewards/margins": 0.11682987213134766,
      "eval_rewards/rejected": -0.2532372772693634,
      "eval_runtime": 726.1194,
      "eval_samples_per_second": 2.754,
      "eval_steps_per_second": 1.377,
      "step": 4600
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.4061506123907925e-06,
      "logits/chosen": -2.129925489425659,
      "logits/rejected": -1.9715219736099243,
      "logps/chosen": -276.8015441894531,
      "logps/rejected": -247.16293334960938,
      "loss": 1926.1814,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12709729373455048,
      "rewards/margins": 0.1066756621003151,
      "rewards/rejected": -0.23377294838428497,
      "step": 4610
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.402451131707519e-06,
      "logits/chosen": -2.346142530441284,
      "logits/rejected": -1.9041502475738525,
      "logps/chosen": -224.49453735351562,
      "logps/rejected": -164.30020141601562,
      "loss": 1624.9742,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1502912938594818,
      "rewards/margins": 0.1528705507516861,
      "rewards/rejected": -0.3031618595123291,
      "step": 4620
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.398741726907215e-06,
      "logits/chosen": -2.377230167388916,
      "logits/rejected": -2.0662665367126465,
      "logps/chosen": -294.00048828125,
      "logps/rejected": -265.50164794921875,
      "loss": 1766.6145,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.14743125438690186,
      "rewards/margins": 0.11827798932790756,
      "rewards/rejected": -0.2657092213630676,
      "step": 4630
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.395022417339955e-06,
      "logits/chosen": -2.127654552459717,
      "logits/rejected": -2.1667144298553467,
      "logps/chosen": -225.1376495361328,
      "logps/rejected": -247.256103515625,
      "loss": 2098.4934,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.18512628972530365,
      "rewards/margins": 0.13629302382469177,
      "rewards/rejected": -0.32141929864883423,
      "step": 4640
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.391293222407479e-06,
      "logits/chosen": -2.220669746398926,
      "logits/rejected": -2.22794771194458,
      "logps/chosen": -148.03746032714844,
      "logps/rejected": -177.59378051757812,
      "loss": 1985.2336,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11480303108692169,
      "rewards/margins": 0.1132628470659256,
      "rewards/rejected": -0.22806589305400848,
      "step": 4650
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.387554161563094e-06,
      "logits/chosen": -2.2385716438293457,
      "logits/rejected": -2.1579957008361816,
      "logps/chosen": -218.38253784179688,
      "logps/rejected": -224.18582153320312,
      "loss": 1701.8543,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.19518354535102844,
      "rewards/margins": 0.17622928321361542,
      "rewards/rejected": -0.37141281366348267,
      "step": 4660
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.383805254311575e-06,
      "logits/chosen": -2.4300432205200195,
      "logits/rejected": -2.0662426948547363,
      "logps/chosen": -271.3805236816406,
      "logps/rejected": -238.52798461914062,
      "loss": 1771.1676,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15239445865154266,
      "rewards/margins": 0.1271374523639679,
      "rewards/rejected": -0.27953192591667175,
      "step": 4670
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.380046520209056e-06,
      "logits/chosen": -2.286498546600342,
      "logits/rejected": -1.929011583328247,
      "logps/chosen": -217.7688751220703,
      "logps/rejected": -208.69705200195312,
      "loss": 1793.9574,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.17502537369728088,
      "rewards/margins": 0.14767295122146606,
      "rewards/rejected": -0.32269835472106934,
      "step": 4680
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.376277978862936e-06,
      "logits/chosen": -2.1442456245422363,
      "logits/rejected": -1.869377851486206,
      "logps/chosen": -238.1748809814453,
      "logps/rejected": -209.87319946289062,
      "loss": 1805.5377,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12186243385076523,
      "rewards/margins": 0.1222982183098793,
      "rewards/rejected": -0.24416062235832214,
      "step": 4690
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.372499649931774e-06,
      "logits/chosen": -2.115865707397461,
      "logits/rejected": -2.2537665367126465,
      "logps/chosen": -226.94723510742188,
      "logps/rejected": -257.67767333984375,
      "loss": 1377.8717,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.17855672538280487,
      "rewards/margins": 0.22774052619934082,
      "rewards/rejected": -0.4062972962856293,
      "step": 4700
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": -2.2539665699005127,
      "eval_logits/rejected": -2.073241710662842,
      "eval_logps/chosen": -246.1263427734375,
      "eval_logps/rejected": -241.43572998046875,
      "eval_loss": 1864.4312744140625,
      "eval_rewards/accuracies": 0.6499999761581421,
      "eval_rewards/chosen": -0.14121387898921967,
      "eval_rewards/margins": 0.15702448785305023,
      "eval_rewards/rejected": -0.2982383668422699,
      "eval_runtime": 725.9924,
      "eval_samples_per_second": 2.755,
      "eval_steps_per_second": 1.377,
      "step": 4700
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.368711553125185e-06,
      "logits/chosen": -2.43339204788208,
      "logits/rejected": -2.213193416595459,
      "logps/chosen": -289.5559997558594,
      "logps/rejected": -244.2785186767578,
      "loss": 1851.9451,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12365945428609848,
      "rewards/margins": 0.13165977597236633,
      "rewards/rejected": -0.2553192377090454,
      "step": 4710
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.364913708203734e-06,
      "logits/chosen": -2.3437886238098145,
      "logits/rejected": -2.010408878326416,
      "logps/chosen": -294.05780029296875,
      "logps/rejected": -237.4075927734375,
      "loss": 1524.1236,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0916152149438858,
      "rewards/margins": 0.18767061829566956,
      "rewards/rejected": -0.27928584814071655,
      "step": 4720
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.361106134978844e-06,
      "logits/chosen": -2.2243950366973877,
      "logits/rejected": -2.024705648422241,
      "logps/chosen": -278.34991455078125,
      "logps/rejected": -280.64630126953125,
      "loss": 1785.0814,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07919471710920334,
      "rewards/margins": 0.1524711400270462,
      "rewards/rejected": -0.23166581988334656,
      "step": 4730
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.357288853312681e-06,
      "logits/chosen": -2.296668767929077,
      "logits/rejected": -2.223452568054199,
      "logps/chosen": -292.02313232421875,
      "logps/rejected": -300.81060791015625,
      "loss": 1704.7045,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08545318245887756,
      "rewards/margins": 0.13524378836154938,
      "rewards/rejected": -0.22069695591926575,
      "step": 4740
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353461883118056e-06,
      "logits/chosen": -2.211477279663086,
      "logits/rejected": -2.0765414237976074,
      "logps/chosen": -243.3046875,
      "logps/rejected": -231.37710571289062,
      "loss": 2130.0371,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.140141099691391,
      "rewards/margins": 0.09223539382219315,
      "rewards/rejected": -0.23237650096416473,
      "step": 4750
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.34962524435832e-06,
      "logits/chosen": -2.0958666801452637,
      "logits/rejected": -1.9986375570297241,
      "logps/chosen": -227.4104461669922,
      "logps/rejected": -209.27566528320312,
      "loss": 1820.6793,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07730014622211456,
      "rewards/margins": 0.18698593974113464,
      "rewards/rejected": -0.264286071062088,
      "step": 4760
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.34577895704726e-06,
      "logits/chosen": -2.3277480602264404,
      "logits/rejected": -2.160508155822754,
      "logps/chosen": -270.6835021972656,
      "logps/rejected": -262.9879150390625,
      "loss": 1982.2266,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09069456160068512,
      "rewards/margins": 0.18028482794761658,
      "rewards/rejected": -0.2709794044494629,
      "step": 4770
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3419230412489954e-06,
      "logits/chosen": -2.4157934188842773,
      "logits/rejected": -2.1890101432800293,
      "logps/chosen": -299.4608459472656,
      "logps/rejected": -239.5835418701172,
      "loss": 1680.4857,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10093015432357788,
      "rewards/margins": 0.1618657112121582,
      "rewards/rejected": -0.2627958655357361,
      "step": 4780
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.338057517077872e-06,
      "logits/chosen": -2.3592638969421387,
      "logits/rejected": -1.976302146911621,
      "logps/chosen": -200.95852661132812,
      "logps/rejected": -185.67758178710938,
      "loss": 1029.8841,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06917314231395721,
      "rewards/margins": 0.28071340918540955,
      "rewards/rejected": -0.34988656640052795,
      "step": 4790
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.334182404698356e-06,
      "logits/chosen": -2.3497655391693115,
      "logits/rejected": -1.9321916103363037,
      "logps/chosen": -242.8307647705078,
      "logps/rejected": -180.7891387939453,
      "loss": 1664.0443,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12723731994628906,
      "rewards/margins": 0.16358184814453125,
      "rewards/rejected": -0.2908191680908203,
      "step": 4800
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": -2.2518417835235596,
      "eval_logits/rejected": -2.0734591484069824,
      "eval_logps/chosen": -242.2026824951172,
      "eval_logps/rejected": -236.78143310546875,
      "eval_loss": 1858.1903076171875,
      "eval_rewards/accuracies": 0.6424999833106995,
      "eval_rewards/chosen": -0.10197733342647552,
      "eval_rewards/margins": 0.14971810579299927,
      "eval_rewards/rejected": -0.2516954243183136,
      "eval_runtime": 724.1682,
      "eval_samples_per_second": 2.762,
      "eval_steps_per_second": 1.381,
      "step": 4800
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.330297724324933e-06,
      "logits/chosen": -2.511172294616699,
      "logits/rejected": -1.9903972148895264,
      "logps/chosen": -316.10528564453125,
      "logps/rejected": -226.95077514648438,
      "loss": 1947.774,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07693891227245331,
      "rewards/margins": 0.12045726925134659,
      "rewards/rejected": -0.1973962038755417,
      "step": 4810
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.326403496221999e-06,
      "logits/chosen": -2.19785475730896,
      "logits/rejected": -2.108128547668457,
      "logps/chosen": -170.43618774414062,
      "logps/rejected": -152.4991455078125,
      "loss": 2344.3492,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.09007158875465393,
      "rewards/margins": 0.08842898905277252,
      "rewards/rejected": -0.17850057780742645,
      "step": 4820
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.322499740703755e-06,
      "logits/chosen": -2.1665303707122803,
      "logits/rejected": -2.2699267864227295,
      "logps/chosen": -202.8117218017578,
      "logps/rejected": -240.5999298095703,
      "loss": 1846.5477,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.10712514072656631,
      "rewards/margins": 0.13264909386634827,
      "rewards/rejected": -0.23977422714233398,
      "step": 4830
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.318586478134101e-06,
      "logits/chosen": -2.194655179977417,
      "logits/rejected": -2.1641483306884766,
      "logps/chosen": -201.90403747558594,
      "logps/rejected": -171.35284423828125,
      "loss": 2246.6045,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.09236544370651245,
      "rewards/margins": 0.09969266504049301,
      "rewards/rejected": -0.19205810129642487,
      "step": 4840
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.314663728926534e-06,
      "logits/chosen": -2.432751178741455,
      "logits/rejected": -2.1915640830993652,
      "logps/chosen": -269.34002685546875,
      "logps/rejected": -270.07049560546875,
      "loss": 1822.3117,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1246429905295372,
      "rewards/margins": 0.11308778822422028,
      "rewards/rejected": -0.23773078620433807,
      "step": 4850
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.310731513544033e-06,
      "logits/chosen": -2.244410753250122,
      "logits/rejected": -2.081212282180786,
      "logps/chosen": -255.2774658203125,
      "logps/rejected": -223.5243682861328,
      "loss": 1767.9674,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12419673055410385,
      "rewards/margins": 0.14326511323451996,
      "rewards/rejected": -0.2674618065357208,
      "step": 4860
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.30678985249896e-06,
      "logits/chosen": -2.251373767852783,
      "logits/rejected": -2.190117835998535,
      "logps/chosen": -170.6262664794922,
      "logps/rejected": -206.30154418945312,
      "loss": 1493.1467,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.13476242125034332,
      "rewards/margins": 0.1741175353527069,
      "rewards/rejected": -0.3088799715042114,
      "step": 4870
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.302838766352952e-06,
      "logits/chosen": -2.2527387142181396,
      "logits/rejected": -2.0389404296875,
      "logps/chosen": -271.576904296875,
      "logps/rejected": -252.8636932373047,
      "loss": 1568.2116,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15983930230140686,
      "rewards/margins": 0.17078331112861633,
      "rewards/rejected": -0.3306226134300232,
      "step": 4880
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.298878275716806e-06,
      "logits/chosen": -2.1781983375549316,
      "logits/rejected": -2.1654231548309326,
      "logps/chosen": -212.4600067138672,
      "logps/rejected": -224.9856414794922,
      "loss": 2034.7523,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.20580561459064484,
      "rewards/margins": 0.15322616696357727,
      "rewards/rejected": -0.3590317964553833,
      "step": 4890
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.294908401250386e-06,
      "logits/chosen": -2.4028632640838623,
      "logits/rejected": -1.969160795211792,
      "logps/chosen": -232.218994140625,
      "logps/rejected": -203.0339813232422,
      "loss": 1791.327,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.17475949227809906,
      "rewards/margins": 0.15347324311733246,
      "rewards/rejected": -0.3282327353954315,
      "step": 4900
    },
    {
      "epoch": 0.32,
      "eval_logits/chosen": -2.2827320098876953,
      "eval_logits/rejected": -2.102172374725342,
      "eval_logps/chosen": -249.71177673339844,
      "eval_logps/rejected": -244.05223083496094,
      "eval_loss": 1849.7412109375,
      "eval_rewards/accuracies": 0.6455000042915344,
      "eval_rewards/chosen": -0.1770685464143753,
      "eval_rewards/margins": 0.14733484387397766,
      "eval_rewards/rejected": -0.3244033753871918,
      "eval_runtime": 723.0729,
      "eval_samples_per_second": 2.766,
      "eval_steps_per_second": 1.383,
      "step": 4900
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.290929163662498e-06,
      "logits/chosen": -2.1477413177490234,
      "logits/rejected": -1.9333131313323975,
      "logps/chosen": -285.38873291015625,
      "logps/rejected": -240.5457763671875,
      "loss": 2015.675,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.15653455257415771,
      "rewards/margins": 0.12873533368110657,
      "rewards/rejected": -0.2852698862552643,
      "step": 4910
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.286940583710796e-06,
      "logits/chosen": -2.3153796195983887,
      "logits/rejected": -2.2042593955993652,
      "logps/chosen": -308.6776123046875,
      "logps/rejected": -272.7545471191406,
      "loss": 1321.0888,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.15928933024406433,
      "rewards/margins": 0.20706868171691895,
      "rewards/rejected": -0.3663580119609833,
      "step": 4920
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.282942682201667e-06,
      "logits/chosen": -2.186399221420288,
      "logits/rejected": -1.9242067337036133,
      "logps/chosen": -267.5376281738281,
      "logps/rejected": -242.9633331298828,
      "loss": 1921.7139,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15429851412773132,
      "rewards/margins": 0.15012209117412567,
      "rewards/rejected": -0.3044206202030182,
      "step": 4930
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.278935479990123e-06,
      "logits/chosen": -2.474672555923462,
      "logits/rejected": -2.2394700050354004,
      "logps/chosen": -218.3623046875,
      "logps/rejected": -187.2513427734375,
      "loss": 1960.6902,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.13698963820934296,
      "rewards/margins": 0.11860468238592148,
      "rewards/rejected": -0.25559431314468384,
      "step": 4940
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.274918997979695e-06,
      "logits/chosen": -2.2295145988464355,
      "logits/rejected": -2.2632548809051514,
      "logps/chosen": -211.71249389648438,
      "logps/rejected": -226.74258422851562,
      "loss": 2047.2387,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.18077467381954193,
      "rewards/margins": 0.1329469382762909,
      "rewards/rejected": -0.31372159719467163,
      "step": 4950
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.270893257122319e-06,
      "logits/chosen": -2.149606227874756,
      "logits/rejected": -1.9314075708389282,
      "logps/chosen": -247.7263641357422,
      "logps/rejected": -294.9955749511719,
      "loss": 1735.017,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.19004389643669128,
      "rewards/margins": 0.17759864032268524,
      "rewards/rejected": -0.36764252185821533,
      "step": 4960
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.266858278418232e-06,
      "logits/chosen": -2.164851427078247,
      "logits/rejected": -1.9139404296875,
      "logps/chosen": -253.0793914794922,
      "logps/rejected": -243.8963623046875,
      "loss": 2009.3545,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1647520512342453,
      "rewards/margins": 0.11168110370635986,
      "rewards/rejected": -0.27643316984176636,
      "step": 4970
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.26281408291586e-06,
      "logits/chosen": -2.3956422805786133,
      "logits/rejected": -2.10038685798645,
      "logps/chosen": -259.8249206542969,
      "logps/rejected": -248.13607788085938,
      "loss": 1627.1444,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17487621307373047,
      "rewards/margins": 0.1784350723028183,
      "rewards/rejected": -0.3533112704753876,
      "step": 4980
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.258760691711706e-06,
      "logits/chosen": -2.3154006004333496,
      "logits/rejected": -2.1934704780578613,
      "logps/chosen": -216.1031494140625,
      "logps/rejected": -224.42495727539062,
      "loss": 1871.7553,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.18641582131385803,
      "rewards/margins": 0.12914636731147766,
      "rewards/rejected": -0.3155621886253357,
      "step": 4990
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.254698125950247e-06,
      "logits/chosen": -2.5614352226257324,
      "logits/rejected": -2.3224945068359375,
      "logps/chosen": -312.9407958984375,
      "logps/rejected": -283.60455322265625,
      "loss": 1540.4815,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11926261335611343,
      "rewards/margins": 0.17329421639442444,
      "rewards/rejected": -0.2925568222999573,
      "step": 5000
    },
    {
      "epoch": 0.33,
      "eval_logits/chosen": -2.3258872032165527,
      "eval_logits/rejected": -2.142472743988037,
      "eval_logps/chosen": -249.7621307373047,
      "eval_logps/rejected": -244.18833923339844,
      "eval_loss": 1836.64892578125,
      "eval_rewards/accuracies": 0.6495000123977661,
      "eval_rewards/chosen": -0.17757174372673035,
      "eval_rewards/margins": 0.1481926590204239,
      "eval_rewards/rejected": -0.32576441764831543,
      "eval_runtime": 723.2528,
      "eval_samples_per_second": 2.765,
      "eval_steps_per_second": 1.383,
      "step": 5000
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.250626406823815e-06,
      "logits/chosen": -2.3754916191101074,
      "logits/rejected": -2.1217591762542725,
      "logps/chosen": -237.7047576904297,
      "logps/rejected": -273.30865478515625,
      "loss": 1869.5283,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.20842047035694122,
      "rewards/margins": 0.19425836205482483,
      "rewards/rejected": -0.40267881751060486,
      "step": 5010
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.246545555572489e-06,
      "logits/chosen": -2.3027451038360596,
      "logits/rejected": -2.1745707988739014,
      "logps/chosen": -171.08348083496094,
      "logps/rejected": -212.65609741210938,
      "loss": 1589.6315,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1688733845949173,
      "rewards/margins": 0.16620567440986633,
      "rewards/rejected": -0.33507904410362244,
      "step": 5020
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.242455593483992e-06,
      "logits/chosen": -2.385986566543579,
      "logits/rejected": -2.1910018920898438,
      "logps/chosen": -240.56057739257812,
      "logps/rejected": -203.0875701904297,
      "loss": 1833.0092,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.21406681835651398,
      "rewards/margins": 0.12236443907022476,
      "rewards/rejected": -0.33643123507499695,
      "step": 5030
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.238356541893567e-06,
      "logits/chosen": -2.2863965034484863,
      "logits/rejected": -2.148052453994751,
      "logps/chosen": -211.9119110107422,
      "logps/rejected": -210.85281372070312,
      "loss": 1867.3238,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.21778789162635803,
      "rewards/margins": 0.14483478665351868,
      "rewards/rejected": -0.36262261867523193,
      "step": 5040
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.234248422183876e-06,
      "logits/chosen": -2.125394582748413,
      "logits/rejected": -2.3238110542297363,
      "logps/chosen": -255.83609008789062,
      "logps/rejected": -266.41595458984375,
      "loss": 1845.3883,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12901371717453003,
      "rewards/margins": 0.15136227011680603,
      "rewards/rejected": -0.28037601709365845,
      "step": 5050
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.230131255784884e-06,
      "logits/chosen": -2.5306344032287598,
      "logits/rejected": -2.224951982498169,
      "logps/chosen": -273.32659912109375,
      "logps/rejected": -273.3016052246094,
      "loss": 1727.2801,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.15443465113639832,
      "rewards/margins": 0.1877146065235138,
      "rewards/rejected": -0.3421493172645569,
      "step": 5060
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.226005064173748e-06,
      "logits/chosen": -2.3271052837371826,
      "logits/rejected": -2.1794817447662354,
      "logps/chosen": -278.8447570800781,
      "logps/rejected": -310.39959716796875,
      "loss": 1935.8678,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1420050710439682,
      "rewards/margins": 0.1309777796268463,
      "rewards/rejected": -0.2729828655719757,
      "step": 5070
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2218698688747035e-06,
      "logits/chosen": -2.1697347164154053,
      "logits/rejected": -2.021205425262451,
      "logps/chosen": -258.52752685546875,
      "logps/rejected": -226.1105499267578,
      "loss": 1880.8791,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2581823170185089,
      "rewards/margins": 0.1271318644285202,
      "rewards/rejected": -0.3853141665458679,
      "step": 5080
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.217725691458957e-06,
      "logits/chosen": -2.477919101715088,
      "logits/rejected": -2.3069875240325928,
      "logps/chosen": -205.07302856445312,
      "logps/rejected": -246.52072143554688,
      "loss": 1650.2766,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2084888517856598,
      "rewards/margins": 0.14501745998859406,
      "rewards/rejected": -0.35350629687309265,
      "step": 5090
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.213572553544565e-06,
      "logits/chosen": -2.3776400089263916,
      "logits/rejected": -2.1426074504852295,
      "logps/chosen": -259.0758361816406,
      "logps/rejected": -266.2682800292969,
      "loss": 1773.1312,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.18104009330272675,
      "rewards/margins": 0.14556989073753357,
      "rewards/rejected": -0.3266100287437439,
      "step": 5100
    },
    {
      "epoch": 0.33,
      "eval_logits/chosen": -2.3287124633789062,
      "eval_logits/rejected": -2.1456401348114014,
      "eval_logps/chosen": -246.8673858642578,
      "eval_logps/rejected": -242.05673217773438,
      "eval_loss": 1836.536865234375,
      "eval_rewards/accuracies": 0.652999997138977,
      "eval_rewards/chosen": -0.1486242264509201,
      "eval_rewards/margins": 0.1558239907026291,
      "eval_rewards/rejected": -0.3044482469558716,
      "eval_runtime": 722.0025,
      "eval_samples_per_second": 2.77,
      "eval_steps_per_second": 1.385,
      "step": 5100
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.209410476796331e-06,
      "logits/chosen": -2.253725290298462,
      "logits/rejected": -2.199178695678711,
      "logps/chosen": -186.88389587402344,
      "logps/rejected": -193.34152221679688,
      "loss": 1685.0238,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15792416036128998,
      "rewards/margins": 0.16446875035762787,
      "rewards/rejected": -0.32239291071891785,
      "step": 5110
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.205239482925686e-06,
      "logits/chosen": -2.1166889667510986,
      "logits/rejected": -2.1698646545410156,
      "logps/chosen": -204.41046142578125,
      "logps/rejected": -239.6005401611328,
      "loss": 2057.5961,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16595105826854706,
      "rewards/margins": 0.13063675165176392,
      "rewards/rejected": -0.29658785462379456,
      "step": 5120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.201059593690577e-06,
      "logits/chosen": -2.375741720199585,
      "logits/rejected": -2.28941011428833,
      "logps/chosen": -242.7406005859375,
      "logps/rejected": -229.38473510742188,
      "loss": 1988.8654,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1715283840894699,
      "rewards/margins": 0.12073174864053726,
      "rewards/rejected": -0.29226014018058777,
      "step": 5130
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.196870830895354e-06,
      "logits/chosen": -2.1859264373779297,
      "logits/rejected": -2.2038087844848633,
      "logps/chosen": -274.9366760253906,
      "logps/rejected": -337.29449462890625,
      "loss": 2076.1455,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.17189116775989532,
      "rewards/margins": 0.12850435078144073,
      "rewards/rejected": -0.30039554834365845,
      "step": 5140
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.192673216390657e-06,
      "logits/chosen": -2.3719637393951416,
      "logits/rejected": -2.1126458644866943,
      "logps/chosen": -250.2086181640625,
      "logps/rejected": -223.68789672851562,
      "loss": 2109.3109,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1723567098379135,
      "rewards/margins": 0.11806859821081161,
      "rewards/rejected": -0.29042530059814453,
      "step": 5150
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.188466772073296e-06,
      "logits/chosen": -2.469306230545044,
      "logits/rejected": -2.159566879272461,
      "logps/chosen": -235.36898803710938,
      "logps/rejected": -225.29098510742188,
      "loss": 1837.1287,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.14844447374343872,
      "rewards/margins": 0.1068364828824997,
      "rewards/rejected": -0.2552809417247772,
      "step": 5160
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.184251519886148e-06,
      "logits/chosen": -2.2152698040008545,
      "logits/rejected": -2.312568187713623,
      "logps/chosen": -219.24130249023438,
      "logps/rejected": -255.0888671875,
      "loss": 2291.0273,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.23133759200572968,
      "rewards/margins": 0.0719255581498146,
      "rewards/rejected": -0.3032631576061249,
      "step": 5170
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.180027481818033e-06,
      "logits/chosen": -2.32781720161438,
      "logits/rejected": -2.3139772415161133,
      "logps/chosen": -280.49932861328125,
      "logps/rejected": -258.47967529296875,
      "loss": 1658.382,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.13801197707653046,
      "rewards/margins": 0.15813925862312317,
      "rewards/rejected": -0.29615122079849243,
      "step": 5180
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.175794679903602e-06,
      "logits/chosen": -2.3832285404205322,
      "logits/rejected": -2.149759531021118,
      "logps/chosen": -239.1299591064453,
      "logps/rejected": -183.6186981201172,
      "loss": 1757.3795,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11489125341176987,
      "rewards/margins": 0.16845735907554626,
      "rewards/rejected": -0.28334861993789673,
      "step": 5190
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.171553136223222e-06,
      "logits/chosen": -2.3328781127929688,
      "logits/rejected": -2.3383240699768066,
      "logps/chosen": -283.13861083984375,
      "logps/rejected": -312.5195007324219,
      "loss": 1598.2573,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15817800164222717,
      "rewards/margins": 0.1867099553346634,
      "rewards/rejected": -0.34488797187805176,
      "step": 5200
    },
    {
      "epoch": 0.34,
      "eval_logits/chosen": -2.352234363555908,
      "eval_logits/rejected": -2.1671369075775146,
      "eval_logps/chosen": -246.12818908691406,
      "eval_logps/rejected": -239.1405792236328,
      "eval_loss": 1844.184326171875,
      "eval_rewards/accuracies": 0.6420000195503235,
      "eval_rewards/chosen": -0.1412324607372284,
      "eval_rewards/margins": 0.13405431807041168,
      "eval_rewards/rejected": -0.2752867639064789,
      "eval_runtime": 722.7781,
      "eval_samples_per_second": 2.767,
      "eval_steps_per_second": 1.384,
      "step": 5200
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.167302872902865e-06,
      "logits/chosen": -2.3499057292938232,
      "logits/rejected": -2.2576987743377686,
      "logps/chosen": -263.365234375,
      "logps/rejected": -274.6778869628906,
      "loss": 1414.8517,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11897093057632446,
      "rewards/margins": 0.2325071394443512,
      "rewards/rejected": -0.35147807002067566,
      "step": 5210
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.163043912113985e-06,
      "logits/chosen": -2.392256259918213,
      "logits/rejected": -2.1617372035980225,
      "logps/chosen": -271.14300537109375,
      "logps/rejected": -249.79519653320312,
      "loss": 1866.4203,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12102854251861572,
      "rewards/margins": 0.12646819651126862,
      "rewards/rejected": -0.24749675393104553,
      "step": 5220
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.15877627607341e-06,
      "logits/chosen": -2.1816821098327637,
      "logits/rejected": -2.0997190475463867,
      "logps/chosen": -231.6128387451172,
      "logps/rejected": -217.5960235595703,
      "loss": 1985.5357,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.15018010139465332,
      "rewards/margins": 0.10205508768558502,
      "rewards/rejected": -0.25223520398139954,
      "step": 5230
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.154499987043217e-06,
      "logits/chosen": -2.422106981277466,
      "logits/rejected": -2.2051637172698975,
      "logps/chosen": -237.9840545654297,
      "logps/rejected": -231.46310424804688,
      "loss": 1672.8949,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13780474662780762,
      "rewards/margins": 0.15658816695213318,
      "rewards/rejected": -0.2943929135799408,
      "step": 5240
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.150215067330625e-06,
      "logits/chosen": -2.2486937046051025,
      "logits/rejected": -2.2005836963653564,
      "logps/chosen": -225.8989715576172,
      "logps/rejected": -255.8649444580078,
      "loss": 1956.0352,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.16850169003009796,
      "rewards/margins": 0.1592865288257599,
      "rewards/rejected": -0.32778820395469666,
      "step": 5250
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.145921539287876e-06,
      "logits/chosen": -2.282233238220215,
      "logits/rejected": -2.0033607482910156,
      "logps/chosen": -207.8868408203125,
      "logps/rejected": -204.358154296875,
      "loss": 1690.5404,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.17294219136238098,
      "rewards/margins": 0.1647462546825409,
      "rewards/rejected": -0.3376884162425995,
      "step": 5260
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.141619425312115e-06,
      "logits/chosen": -2.361851692199707,
      "logits/rejected": -1.998276948928833,
      "logps/chosen": -227.4818878173828,
      "logps/rejected": -210.9532470703125,
      "loss": 2334.2115,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18830615282058716,
      "rewards/margins": 0.07913248240947723,
      "rewards/rejected": -0.2674386501312256,
      "step": 5270
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.1373087478452735e-06,
      "logits/chosen": -2.4881348609924316,
      "logits/rejected": -2.080901622772217,
      "logps/chosen": -239.4554901123047,
      "logps/rejected": -209.3617706298828,
      "loss": 1104.9947,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.14214879274368286,
      "rewards/margins": 0.24520206451416016,
      "rewards/rejected": -0.3873508870601654,
      "step": 5280
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.132989529373959e-06,
      "logits/chosen": -2.3881518840789795,
      "logits/rejected": -1.9492473602294922,
      "logps/chosen": -278.64263916015625,
      "logps/rejected": -210.50772094726562,
      "loss": 1829.0125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2272980511188507,
      "rewards/margins": 0.10863574594259262,
      "rewards/rejected": -0.3359338045120239,
      "step": 5290
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.128661792429331e-06,
      "logits/chosen": -2.3927273750305176,
      "logits/rejected": -2.2233188152313232,
      "logps/chosen": -271.4683837890625,
      "logps/rejected": -287.85369873046875,
      "loss": 2181.6221,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.17697831988334656,
      "rewards/margins": 0.11488916724920273,
      "rewards/rejected": -0.2918674945831299,
      "step": 5300
    },
    {
      "epoch": 0.35,
      "eval_logits/chosen": -2.3462629318237305,
      "eval_logits/rejected": -2.1606616973876953,
      "eval_logps/chosen": -248.34812927246094,
      "eval_logps/rejected": -243.24807739257812,
      "eval_loss": 1859.457763671875,
      "eval_rewards/accuracies": 0.6464999914169312,
      "eval_rewards/chosen": -0.16343174874782562,
      "eval_rewards/margins": 0.15293008089065552,
      "eval_rewards/rejected": -0.31636181473731995,
      "eval_runtime": 723.479,
      "eval_samples_per_second": 2.764,
      "eval_steps_per_second": 1.382,
      "step": 5300
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.124325559586985e-06,
      "logits/chosen": -2.101710557937622,
      "logits/rejected": -2.149610996246338,
      "logps/chosen": -205.6929473876953,
      "logps/rejected": -224.6509246826172,
      "loss": 2794.7621,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.1763622760772705,
      "rewards/margins": 0.06034000962972641,
      "rewards/rejected": -0.23670227825641632,
      "step": 5310
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.119980853466835e-06,
      "logits/chosen": -2.3114354610443115,
      "logits/rejected": -1.9271361827850342,
      "logps/chosen": -223.38388061523438,
      "logps/rejected": -211.22024536132812,
      "loss": 1752.8992,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.14229492843151093,
      "rewards/margins": 0.14950500428676605,
      "rewards/rejected": -0.29179996252059937,
      "step": 5320
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.115627696732997e-06,
      "logits/chosen": -2.236950159072876,
      "logits/rejected": -2.043684720993042,
      "logps/chosen": -201.57254028320312,
      "logps/rejected": -193.969482421875,
      "loss": 1977.1037,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.10158295929431915,
      "rewards/margins": 0.11717870086431503,
      "rewards/rejected": -0.21876168251037598,
      "step": 5330
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.111266112093668e-06,
      "logits/chosen": -2.333730459213257,
      "logits/rejected": -2.1332597732543945,
      "logps/chosen": -217.0889129638672,
      "logps/rejected": -250.2196502685547,
      "loss": 1730.2516,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1362122893333435,
      "rewards/margins": 0.1473294198513031,
      "rewards/rejected": -0.283541738986969,
      "step": 5340
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.1068961223010115e-06,
      "logits/chosen": -2.3114235401153564,
      "logits/rejected": -2.0312106609344482,
      "logps/chosen": -285.64080810546875,
      "logps/rejected": -274.3442687988281,
      "loss": 2088.9748,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.14940163493156433,
      "rewards/margins": 0.113797627389431,
      "rewards/rejected": -0.26319923996925354,
      "step": 5350
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.102517750151034e-06,
      "logits/chosen": -2.374783992767334,
      "logits/rejected": -2.134248971939087,
      "logps/chosen": -300.9389343261719,
      "logps/rejected": -243.3031768798828,
      "loss": 1755.3168,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05894102901220322,
      "rewards/margins": 0.13677141070365906,
      "rewards/rejected": -0.19571244716644287,
      "step": 5360
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.09813101848347e-06,
      "logits/chosen": -2.248469829559326,
      "logits/rejected": -2.2975306510925293,
      "logps/chosen": -224.8818359375,
      "logps/rejected": -256.2076416015625,
      "loss": 1819.8633,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.058958910405635834,
      "rewards/margins": 0.14171640574932098,
      "rewards/rejected": -0.20067529380321503,
      "step": 5370
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.093735950181659e-06,
      "logits/chosen": -2.213391065597534,
      "logits/rejected": -2.11991548538208,
      "logps/chosen": -229.80532836914062,
      "logps/rejected": -271.97998046875,
      "loss": 1547.5715,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04331979155540466,
      "rewards/margins": 0.17631354928016663,
      "rewards/rejected": -0.2196333408355713,
      "step": 5380
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.0893325681724326e-06,
      "logits/chosen": -2.3199193477630615,
      "logits/rejected": -2.2667202949523926,
      "logps/chosen": -269.91607666015625,
      "logps/rejected": -274.984375,
      "loss": 1574.0112,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11494427919387817,
      "rewards/margins": 0.17347653210163116,
      "rewards/rejected": -0.28842082619667053,
      "step": 5390
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.084920895425988e-06,
      "logits/chosen": -2.283707618713379,
      "logits/rejected": -2.2867934703826904,
      "logps/chosen": -253.4472198486328,
      "logps/rejected": -281.6402893066406,
      "loss": 1823.2314,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16082549095153809,
      "rewards/margins": 0.14548133313655853,
      "rewards/rejected": -0.3063068091869354,
      "step": 5400
    },
    {
      "epoch": 0.35,
      "eval_logits/chosen": -2.3460588455200195,
      "eval_logits/rejected": -2.160550355911255,
      "eval_logps/chosen": -242.38352966308594,
      "eval_logps/rejected": -237.32022094726562,
      "eval_loss": 1852.075439453125,
      "eval_rewards/accuracies": 0.6539999842643738,
      "eval_rewards/chosen": -0.10378566384315491,
      "eval_rewards/margins": 0.15329769253730774,
      "eval_rewards/rejected": -0.25708335638046265,
      "eval_runtime": 725.1816,
      "eval_samples_per_second": 2.758,
      "eval_steps_per_second": 1.379,
      "step": 5400
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.080500954955769e-06,
      "logits/chosen": -2.2673728466033936,
      "logits/rejected": -1.979201078414917,
      "logps/chosen": -267.6470642089844,
      "logps/rejected": -267.2630615234375,
      "loss": 1781.8809,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11775131523609161,
      "rewards/margins": 0.17902211844921112,
      "rewards/rejected": -0.2967734634876251,
      "step": 5410
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.076072769818354e-06,
      "logits/chosen": -2.5131490230560303,
      "logits/rejected": -2.0682785511016846,
      "logps/chosen": -252.3065643310547,
      "logps/rejected": -207.11624145507812,
      "loss": 1578.0489,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08584243804216385,
      "rewards/margins": 0.16775313019752502,
      "rewards/rejected": -0.2535955309867859,
      "step": 5420
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.071636363113323e-06,
      "logits/chosen": -2.1224284172058105,
      "logits/rejected": -2.046861171722412,
      "logps/chosen": -277.6747131347656,
      "logps/rejected": -232.834716796875,
      "loss": 2311.8592,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09544602036476135,
      "rewards/margins": 0.09202627837657928,
      "rewards/rejected": -0.18747231364250183,
      "step": 5430
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.067191757983146e-06,
      "logits/chosen": -2.0987110137939453,
      "logits/rejected": -2.149458885192871,
      "logps/chosen": -242.80322265625,
      "logps/rejected": -255.27157592773438,
      "loss": 1751.2318,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1322755366563797,
      "rewards/margins": 0.15682359039783478,
      "rewards/rejected": -0.28909915685653687,
      "step": 5440
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.062738977613063e-06,
      "logits/chosen": -2.303696870803833,
      "logits/rejected": -2.177271604537964,
      "logps/chosen": -244.327880859375,
      "logps/rejected": -208.11434936523438,
      "loss": 1977.2764,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09851580113172531,
      "rewards/margins": 0.12179714441299438,
      "rewards/rejected": -0.2203129231929779,
      "step": 5450
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.058278045230957e-06,
      "logits/chosen": -2.2384145259857178,
      "logits/rejected": -2.249328136444092,
      "logps/chosen": -231.9445037841797,
      "logps/rejected": -242.25210571289062,
      "loss": 2202.6955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13577644526958466,
      "rewards/margins": 0.08719965070486069,
      "rewards/rejected": -0.22297611832618713,
      "step": 5460
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.053808984107235e-06,
      "logits/chosen": -2.4486615657806396,
      "logits/rejected": -2.122954845428467,
      "logps/chosen": -243.2438507080078,
      "logps/rejected": -213.0865020751953,
      "loss": 1996.8475,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.10375523567199707,
      "rewards/margins": 0.10135851800441742,
      "rewards/rejected": -0.2051137387752533,
      "step": 5470
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.04933181755471e-06,
      "logits/chosen": -2.4395668506622314,
      "logits/rejected": -2.3647377490997314,
      "logps/chosen": -217.07693481445312,
      "logps/rejected": -220.393798828125,
      "loss": 1927.951,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10496320575475693,
      "rewards/margins": 0.10529372841119766,
      "rewards/rejected": -0.2102569341659546,
      "step": 5480
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.044846568928477e-06,
      "logits/chosen": -2.3208203315734863,
      "logits/rejected": -2.3979101181030273,
      "logps/chosen": -272.79937744140625,
      "logps/rejected": -275.45074462890625,
      "loss": 1987.8191,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11275209486484528,
      "rewards/margins": 0.1150924414396286,
      "rewards/rejected": -0.22784452140331268,
      "step": 5490
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.040353261625788e-06,
      "logits/chosen": -2.493821859359741,
      "logits/rejected": -2.1161201000213623,
      "logps/chosen": -285.68231201171875,
      "logps/rejected": -261.2396545410156,
      "loss": 1672.017,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09137077629566193,
      "rewards/margins": 0.14187830686569214,
      "rewards/rejected": -0.23324909806251526,
      "step": 5500
    },
    {
      "epoch": 0.36,
      "eval_logits/chosen": -2.3531739711761475,
      "eval_logits/rejected": -2.1676862239837646,
      "eval_logps/chosen": -242.32733154296875,
      "eval_logps/rejected": -235.50111389160156,
      "eval_loss": 1837.0867919921875,
      "eval_rewards/accuracies": 0.6499999761581421,
      "eval_rewards/chosen": -0.10322368890047073,
      "eval_rewards/margins": 0.13566839694976807,
      "eval_rewards/rejected": -0.2388920933008194,
      "eval_runtime": 724.875,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.38,
      "step": 5500
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.035851919085936e-06,
      "logits/chosen": -2.330836772918701,
      "logits/rejected": -2.211599826812744,
      "logps/chosen": -276.9023132324219,
      "logps/rejected": -211.7489471435547,
      "loss": 2047.8697,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09508221596479416,
      "rewards/margins": 0.13558414578437805,
      "rewards/rejected": -0.23066632449626923,
      "step": 5510
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.031342564790128e-06,
      "logits/chosen": -2.2656824588775635,
      "logits/rejected": -2.1317713260650635,
      "logps/chosen": -213.4837646484375,
      "logps/rejected": -224.5988311767578,
      "loss": 1587.1312,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07512074708938599,
      "rewards/margins": 0.15887776017189026,
      "rewards/rejected": -0.23399850726127625,
      "step": 5520
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.026825222261367e-06,
      "logits/chosen": -2.299889087677002,
      "logits/rejected": -2.0493857860565186,
      "logps/chosen": -190.53529357910156,
      "logps/rejected": -191.00079345703125,
      "loss": 2531.7961,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.17460234463214874,
      "rewards/margins": 0.07608696818351746,
      "rewards/rejected": -0.2506893277168274,
      "step": 5530
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.022299915064321e-06,
      "logits/chosen": -2.3498470783233643,
      "logits/rejected": -2.1532254219055176,
      "logps/chosen": -312.34197998046875,
      "logps/rejected": -275.0274658203125,
      "loss": 1565.0791,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06494022160768509,
      "rewards/margins": 0.15790972113609314,
      "rewards/rejected": -0.22284996509552002,
      "step": 5540
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.017766666805213e-06,
      "logits/chosen": -2.170281171798706,
      "logits/rejected": -2.132956027984619,
      "logps/chosen": -223.19058227539062,
      "logps/rejected": -199.3626708984375,
      "loss": 2156.3916,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09670194238424301,
      "rewards/margins": 0.11533693969249725,
      "rewards/rejected": -0.21203890442848206,
      "step": 5550
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.013225501131684e-06,
      "logits/chosen": -2.372115135192871,
      "logits/rejected": -2.09326171875,
      "logps/chosen": -230.5290985107422,
      "logps/rejected": -213.18588256835938,
      "loss": 1988.3984,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13151514530181885,
      "rewards/margins": 0.09629587829113007,
      "rewards/rejected": -0.22781100869178772,
      "step": 5560
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.008676441732679e-06,
      "logits/chosen": -2.316739559173584,
      "logits/rejected": -1.9960857629776,
      "logps/chosen": -222.58773803710938,
      "logps/rejected": -181.80226135253906,
      "loss": 2119.3896,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.1629338562488556,
      "rewards/margins": 0.07215423882007599,
      "rewards/rejected": -0.23508810997009277,
      "step": 5570
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.00411951233832e-06,
      "logits/chosen": -2.4594318866729736,
      "logits/rejected": -2.123006582260132,
      "logps/chosen": -234.63711547851562,
      "logps/rejected": -204.8271942138672,
      "loss": 1579.9379,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1491195410490036,
      "rewards/margins": 0.14863654971122742,
      "rewards/rejected": -0.2977561354637146,
      "step": 5580
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.999554736719785e-06,
      "logits/chosen": -2.2024471759796143,
      "logits/rejected": -2.087955951690674,
      "logps/chosen": -303.3402404785156,
      "logps/rejected": -271.7759704589844,
      "loss": 1785.8312,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11363334953784943,
      "rewards/margins": 0.11953709274530411,
      "rewards/rejected": -0.23317043483257294,
      "step": 5590
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.994982138689177e-06,
      "logits/chosen": -2.484562397003174,
      "logits/rejected": -2.2941088676452637,
      "logps/chosen": -245.7768096923828,
      "logps/rejected": -256.5212097167969,
      "loss": 1779.0814,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08376649767160416,
      "rewards/margins": 0.12595562636852264,
      "rewards/rejected": -0.2097221314907074,
      "step": 5600
    },
    {
      "epoch": 0.37,
      "eval_logits/chosen": -2.345733880996704,
      "eval_logits/rejected": -2.1606533527374268,
      "eval_logps/chosen": -242.05909729003906,
      "eval_logps/rejected": -235.68272399902344,
      "eval_loss": 1833.4417724609375,
      "eval_rewards/accuracies": 0.6480000019073486,
      "eval_rewards/chosen": -0.1005413755774498,
      "eval_rewards/margins": 0.14016689360141754,
      "eval_rewards/rejected": -0.24070827662944794,
      "eval_runtime": 723.2346,
      "eval_samples_per_second": 2.765,
      "eval_steps_per_second": 1.383,
      "step": 5600
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.990401742099408e-06,
      "logits/chosen": -2.159879684448242,
      "logits/rejected": -2.1814544200897217,
      "logps/chosen": -187.76846313476562,
      "logps/rejected": -188.0281524658203,
      "loss": 2415.7469,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.08905371278524399,
      "rewards/margins": 0.07608432322740555,
      "rewards/rejected": -0.16513803601264954,
      "step": 5610
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.985813570844072e-06,
      "logits/chosen": -2.278538465499878,
      "logits/rejected": -2.141242027282715,
      "logps/chosen": -302.8232421875,
      "logps/rejected": -292.3064270019531,
      "loss": 1833.266,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0843651071190834,
      "rewards/margins": 0.1703469157218933,
      "rewards/rejected": -0.2547120451927185,
      "step": 5620
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.981217648857316e-06,
      "logits/chosen": -2.3753318786621094,
      "logits/rejected": -2.1800878047943115,
      "logps/chosen": -181.85874938964844,
      "logps/rejected": -199.9864959716797,
      "loss": 1433.0758,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09318314492702484,
      "rewards/margins": 0.16345934569835663,
      "rewards/rejected": -0.25664249062538147,
      "step": 5630
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.97661400011372e-06,
      "logits/chosen": -2.214317560195923,
      "logits/rejected": -2.2591793537139893,
      "logps/chosen": -250.345947265625,
      "logps/rejected": -252.46676635742188,
      "loss": 1889.515,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07318656146526337,
      "rewards/margins": 0.13543672859668732,
      "rewards/rejected": -0.20862329006195068,
      "step": 5640
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.972002648628174e-06,
      "logits/chosen": -2.2506039142608643,
      "logits/rejected": -1.9229838848114014,
      "logps/chosen": -290.96160888671875,
      "logps/rejected": -256.6331481933594,
      "loss": 1863.8377,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13930806517601013,
      "rewards/margins": 0.13363361358642578,
      "rewards/rejected": -0.2729416787624359,
      "step": 5650
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.967383618455743e-06,
      "logits/chosen": -2.403590679168701,
      "logits/rejected": -2.2283291816711426,
      "logps/chosen": -240.9685516357422,
      "logps/rejected": -274.12054443359375,
      "loss": 1954.8783,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13848820328712463,
      "rewards/margins": 0.1496153622865677,
      "rewards/rejected": -0.2881035804748535,
      "step": 5660
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9627569336915515e-06,
      "logits/chosen": -2.521933078765869,
      "logits/rejected": -2.2157063484191895,
      "logps/chosen": -255.2565460205078,
      "logps/rejected": -214.8271942138672,
      "loss": 1625.9281,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07793761789798737,
      "rewards/margins": 0.1624729335308075,
      "rewards/rejected": -0.24041053652763367,
      "step": 5670
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9581226184706555e-06,
      "logits/chosen": -2.3785665035247803,
      "logits/rejected": -2.479404926300049,
      "logps/chosen": -206.0277862548828,
      "logps/rejected": -290.6431884765625,
      "loss": 1778.9562,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11362370103597641,
      "rewards/margins": 0.13917763531208038,
      "rewards/rejected": -0.2528013586997986,
      "step": 5680
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.953480696967912e-06,
      "logits/chosen": -2.0409164428710938,
      "logits/rejected": -2.2784464359283447,
      "logps/chosen": -216.86697387695312,
      "logps/rejected": -266.86175537109375,
      "loss": 1623.332,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07495345920324326,
      "rewards/margins": 0.135491281747818,
      "rewards/rejected": -0.21044473350048065,
      "step": 5690
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.948831193397857e-06,
      "logits/chosen": -2.2667810916900635,
      "logits/rejected": -2.2484869956970215,
      "logps/chosen": -177.64291381835938,
      "logps/rejected": -189.92938232421875,
      "loss": 2283.9326,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.11627526581287384,
      "rewards/margins": 0.06460537016391754,
      "rewards/rejected": -0.1808806210756302,
      "step": 5700
    },
    {
      "epoch": 0.37,
      "eval_logits/chosen": -2.334646701812744,
      "eval_logits/rejected": -2.150752544403076,
      "eval_logps/chosen": -241.03411865234375,
      "eval_logps/rejected": -235.60592651367188,
      "eval_loss": 1836.9827880859375,
      "eval_rewards/accuracies": 0.656000018119812,
      "eval_rewards/chosen": -0.09029180556535721,
      "eval_rewards/margins": 0.14964844286441803,
      "eval_rewards/rejected": -0.23994024097919464,
      "eval_runtime": 723.2472,
      "eval_samples_per_second": 2.765,
      "eval_steps_per_second": 1.383,
      "step": 5700
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.94417413201458e-06,
      "logits/chosen": -2.228362560272217,
      "logits/rejected": -2.0915615558624268,
      "logps/chosen": -219.13333129882812,
      "logps/rejected": -214.9150848388672,
      "loss": 2166.6598,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07561827450990677,
      "rewards/margins": 0.13548563420772552,
      "rewards/rejected": -0.2111039161682129,
      "step": 5710
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9395095371115935e-06,
      "logits/chosen": -2.3947396278381348,
      "logits/rejected": -2.108790874481201,
      "logps/chosen": -220.9462890625,
      "logps/rejected": -224.45059204101562,
      "loss": 1974.1141,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08289176225662231,
      "rewards/margins": 0.1411549299955368,
      "rewards/rejected": -0.22404666244983673,
      "step": 5720
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.93483743302171e-06,
      "logits/chosen": -2.3456485271453857,
      "logits/rejected": -2.112895965576172,
      "logps/chosen": -217.47824096679688,
      "logps/rejected": -206.06838989257812,
      "loss": 1963.5992,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04686587676405907,
      "rewards/margins": 0.12325469404459,
      "rewards/rejected": -0.17012058198451996,
      "step": 5730
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.930157844116913e-06,
      "logits/chosen": -2.19012451171875,
      "logits/rejected": -2.1317851543426514,
      "logps/chosen": -215.0693817138672,
      "logps/rejected": -210.9638671875,
      "loss": 2180.9488,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.053120873868465424,
      "rewards/margins": 0.1061321273446083,
      "rewards/rejected": -0.15925300121307373,
      "step": 5740
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.925470794808229e-06,
      "logits/chosen": -2.343135356903076,
      "logits/rejected": -1.9810895919799805,
      "logps/chosen": -245.66616821289062,
      "logps/rejected": -228.5177764892578,
      "loss": 1596.3757,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06678466498851776,
      "rewards/margins": 0.15668952465057373,
      "rewards/rejected": -0.2234741747379303,
      "step": 5750
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.920776309545606e-06,
      "logits/chosen": -2.4004764556884766,
      "logits/rejected": -2.2372593879699707,
      "logps/chosen": -153.7781982421875,
      "logps/rejected": -159.7616729736328,
      "loss": 1799.1465,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.010711869224905968,
      "rewards/margins": 0.12815974652767181,
      "rewards/rejected": -0.13887162506580353,
      "step": 5760
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.916074412817778e-06,
      "logits/chosen": -2.2932066917419434,
      "logits/rejected": -1.9272019863128662,
      "logps/chosen": -250.5941925048828,
      "logps/rejected": -271.01593017578125,
      "loss": 1309.2438,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07260191440582275,
      "rewards/margins": 0.19899395108222961,
      "rewards/rejected": -0.27159583568573,
      "step": 5770
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.911365129152139e-06,
      "logits/chosen": -2.42976713180542,
      "logits/rejected": -2.275913715362549,
      "logps/chosen": -238.23876953125,
      "logps/rejected": -248.1154327392578,
      "loss": 1842.8461,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05804906040430069,
      "rewards/margins": 0.1541648805141449,
      "rewards/rejected": -0.21221396327018738,
      "step": 5780
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.906648483114623e-06,
      "logits/chosen": -2.3079278469085693,
      "logits/rejected": -2.209843158721924,
      "logps/chosen": -208.1516876220703,
      "logps/rejected": -193.66856384277344,
      "loss": 1672.7014,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10828059911727905,
      "rewards/margins": 0.1810188740491867,
      "rewards/rejected": -0.28929945826530457,
      "step": 5790
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.901924499309564e-06,
      "logits/chosen": -2.258023738861084,
      "logits/rejected": -2.0684800148010254,
      "logps/chosen": -237.38821411132812,
      "logps/rejected": -218.6651611328125,
      "loss": 1948.3748,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08683280646800995,
      "rewards/margins": 0.13882941007614136,
      "rewards/rejected": -0.2256622016429901,
      "step": 5800
    },
    {
      "epoch": 0.38,
      "eval_logits/chosen": -2.346818208694458,
      "eval_logits/rejected": -2.161313056945801,
      "eval_logps/chosen": -240.79034423828125,
      "eval_logps/rejected": -235.4921875,
      "eval_loss": 1821.662353515625,
      "eval_rewards/accuracies": 0.6549999713897705,
      "eval_rewards/chosen": -0.08785369992256165,
      "eval_rewards/margins": 0.15094910562038422,
      "eval_rewards/rejected": -0.23880282044410706,
      "eval_runtime": 724.1961,
      "eval_samples_per_second": 2.762,
      "eval_steps_per_second": 1.381,
      "step": 5800
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.897193202379575e-06,
      "logits/chosen": -2.3440444469451904,
      "logits/rejected": -2.175175905227661,
      "logps/chosen": -207.85714721679688,
      "logps/rejected": -207.29922485351562,
      "loss": 1816.1107,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07379438728094101,
      "rewards/margins": 0.16414874792099,
      "rewards/rejected": -0.2379431426525116,
      "step": 5810
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.8924546170054215e-06,
      "logits/chosen": -2.2868430614471436,
      "logits/rejected": -2.2411949634552,
      "logps/chosen": -223.8891143798828,
      "logps/rejected": -219.9317169189453,
      "loss": 1833.4615,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07981501519680023,
      "rewards/margins": 0.1551598608493805,
      "rewards/rejected": -0.23497489094734192,
      "step": 5820
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.887708767905883e-06,
      "logits/chosen": -2.5371270179748535,
      "logits/rejected": -2.1372833251953125,
      "logps/chosen": -253.84024047851562,
      "logps/rejected": -200.68722534179688,
      "loss": 1781.9459,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10073025524616241,
      "rewards/margins": 0.12689939141273499,
      "rewards/rejected": -0.2276296317577362,
      "step": 5830
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.882955679837636e-06,
      "logits/chosen": -2.2934184074401855,
      "logits/rejected": -2.178283929824829,
      "logps/chosen": -254.0140380859375,
      "logps/rejected": -270.5880126953125,
      "loss": 1916.2676,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.14637038111686707,
      "rewards/margins": 0.1419863998889923,
      "rewards/rejected": -0.2883567810058594,
      "step": 5840
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.878195377595113e-06,
      "logits/chosen": -2.3741116523742676,
      "logits/rejected": -2.1962015628814697,
      "logps/chosen": -241.71084594726562,
      "logps/rejected": -255.45974731445312,
      "loss": 1891.0613,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.05805094167590141,
      "rewards/margins": 0.19312414526939392,
      "rewards/rejected": -0.2511751055717468,
      "step": 5850
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.873427886010384e-06,
      "logits/chosen": -2.318110942840576,
      "logits/rejected": -2.1745448112487793,
      "logps/chosen": -197.3480224609375,
      "logps/rejected": -193.17356872558594,
      "loss": 1736.1059,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06896080821752548,
      "rewards/margins": 0.14914639294147491,
      "rewards/rejected": -0.2181071788072586,
      "step": 5860
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.868653229953021e-06,
      "logits/chosen": -2.3854384422302246,
      "logits/rejected": -2.162374258041382,
      "logps/chosen": -244.8651885986328,
      "logps/rejected": -265.3497619628906,
      "loss": 1352.0979,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07650326192378998,
      "rewards/margins": 0.22548480331897736,
      "rewards/rejected": -0.30198806524276733,
      "step": 5870
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.8638714343299675e-06,
      "logits/chosen": -2.2928366661071777,
      "logits/rejected": -2.210357189178467,
      "logps/chosen": -227.2082977294922,
      "logps/rejected": -253.7676239013672,
      "loss": 1918.273,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08865579217672348,
      "rewards/margins": 0.11443565785884857,
      "rewards/rejected": -0.20309145748615265,
      "step": 5880
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.859082524085414e-06,
      "logits/chosen": -2.3190252780914307,
      "logits/rejected": -1.9635696411132812,
      "logps/chosen": -279.509521484375,
      "logps/rejected": -232.806884765625,
      "loss": 1930.0092,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07166347652673721,
      "rewards/margins": 0.13536502420902252,
      "rewards/rejected": -0.20702850818634033,
      "step": 5890
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.854286524200659e-06,
      "logits/chosen": -2.4598076343536377,
      "logits/rejected": -2.2010176181793213,
      "logps/chosen": -285.62982177734375,
      "logps/rejected": -242.8809051513672,
      "loss": 2247.3281,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.09212378412485123,
      "rewards/margins": 0.07887519896030426,
      "rewards/rejected": -0.1709989756345749,
      "step": 5900
    },
    {
      "epoch": 0.39,
      "eval_logits/chosen": -2.3276898860931396,
      "eval_logits/rejected": -2.144029140472412,
      "eval_logps/chosen": -241.37901306152344,
      "eval_logps/rejected": -234.9075469970703,
      "eval_loss": 1835.3883056640625,
      "eval_rewards/accuracies": 0.6539999842643738,
      "eval_rewards/chosen": -0.09374046325683594,
      "eval_rewards/margins": 0.1392161101102829,
      "eval_rewards/rejected": -0.23295657336711884,
      "eval_runtime": 724.7915,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.38,
      "step": 5900
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.849483459693991e-06,
      "logits/chosen": -2.4043338298797607,
      "logits/rejected": -2.275143623352051,
      "logps/chosen": -219.0234375,
      "logps/rejected": -196.8841094970703,
      "loss": 1608.4016,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09346772730350494,
      "rewards/margins": 0.1652974635362625,
      "rewards/rejected": -0.25876516103744507,
      "step": 5910
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.844673355620544e-06,
      "logits/chosen": -2.291963577270508,
      "logits/rejected": -2.1587347984313965,
      "logps/chosen": -258.6552734375,
      "logps/rejected": -235.81570434570312,
      "loss": 1736.4279,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12082358449697495,
      "rewards/margins": 0.1544593870639801,
      "rewards/rejected": -0.27528297901153564,
      "step": 5920
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.839856237072178e-06,
      "logits/chosen": -2.134535551071167,
      "logits/rejected": -2.1370232105255127,
      "logps/chosen": -195.16397094726562,
      "logps/rejected": -224.6689453125,
      "loss": 1963.8773,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1258985698223114,
      "rewards/margins": 0.12312869727611542,
      "rewards/rejected": -0.249027281999588,
      "step": 5930
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8350321291773455e-06,
      "logits/chosen": -2.064373016357422,
      "logits/rejected": -2.0094499588012695,
      "logps/chosen": -205.4337921142578,
      "logps/rejected": -178.10940551757812,
      "loss": 1742.5545,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02911827340722084,
      "rewards/margins": 0.16871920228004456,
      "rewards/rejected": -0.1978374719619751,
      "step": 5940
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.830201057100953e-06,
      "logits/chosen": -2.3538615703582764,
      "logits/rejected": -2.3942766189575195,
      "logps/chosen": -203.4180908203125,
      "logps/rejected": -234.2006378173828,
      "loss": 1670.6439,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11645101010799408,
      "rewards/margins": 0.15289771556854248,
      "rewards/rejected": -0.26934871077537537,
      "step": 5950
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.82536304604424e-06,
      "logits/chosen": -2.1328577995300293,
      "logits/rejected": -2.0833983421325684,
      "logps/chosen": -245.0559539794922,
      "logps/rejected": -227.7585906982422,
      "loss": 1808.9207,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08891718089580536,
      "rewards/margins": 0.14438779652118683,
      "rewards/rejected": -0.2333049774169922,
      "step": 5960
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8205181212446435e-06,
      "logits/chosen": -2.498424530029297,
      "logits/rejected": -2.272063732147217,
      "logps/chosen": -275.7486267089844,
      "logps/rejected": -250.5533447265625,
      "loss": 1748.2926,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.051897335797548294,
      "rewards/margins": 0.1520821750164032,
      "rewards/rejected": -0.2039795219898224,
      "step": 5970
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.815666307975664e-06,
      "logits/chosen": -2.3104419708251953,
      "logits/rejected": -2.1840157508850098,
      "logps/chosen": -249.99752807617188,
      "logps/rejected": -244.8012237548828,
      "loss": 1731.6719,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1267974078655243,
      "rewards/margins": 0.13985633850097656,
      "rewards/rejected": -0.26665374636650085,
      "step": 5980
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8108076315467346e-06,
      "logits/chosen": -2.3855650424957275,
      "logits/rejected": -2.255192279815674,
      "logps/chosen": -277.58660888671875,
      "logps/rejected": -216.82421875,
      "loss": 1885.3119,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20303857326507568,
      "rewards/margins": 0.13964101672172546,
      "rewards/rejected": -0.34267958998680115,
      "step": 5990
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.805942117303093e-06,
      "logits/chosen": -2.5637707710266113,
      "logits/rejected": -2.2310338020324707,
      "logps/chosen": -333.25775146484375,
      "logps/rejected": -303.8687744140625,
      "loss": 1582.7374,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.17222166061401367,
      "rewards/margins": 0.17134621739387512,
      "rewards/rejected": -0.3435679078102112,
      "step": 6000
    },
    {
      "epoch": 0.39,
      "eval_logits/chosen": -2.3215813636779785,
      "eval_logits/rejected": -2.1373915672302246,
      "eval_logps/chosen": -251.67835998535156,
      "eval_logps/rejected": -245.36856079101562,
      "eval_loss": 1837.457763671875,
      "eval_rewards/accuracies": 0.6545000076293945,
      "eval_rewards/chosen": -0.19673386216163635,
      "eval_rewards/margins": 0.1408328264951706,
      "eval_rewards/rejected": -0.33756670355796814,
      "eval_runtime": 724.3827,
      "eval_samples_per_second": 2.761,
      "eval_steps_per_second": 1.38,
      "step": 6000
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8010697906256446e-06,
      "logits/chosen": -2.1542887687683105,
      "logits/rejected": -2.1475603580474854,
      "logps/chosen": -233.7429656982422,
      "logps/rejected": -216.17855834960938,
      "loss": 2242.7635,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.27046507596969604,
      "rewards/margins": 0.1026582345366478,
      "rewards/rejected": -0.37312331795692444,
      "step": 6010
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7961906769308323e-06,
      "logits/chosen": -2.209507465362549,
      "logits/rejected": -2.02146315574646,
      "logps/chosen": -232.28689575195312,
      "logps/rejected": -254.2529754638672,
      "loss": 1794.5312,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.26051703095436096,
      "rewards/margins": 0.11891166865825653,
      "rewards/rejected": -0.3794287145137787,
      "step": 6020
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7913048016705028e-06,
      "logits/chosen": -2.253190040588379,
      "logits/rejected": -2.2125744819641113,
      "logps/chosen": -283.858642578125,
      "logps/rejected": -293.50457763671875,
      "loss": 2045.3717,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.239190936088562,
      "rewards/margins": 0.11693974584341049,
      "rewards/rejected": -0.3561306893825531,
      "step": 6030
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.786412190331775e-06,
      "logits/chosen": -2.465777635574341,
      "logits/rejected": -2.146012544631958,
      "logps/chosen": -219.91726684570312,
      "logps/rejected": -193.1280975341797,
      "loss": 2000.1277,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.17056769132614136,
      "rewards/margins": 0.10310175269842148,
      "rewards/rejected": -0.27366942167282104,
      "step": 6040
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.781512868436906e-06,
      "logits/chosen": -2.464073896408081,
      "logits/rejected": -2.3181827068328857,
      "logps/chosen": -147.85577392578125,
      "logps/rejected": -168.68588256835938,
      "loss": 1561.3501,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1524503231048584,
      "rewards/margins": 0.13041134178638458,
      "rewards/rejected": -0.2828616499900818,
      "step": 6050
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7766068615431605e-06,
      "logits/chosen": -2.287479877471924,
      "logits/rejected": -2.2375855445861816,
      "logps/chosen": -275.55511474609375,
      "logps/rejected": -237.14199829101562,
      "loss": 2315.9816,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18468059599399567,
      "rewards/margins": 0.10221859067678452,
      "rewards/rejected": -0.2868991792201996,
      "step": 6060
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.771694195242671e-06,
      "logits/chosen": -2.5044009685516357,
      "logits/rejected": -2.073282480239868,
      "logps/chosen": -314.542236328125,
      "logps/rejected": -218.26248168945312,
      "loss": 1707.1232,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15363477170467377,
      "rewards/margins": 0.15451176464557648,
      "rewards/rejected": -0.30814653635025024,
      "step": 6070
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.766774895162314e-06,
      "logits/chosen": -2.3283276557922363,
      "logits/rejected": -2.2265541553497314,
      "logps/chosen": -265.23028564453125,
      "logps/rejected": -218.9558868408203,
      "loss": 2090.2014,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1869053840637207,
      "rewards/margins": 0.09861340373754501,
      "rewards/rejected": -0.2855187654495239,
      "step": 6080
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7618489869635666e-06,
      "logits/chosen": -2.2331745624542236,
      "logits/rejected": -2.183838129043579,
      "logps/chosen": -272.5234069824219,
      "logps/rejected": -266.155517578125,
      "loss": 1883.102,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2549645006656647,
      "rewards/margins": 0.13375045359134674,
      "rewards/rejected": -0.388714998960495,
      "step": 6090
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.756916496342379e-06,
      "logits/chosen": -2.2571969032287598,
      "logits/rejected": -2.3134007453918457,
      "logps/chosen": -212.2631072998047,
      "logps/rejected": -232.08206176757812,
      "loss": 2022.2906,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2539118826389313,
      "rewards/margins": 0.11366178840398788,
      "rewards/rejected": -0.36757364869117737,
      "step": 6100
    },
    {
      "epoch": 0.4,
      "eval_logits/chosen": -2.3228988647460938,
      "eval_logits/rejected": -2.138225793838501,
      "eval_logps/chosen": -255.5701446533203,
      "eval_logps/rejected": -247.9512939453125,
      "eval_loss": 1846.5860595703125,
      "eval_rewards/accuracies": 0.6510000228881836,
      "eval_rewards/chosen": -0.23565174639225006,
      "eval_rewards/margins": 0.1277419775724411,
      "eval_rewards/rejected": -0.3633936941623688,
      "eval_runtime": 722.4831,
      "eval_samples_per_second": 2.768,
      "eval_steps_per_second": 1.384,
      "step": 6100
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.751977449029039e-06,
      "logits/chosen": -1.982560396194458,
      "logits/rejected": -1.9873220920562744,
      "logps/chosen": -285.7325439453125,
      "logps/rejected": -259.8460693359375,
      "loss": 2041.9143,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2565609812736511,
      "rewards/margins": 0.1150914877653122,
      "rewards/rejected": -0.3716525435447693,
      "step": 6110
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.747031870788037e-06,
      "logits/chosen": -2.4981637001037598,
      "logits/rejected": -2.1473708152770996,
      "logps/chosen": -337.43927001953125,
      "logps/rejected": -270.34967041015625,
      "loss": 1604.6782,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2068602293729782,
      "rewards/margins": 0.1532965451478958,
      "rewards/rejected": -0.3601568043231964,
      "step": 6120
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7420797874179326e-06,
      "logits/chosen": -2.273043394088745,
      "logits/rejected": -2.018402338027954,
      "logps/chosen": -259.5672302246094,
      "logps/rejected": -209.3104705810547,
      "loss": 2124.3014,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.22457745671272278,
      "rewards/margins": 0.08632310479879379,
      "rewards/rejected": -0.31090056896209717,
      "step": 6130
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7371212247512167e-06,
      "logits/chosen": -2.600759983062744,
      "logits/rejected": -2.2669732570648193,
      "logps/chosen": -338.55413818359375,
      "logps/rejected": -288.6649475097656,
      "loss": 1795.9818,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12159669399261475,
      "rewards/margins": 0.15219008922576904,
      "rewards/rejected": -0.2737867832183838,
      "step": 6140
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7321562086541817e-06,
      "logits/chosen": -2.376091480255127,
      "logits/rejected": -2.273066997528076,
      "logps/chosen": -271.08477783203125,
      "logps/rejected": -285.23175048828125,
      "loss": 1733.2641,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1810462772846222,
      "rewards/margins": 0.13356760144233704,
      "rewards/rejected": -0.3146139085292816,
      "step": 6150
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7271847650267834e-06,
      "logits/chosen": -2.1715023517608643,
      "logits/rejected": -2.0725510120391846,
      "logps/chosen": -220.02145385742188,
      "logps/rejected": -229.8434295654297,
      "loss": 1948.7779,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2008678913116455,
      "rewards/margins": 0.10712286084890366,
      "rewards/rejected": -0.30799075961112976,
      "step": 6160
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7222069198025086e-06,
      "logits/chosen": -2.1698288917541504,
      "logits/rejected": -1.9773457050323486,
      "logps/chosen": -231.421142578125,
      "logps/rejected": -222.72280883789062,
      "loss": 1984.9541,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.228786900639534,
      "rewards/margins": 0.11893894523382187,
      "rewards/rejected": -0.34772583842277527,
      "step": 6170
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7172226989482353e-06,
      "logits/chosen": -2.1827809810638428,
      "logits/rejected": -1.970580816268921,
      "logps/chosen": -228.5430145263672,
      "logps/rejected": -239.3963165283203,
      "loss": 1866.3375,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.18570196628570557,
      "rewards/margins": 0.1299847662448883,
      "rewards/rejected": -0.31568676233291626,
      "step": 6180
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7122321284641007e-06,
      "logits/chosen": -2.4312362670898438,
      "logits/rejected": -2.0049402713775635,
      "logps/chosen": -386.119140625,
      "logps/rejected": -289.6749572753906,
      "loss": 1797.6766,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2333250492811203,
      "rewards/margins": 0.14045672118663788,
      "rewards/rejected": -0.3737817704677582,
      "step": 6190
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.707235234383365e-06,
      "logits/chosen": -2.3579580783843994,
      "logits/rejected": -2.014785051345825,
      "logps/chosen": -274.70660400390625,
      "logps/rejected": -202.1422882080078,
      "loss": 1575.1918,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1345268189907074,
      "rewards/margins": 0.14214524626731873,
      "rewards/rejected": -0.27667203545570374,
      "step": 6200
    },
    {
      "epoch": 0.41,
      "eval_logits/chosen": -2.317455530166626,
      "eval_logits/rejected": -2.133312940597534,
      "eval_logps/chosen": -246.5572052001953,
      "eval_logps/rejected": -241.9267120361328,
      "eval_loss": 1803.5977783203125,
      "eval_rewards/accuracies": 0.6604999899864197,
      "eval_rewards/chosen": -0.14552246034145355,
      "eval_rewards/margins": 0.1576259285211563,
      "eval_rewards/rejected": -0.30314841866493225,
      "eval_runtime": 726.1251,
      "eval_samples_per_second": 2.754,
      "eval_steps_per_second": 1.377,
      "step": 6200
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.702232042772277e-06,
      "logits/chosen": -2.160836696624756,
      "logits/rejected": -2.10648250579834,
      "logps/chosen": -221.47750854492188,
      "logps/rejected": -217.98605346679688,
      "loss": 1523.7708,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18931357562541962,
      "rewards/margins": 0.1886318176984787,
      "rewards/rejected": -0.3779454231262207,
      "step": 6210
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6972225797299325e-06,
      "logits/chosen": -2.270308017730713,
      "logits/rejected": -2.3070664405822754,
      "logps/chosen": -266.1285095214844,
      "logps/rejected": -267.2669982910156,
      "loss": 1845.6088,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15619874000549316,
      "rewards/margins": 0.17074045538902283,
      "rewards/rejected": -0.326939195394516,
      "step": 6220
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.692206871388147e-06,
      "logits/chosen": -2.3865723609924316,
      "logits/rejected": -1.942652940750122,
      "logps/chosen": -247.4679718017578,
      "logps/rejected": -225.7694091796875,
      "loss": 1537.6834,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15079227089881897,
      "rewards/margins": 0.19530631601810455,
      "rewards/rejected": -0.34609857201576233,
      "step": 6230
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6871849439113115e-06,
      "logits/chosen": -2.097397565841675,
      "logits/rejected": -2.0197715759277344,
      "logps/chosen": -247.7275848388672,
      "logps/rejected": -250.33688354492188,
      "loss": 2091.8342,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17816750705242157,
      "rewards/margins": 0.12157128006219864,
      "rewards/rejected": -0.2997387945652008,
      "step": 6240
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.682156823496259e-06,
      "logits/chosen": -2.333956241607666,
      "logits/rejected": -2.0595738887786865,
      "logps/chosen": -233.3226318359375,
      "logps/rejected": -220.8282012939453,
      "loss": 1475.5063,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1940866857767105,
      "rewards/margins": 0.19871152937412262,
      "rewards/rejected": -0.39279818534851074,
      "step": 6250
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.67712253637213e-06,
      "logits/chosen": -2.3630833625793457,
      "logits/rejected": -2.1345248222351074,
      "logps/chosen": -313.71661376953125,
      "logps/rejected": -241.8723602294922,
      "loss": 1499.583,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2142215520143509,
      "rewards/margins": 0.180682972073555,
      "rewards/rejected": -0.3949044644832611,
      "step": 6260
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.672082108800231e-06,
      "logits/chosen": -2.1950974464416504,
      "logits/rejected": -1.9880892038345337,
      "logps/chosen": -248.4373016357422,
      "logps/rejected": -224.557373046875,
      "loss": 2001.3568,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3462149202823639,
      "rewards/margins": 0.10159517824649811,
      "rewards/rejected": -0.44781002402305603,
      "step": 6270
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6670355670739012e-06,
      "logits/chosen": -2.308696746826172,
      "logits/rejected": -2.1200101375579834,
      "logps/chosen": -181.59396362304688,
      "logps/rejected": -195.4339141845703,
      "loss": 1520.7145,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19640548527240753,
      "rewards/margins": 0.16478656232357025,
      "rewards/rejected": -0.3611920475959778,
      "step": 6280
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6619829375183745e-06,
      "logits/chosen": -2.470808506011963,
      "logits/rejected": -2.2788443565368652,
      "logps/chosen": -244.7198028564453,
      "logps/rejected": -246.4285125732422,
      "loss": 1809.3705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18717266619205475,
      "rewards/margins": 0.17779619991779327,
      "rewards/rejected": -0.36496883630752563,
      "step": 6290
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6569242464906427e-06,
      "logits/chosen": -2.3372724056243896,
      "logits/rejected": -2.159214496612549,
      "logps/chosen": -217.8628692626953,
      "logps/rejected": -258.60821533203125,
      "loss": 1673.5182,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10133813321590424,
      "rewards/margins": 0.1969621181488037,
      "rewards/rejected": -0.29830026626586914,
      "step": 6300
    },
    {
      "epoch": 0.41,
      "eval_logits/chosen": -2.3376824855804443,
      "eval_logits/rejected": -2.151590347290039,
      "eval_logps/chosen": -244.78695678710938,
      "eval_logps/rejected": -240.61180114746094,
      "eval_loss": 1812.673828125,
      "eval_rewards/accuracies": 0.656499981880188,
      "eval_rewards/chosen": -0.1278199702501297,
      "eval_rewards/margins": 0.16217908263206482,
      "eval_rewards/rejected": -0.2899990975856781,
      "eval_runtime": 722.487,
      "eval_samples_per_second": 2.768,
      "eval_steps_per_second": 1.384,
      "step": 6300
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6518595203793156e-06,
      "logits/chosen": -2.178379774093628,
      "logits/rejected": -2.198406934738159,
      "logps/chosen": -273.59149169921875,
      "logps/rejected": -306.0458984375,
      "loss": 1618.5656,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.148772194981575,
      "rewards/margins": 0.18979643285274506,
      "rewards/rejected": -0.33856862783432007,
      "step": 6310
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.646788785604485e-06,
      "logits/chosen": -2.2992780208587646,
      "logits/rejected": -2.1557576656341553,
      "logps/chosen": -221.5495147705078,
      "logps/rejected": -232.4991455078125,
      "loss": 2059.0676,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1589115560054779,
      "rewards/margins": 0.1319461166858673,
      "rewards/rejected": -0.2908576726913452,
      "step": 6320
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.641712068617588e-06,
      "logits/chosen": -2.323676347732544,
      "logits/rejected": -2.2105374336242676,
      "logps/chosen": -276.9544677734375,
      "logps/rejected": -239.6781005859375,
      "loss": 1482.207,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1559438407421112,
      "rewards/margins": 0.16268186271190643,
      "rewards/rejected": -0.31862568855285645,
      "step": 6330
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6366293959012673e-06,
      "logits/chosen": -2.2658047676086426,
      "logits/rejected": -2.0348846912384033,
      "logps/chosen": -194.25845336914062,
      "logps/rejected": -192.47836303710938,
      "loss": 1675.3195,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1305646449327469,
      "rewards/margins": 0.17743127048015594,
      "rewards/rejected": -0.30799591541290283,
      "step": 6340
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.631540793969233e-06,
      "logits/chosen": -2.401700258255005,
      "logits/rejected": -2.3811542987823486,
      "logps/chosen": -197.39588928222656,
      "logps/rejected": -212.9019012451172,
      "loss": 1736.9973,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08778167515993118,
      "rewards/margins": 0.13827702403068542,
      "rewards/rejected": -0.22605867683887482,
      "step": 6350
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.626446289366127e-06,
      "logits/chosen": -2.5857996940612793,
      "logits/rejected": -2.146826982498169,
      "logps/chosen": -235.07467651367188,
      "logps/rejected": -176.82415771484375,
      "loss": 1670.159,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1492549479007721,
      "rewards/margins": 0.1326829344034195,
      "rewards/rejected": -0.281937837600708,
      "step": 6360
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6213459086673786e-06,
      "logits/chosen": -2.3118743896484375,
      "logits/rejected": -2.372246742248535,
      "logps/chosen": -182.10699462890625,
      "logps/rejected": -205.96987915039062,
      "loss": 2017.7496,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11169429123401642,
      "rewards/margins": 0.14489558339118958,
      "rewards/rejected": -0.2565898597240448,
      "step": 6370
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6162396784790737e-06,
      "logits/chosen": -2.2020115852355957,
      "logits/rejected": -2.2304563522338867,
      "logps/chosen": -232.52236938476562,
      "logps/rejected": -243.04067993164062,
      "loss": 2114.6471,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.09515275061130524,
      "rewards/margins": 0.10677912086248398,
      "rewards/rejected": -0.20193186402320862,
      "step": 6380
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6111276254378095e-06,
      "logits/chosen": -2.28242564201355,
      "logits/rejected": -2.2833101749420166,
      "logps/chosen": -234.3068084716797,
      "logps/rejected": -247.7659912109375,
      "loss": 1578.4936,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06349892169237137,
      "rewards/margins": 0.17872732877731323,
      "rewards/rejected": -0.242226243019104,
      "step": 6390
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.606009776210559e-06,
      "logits/chosen": -2.269806146621704,
      "logits/rejected": -2.038888454437256,
      "logps/chosen": -256.7181091308594,
      "logps/rejected": -229.2235870361328,
      "loss": 1820.5289,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1609443873167038,
      "rewards/margins": 0.16164250671863556,
      "rewards/rejected": -0.32258686423301697,
      "step": 6400
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": -2.301616668701172,
      "eval_logits/rejected": -2.1188178062438965,
      "eval_logps/chosen": -245.5502471923828,
      "eval_logps/rejected": -240.75802612304688,
      "eval_loss": 1817.4549560546875,
      "eval_rewards/accuracies": 0.6514999866485596,
      "eval_rewards/chosen": -0.13545304536819458,
      "eval_rewards/margins": 0.15600834786891937,
      "eval_rewards/rejected": -0.29146140813827515,
      "eval_runtime": 723.5291,
      "eval_samples_per_second": 2.764,
      "eval_steps_per_second": 1.382,
      "step": 6400
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.600886157494531e-06,
      "logits/chosen": -2.3986454010009766,
      "logits/rejected": -2.279700517654419,
      "logps/chosen": -276.1898193359375,
      "logps/rejected": -282.24639892578125,
      "loss": 1617.5596,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13333269953727722,
      "rewards/margins": 0.1783868670463562,
      "rewards/rejected": -0.3117195963859558,
      "step": 6410
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5957567960170304e-06,
      "logits/chosen": -2.485504627227783,
      "logits/rejected": -1.7878259420394897,
      "logps/chosen": -303.04351806640625,
      "logps/rejected": -204.10830688476562,
      "loss": 1415.637,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11858425289392471,
      "rewards/margins": 0.1972246915102005,
      "rewards/rejected": -0.3158089518547058,
      "step": 6420
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.590621718535319e-06,
      "logits/chosen": -2.114732503890991,
      "logits/rejected": -1.928137183189392,
      "logps/chosen": -216.5114288330078,
      "logps/rejected": -232.0654754638672,
      "loss": 1787.9346,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1978907436132431,
      "rewards/margins": 0.16446657478809357,
      "rewards/rejected": -0.3623572885990143,
      "step": 6430
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5854809518364775e-06,
      "logits/chosen": -2.3756461143493652,
      "logits/rejected": -2.071209192276001,
      "logps/chosen": -254.34521484375,
      "logps/rejected": -230.24819946289062,
      "loss": 1597.4032,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08198460191488266,
      "rewards/margins": 0.20415684580802917,
      "rewards/rejected": -0.2861414849758148,
      "step": 6440
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.580334522737262e-06,
      "logits/chosen": -2.2984519004821777,
      "logits/rejected": -2.0156452655792236,
      "logps/chosen": -217.0636749267578,
      "logps/rejected": -197.22459411621094,
      "loss": 1980.9646,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.15119247138500214,
      "rewards/margins": 0.13810545206069946,
      "rewards/rejected": -0.2892979383468628,
      "step": 6450
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.575182458083968e-06,
      "logits/chosen": -2.239943265914917,
      "logits/rejected": -2.189892530441284,
      "logps/chosen": -254.2470245361328,
      "logps/rejected": -242.56753540039062,
      "loss": 1728.4674,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13441982865333557,
      "rewards/margins": 0.16325482726097107,
      "rewards/rejected": -0.29767465591430664,
      "step": 6460
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5700247847522883e-06,
      "logits/chosen": -2.3368194103240967,
      "logits/rejected": -2.2533984184265137,
      "logps/chosen": -213.6424102783203,
      "logps/rejected": -231.5264434814453,
      "loss": 1711.3686,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10844433307647705,
      "rewards/margins": 0.17902222275733948,
      "rewards/rejected": -0.28746655583381653,
      "step": 6470
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5648615296471743e-06,
      "logits/chosen": -2.135636806488037,
      "logits/rejected": -2.104698419570923,
      "logps/chosen": -212.5499725341797,
      "logps/rejected": -265.55413818359375,
      "loss": 1671.6654,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15187156200408936,
      "rewards/margins": 0.1493034064769745,
      "rewards/rejected": -0.30117496848106384,
      "step": 6480
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.559692719702693e-06,
      "logits/chosen": -2.1498494148254395,
      "logits/rejected": -1.84506094455719,
      "logps/chosen": -298.0892639160156,
      "logps/rejected": -260.2900695800781,
      "loss": 1792.5463,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11944104731082916,
      "rewards/margins": 0.18930336833000183,
      "rewards/rejected": -0.3087444007396698,
      "step": 6490
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.55451838188189e-06,
      "logits/chosen": -2.268026828765869,
      "logits/rejected": -2.1630935668945312,
      "logps/chosen": -270.82183837890625,
      "logps/rejected": -307.6367492675781,
      "loss": 1858.2232,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09535042196512222,
      "rewards/margins": 0.148817777633667,
      "rewards/rejected": -0.24416819214820862,
      "step": 6500
    },
    {
      "epoch": 0.43,
      "eval_logits/chosen": -2.3055551052093506,
      "eval_logits/rejected": -2.121851921081543,
      "eval_logps/chosen": -246.60879516601562,
      "eval_logps/rejected": -240.28997802734375,
      "eval_loss": 1821.5572509765625,
      "eval_rewards/accuracies": 0.6510000228881836,
      "eval_rewards/chosen": -0.14603851735591888,
      "eval_rewards/margins": 0.14074237644672394,
      "eval_rewards/rejected": -0.2867808938026428,
      "eval_runtime": 725.9926,
      "eval_samples_per_second": 2.755,
      "eval_steps_per_second": 1.377,
      "step": 6500
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.549338543176645e-06,
      "logits/chosen": -2.314718723297119,
      "logits/rejected": -2.0543487071990967,
      "logps/chosen": -318.36981201171875,
      "logps/rejected": -301.492431640625,
      "loss": 1700.1051,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11881273984909058,
      "rewards/margins": 0.15475775301456451,
      "rewards/rejected": -0.2735705077648163,
      "step": 6510
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5441532306075342e-06,
      "logits/chosen": -2.2688286304473877,
      "logits/rejected": -2.2558884620666504,
      "logps/chosen": -247.38119506835938,
      "logps/rejected": -306.28326416015625,
      "loss": 2101.4018,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.15962006151676178,
      "rewards/margins": 0.12336771190166473,
      "rewards/rejected": -0.2829877734184265,
      "step": 6520
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5389624712236894e-06,
      "logits/chosen": -2.2822506427764893,
      "logits/rejected": -2.0567774772644043,
      "logps/chosen": -227.70010375976562,
      "logps/rejected": -209.6324920654297,
      "loss": 2123.4619,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13664281368255615,
      "rewards/margins": 0.08860232681035995,
      "rewards/rejected": -0.22524511814117432,
      "step": 6530
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.533766292102653e-06,
      "logits/chosen": -2.250833034515381,
      "logits/rejected": -2.1126649379730225,
      "logps/chosen": -226.72494506835938,
      "logps/rejected": -228.54183959960938,
      "loss": 1740.1117,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.16880393028259277,
      "rewards/margins": 0.1350560188293457,
      "rewards/rejected": -0.30385997891426086,
      "step": 6540
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5285647203502404e-06,
      "logits/chosen": -2.445377826690674,
      "logits/rejected": -2.3247883319854736,
      "logps/chosen": -269.4084167480469,
      "logps/rejected": -244.4662628173828,
      "loss": 1756.2137,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12137410789728165,
      "rewards/margins": 0.12823879718780518,
      "rewards/rejected": -0.24961289763450623,
      "step": 6550
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5233577831003983e-06,
      "logits/chosen": -2.277635097503662,
      "logits/rejected": -2.130423069000244,
      "logps/chosen": -265.00054931640625,
      "logps/rejected": -251.6302032470703,
      "loss": 1936.7143,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12824609875679016,
      "rewards/margins": 0.1029827818274498,
      "rewards/rejected": -0.23122890293598175,
      "step": 6560
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5181455075150628e-06,
      "logits/chosen": -2.204834461212158,
      "logits/rejected": -1.831272840499878,
      "logps/chosen": -192.53721618652344,
      "logps/rejected": -159.6791534423828,
      "loss": 1779.9844,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10591045767068863,
      "rewards/margins": 0.137648344039917,
      "rewards/rejected": -0.24355880916118622,
      "step": 6570
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.512927920784016e-06,
      "logits/chosen": -2.373384952545166,
      "logits/rejected": -2.2336928844451904,
      "logps/chosen": -234.6550750732422,
      "logps/rejected": -231.44595336914062,
      "loss": 1797.485,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08980982005596161,
      "rewards/margins": 0.1677091419696808,
      "rewards/rejected": -0.2575189769268036,
      "step": 6580
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5077050501247457e-06,
      "logits/chosen": -2.461190700531006,
      "logits/rejected": -2.015806198120117,
      "logps/chosen": -287.93109130859375,
      "logps/rejected": -236.8423309326172,
      "loss": 1750.6951,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06916189938783646,
      "rewards/margins": 0.16299793124198914,
      "rewards/rejected": -0.232159823179245,
      "step": 6590
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5024769227823042e-06,
      "logits/chosen": -2.3659586906433105,
      "logits/rejected": -2.2356834411621094,
      "logps/chosen": -178.3804473876953,
      "logps/rejected": -149.21047973632812,
      "loss": 2088.2963,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15509124100208282,
      "rewards/margins": 0.11627724021673203,
      "rewards/rejected": -0.27136847376823425,
      "step": 6600
    },
    {
      "epoch": 0.43,
      "eval_logits/chosen": -2.349168300628662,
      "eval_logits/rejected": -2.1621055603027344,
      "eval_logps/chosen": -245.64903259277344,
      "eval_logps/rejected": -239.01058959960938,
      "eval_loss": 1821.9971923828125,
      "eval_rewards/accuracies": 0.6474999785423279,
      "eval_rewards/chosen": -0.13644078373908997,
      "eval_rewards/margins": 0.13754619657993317,
      "eval_rewards/rejected": -0.2739869952201843,
      "eval_runtime": 725.2067,
      "eval_samples_per_second": 2.758,
      "eval_steps_per_second": 1.379,
      "step": 6600
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.4972435660291646e-06,
      "logits/chosen": -2.4346280097961426,
      "logits/rejected": -2.29125714302063,
      "logps/chosen": -259.4081726074219,
      "logps/rejected": -248.4931182861328,
      "loss": 1667.1441,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16263893246650696,
      "rewards/margins": 0.14024636149406433,
      "rewards/rejected": -0.3028852939605713,
      "step": 6610
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.492005007165079e-06,
      "logits/chosen": -2.3081727027893066,
      "logits/rejected": -2.0565145015716553,
      "logps/chosen": -242.61135864257812,
      "logps/rejected": -259.98260498046875,
      "loss": 2118.1311,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13074350357055664,
      "rewards/margins": 0.11214659363031387,
      "rewards/rejected": -0.2428901195526123,
      "step": 6620
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.4867612735169377e-06,
      "logits/chosen": -2.452394485473633,
      "logits/rejected": -2.1109442710876465,
      "logps/chosen": -240.4641876220703,
      "logps/rejected": -180.43972778320312,
      "loss": 1365.1011,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15429827570915222,
      "rewards/margins": 0.182037353515625,
      "rewards/rejected": -0.3363357186317444,
      "step": 6630
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.4815123924386226e-06,
      "logits/chosen": -2.607165575027466,
      "logits/rejected": -2.3029696941375732,
      "logps/chosen": -325.56683349609375,
      "logps/rejected": -266.5555725097656,
      "loss": 1970.8203,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.19025450944900513,
      "rewards/margins": 0.10661245882511139,
      "rewards/rejected": -0.2968669533729553,
      "step": 6640
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4762583913108696e-06,
      "logits/chosen": -2.1845288276672363,
      "logits/rejected": -1.9540420770645142,
      "logps/chosen": -288.0469970703125,
      "logps/rejected": -261.729736328125,
      "loss": 1996.6184,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1828862875699997,
      "rewards/margins": 0.1459795981645584,
      "rewards/rejected": -0.3288659155368805,
      "step": 6650
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4709992975411217e-06,
      "logits/chosen": -2.304579019546509,
      "logits/rejected": -1.8208246231079102,
      "logps/chosen": -277.58221435546875,
      "logps/rejected": -243.2197723388672,
      "loss": 1692.5258,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2262284755706787,
      "rewards/margins": 0.17904505133628845,
      "rewards/rejected": -0.4052734971046448,
      "step": 6660
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4657351385633886e-06,
      "logits/chosen": -2.433800220489502,
      "logits/rejected": -2.0775046348571777,
      "logps/chosen": -204.56060791015625,
      "logps/rejected": -209.0178985595703,
      "loss": 1613.1366,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1926989108324051,
      "rewards/margins": 0.1665751039981842,
      "rewards/rejected": -0.3592740297317505,
      "step": 6670
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4604659418381024e-06,
      "logits/chosen": -2.34576153755188,
      "logits/rejected": -1.9419485330581665,
      "logps/chosen": -243.0738983154297,
      "logps/rejected": -228.9387664794922,
      "loss": 1889.6242,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.25019600987434387,
      "rewards/margins": 0.1539548635482788,
      "rewards/rejected": -0.4041508138179779,
      "step": 6680
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4551917348519744e-06,
      "logits/chosen": -2.4607691764831543,
      "logits/rejected": -2.187936544418335,
      "logps/chosen": -297.63330078125,
      "logps/rejected": -264.26025390625,
      "loss": 1638.1448,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1613462269306183,
      "rewards/margins": 0.13827499747276306,
      "rewards/rejected": -0.29962125420570374,
      "step": 6690
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4499125451178505e-06,
      "logits/chosen": -2.046312093734741,
      "logits/rejected": -2.137775421142578,
      "logps/chosen": -225.225341796875,
      "logps/rejected": -260.2182312011719,
      "loss": 1938.0561,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.20934048295021057,
      "rewards/margins": 0.10380176454782486,
      "rewards/rejected": -0.31314224004745483,
      "step": 6700
    },
    {
      "epoch": 0.44,
      "eval_logits/chosen": -2.3692967891693115,
      "eval_logits/rejected": -2.180675506591797,
      "eval_logps/chosen": -251.37413024902344,
      "eval_logps/rejected": -245.4427032470703,
      "eval_loss": 1810.6575927734375,
      "eval_rewards/accuracies": 0.6489999890327454,
      "eval_rewards/chosen": -0.19369174540042877,
      "eval_rewards/margins": 0.14461614191532135,
      "eval_rewards/rejected": -0.33830785751342773,
      "eval_runtime": 723.8586,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.381,
      "step": 6700
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4446284001745723e-06,
      "logits/chosen": -2.188316822052002,
      "logits/rejected": -1.9344806671142578,
      "logps/chosen": -230.5163116455078,
      "logps/rejected": -260.1301574707031,
      "loss": 1848.2039,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.242826908826828,
      "rewards/margins": 0.1286681443452835,
      "rewards/rejected": -0.3714950680732727,
      "step": 6710
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.439339327586827e-06,
      "logits/chosen": -2.333066940307617,
      "logits/rejected": -2.2780447006225586,
      "logps/chosen": -203.406494140625,
      "logps/rejected": -212.65017700195312,
      "loss": 1913.4416,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.13388803601264954,
      "rewards/margins": 0.14607299864292145,
      "rewards/rejected": -0.2799610495567322,
      "step": 6720
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.434045354945008e-06,
      "logits/chosen": -2.424826145172119,
      "logits/rejected": -2.161849021911621,
      "logps/chosen": -299.0447082519531,
      "logps/rejected": -310.2620849609375,
      "loss": 1639.3686,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2502012848854065,
      "rewards/margins": 0.15953245759010315,
      "rewards/rejected": -0.40973377227783203,
      "step": 6730
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4287465098650713e-06,
      "logits/chosen": -2.4182772636413574,
      "logits/rejected": -2.4277045726776123,
      "logps/chosen": -268.342529296875,
      "logps/rejected": -267.02447509765625,
      "loss": 1897.8031,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.20367029309272766,
      "rewards/margins": 0.13523490726947784,
      "rewards/rejected": -0.3389051556587219,
      "step": 6740
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.423442819988387e-06,
      "logits/chosen": -2.2725331783294678,
      "logits/rejected": -2.1603612899780273,
      "logps/chosen": -208.27682495117188,
      "logps/rejected": -214.9034881591797,
      "loss": 2302.1482,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.24889573454856873,
      "rewards/margins": 0.13816052675247192,
      "rewards/rejected": -0.38705623149871826,
      "step": 6750
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4181343129816e-06,
      "logits/chosen": -2.4681544303894043,
      "logits/rejected": -2.1562418937683105,
      "logps/chosen": -190.3780059814453,
      "logps/rejected": -188.13047790527344,
      "loss": 2212.4684,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.14016306400299072,
      "rewards/margins": 0.1204262375831604,
      "rewards/rejected": -0.2605893015861511,
      "step": 6760
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4128210165364837e-06,
      "logits/chosen": -2.191530227661133,
      "logits/rejected": -2.2053451538085938,
      "logps/chosen": -206.8775634765625,
      "logps/rejected": -237.47329711914062,
      "loss": 1608.8537,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15468339622020721,
      "rewards/margins": 0.22072303295135498,
      "rewards/rejected": -0.3754064440727234,
      "step": 6770
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.407502958369795e-06,
      "logits/chosen": -2.3007140159606934,
      "logits/rejected": -2.1566667556762695,
      "logps/chosen": -261.19329833984375,
      "logps/rejected": -246.59744262695312,
      "loss": 1946.2852,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.18745502829551697,
      "rewards/margins": 0.1461646854877472,
      "rewards/rejected": -0.33361974358558655,
      "step": 6780
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4021801662231297e-06,
      "logits/chosen": -2.338116407394409,
      "logits/rejected": -2.188136577606201,
      "logps/chosen": -268.2184143066406,
      "logps/rejected": -258.36224365234375,
      "loss": 1982.3238,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.19341619312763214,
      "rewards/margins": 0.12419227510690689,
      "rewards/rejected": -0.31760844588279724,
      "step": 6790
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.3968526678627793e-06,
      "logits/chosen": -2.2878658771514893,
      "logits/rejected": -1.9563350677490234,
      "logps/chosen": -279.77392578125,
      "logps/rejected": -251.26205444335938,
      "loss": 1672.7229,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.14743436872959137,
      "rewards/margins": 0.15591710805892944,
      "rewards/rejected": -0.303351491689682,
      "step": 6800
    },
    {
      "epoch": 0.44,
      "eval_logits/chosen": -2.351219415664673,
      "eval_logits/rejected": -2.164029359817505,
      "eval_logps/chosen": -250.83648681640625,
      "eval_logps/rejected": -244.66224670410156,
      "eval_loss": 1816.673095703125,
      "eval_rewards/accuracies": 0.652999997138977,
      "eval_rewards/chosen": -0.1883152276277542,
      "eval_rewards/margins": 0.14218826591968536,
      "eval_rewards/rejected": -0.33050352334976196,
      "eval_runtime": 724.154,
      "eval_samples_per_second": 2.762,
      "eval_steps_per_second": 1.381,
      "step": 6800
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.391520491079586e-06,
      "logits/chosen": -2.4422621726989746,
      "logits/rejected": -2.4687561988830566,
      "logps/chosen": -210.08804321289062,
      "logps/rejected": -193.15184020996094,
      "loss": 1662.1758,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1412539780139923,
      "rewards/margins": 0.1359347254037857,
      "rewards/rejected": -0.2771887183189392,
      "step": 6810
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3861836636887936e-06,
      "logits/chosen": -2.3429911136627197,
      "logits/rejected": -2.1664414405822754,
      "logps/chosen": -285.408203125,
      "logps/rejected": -253.633056640625,
      "loss": 1821.6008,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1858934909105301,
      "rewards/margins": 0.1470334827899933,
      "rewards/rejected": -0.3329269587993622,
      "step": 6820
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3808422135299106e-06,
      "logits/chosen": -2.3195672035217285,
      "logits/rejected": -2.4099817276000977,
      "logps/chosen": -303.9296875,
      "logps/rejected": -370.9049987792969,
      "loss": 1800.8039,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1825186312198639,
      "rewards/margins": 0.138997882604599,
      "rewards/rejected": -0.3215165138244629,
      "step": 6830
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.375496168466556e-06,
      "logits/chosen": -2.50669527053833,
      "logits/rejected": -2.0542101860046387,
      "logps/chosen": -230.8943328857422,
      "logps/rejected": -188.94871520996094,
      "loss": 1934.2203,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15956223011016846,
      "rewards/margins": 0.1398100107908249,
      "rewards/rejected": -0.29937228560447693,
      "step": 6840
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3701455563863205e-06,
      "logits/chosen": -2.559258222579956,
      "logits/rejected": -2.107353687286377,
      "logps/chosen": -316.3973083496094,
      "logps/rejected": -295.91229248046875,
      "loss": 1993.9049,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.24522638320922852,
      "rewards/margins": 0.1583399623632431,
      "rewards/rejected": -0.4035663604736328,
      "step": 6850
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3647904052006174e-06,
      "logits/chosen": -2.3529694080352783,
      "logits/rejected": -2.2934610843658447,
      "logps/chosen": -282.9671325683594,
      "logps/rejected": -302.84149169921875,
      "loss": 1705.6371,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.17668868601322174,
      "rewards/margins": 0.16867773234844208,
      "rewards/rejected": -0.3453664183616638,
      "step": 6860
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3594307428445383e-06,
      "logits/chosen": -2.5529115200042725,
      "logits/rejected": -2.1537489891052246,
      "logps/chosen": -345.7641906738281,
      "logps/rejected": -332.9287109375,
      "loss": 1384.0822,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12513437867164612,
      "rewards/margins": 0.1858106553554535,
      "rewards/rejected": -0.3109450340270996,
      "step": 6870
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.354066597276707e-06,
      "logits/chosen": -2.222578287124634,
      "logits/rejected": -2.2239785194396973,
      "logps/chosen": -237.1717987060547,
      "logps/rejected": -285.36175537109375,
      "loss": 2059.5422,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13607537746429443,
      "rewards/margins": 0.12829408049583435,
      "rewards/rejected": -0.2643694281578064,
      "step": 6880
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.348697996479136e-06,
      "logits/chosen": -2.4354445934295654,
      "logits/rejected": -2.1811060905456543,
      "logps/chosen": -244.497314453125,
      "logps/rejected": -204.57057189941406,
      "loss": 1864.0285,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1668837070465088,
      "rewards/margins": 0.13544951379299164,
      "rewards/rejected": -0.30233320593833923,
      "step": 6890
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3433249684570757e-06,
      "logits/chosen": -2.26200270652771,
      "logits/rejected": -2.129823684692383,
      "logps/chosen": -190.80728149414062,
      "logps/rejected": -162.08505249023438,
      "loss": 1800.1967,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12977729737758636,
      "rewards/margins": 0.1699616014957428,
      "rewards/rejected": -0.29973888397216797,
      "step": 6900
    },
    {
      "epoch": 0.45,
      "eval_logits/chosen": -2.350261688232422,
      "eval_logits/rejected": -2.163565158843994,
      "eval_logps/chosen": -246.64015197753906,
      "eval_logps/rejected": -241.98065185546875,
      "eval_loss": 1805.9200439453125,
      "eval_rewards/accuracies": 0.6514999866485596,
      "eval_rewards/chosen": -0.14635208249092102,
      "eval_rewards/margins": 0.15733551979064941,
      "eval_rewards/rejected": -0.3036876320838928,
      "eval_runtime": 723.7743,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.382,
      "step": 6900
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3379475412388724e-06,
      "logits/chosen": -2.432903289794922,
      "logits/rejected": -2.272477626800537,
      "logps/chosen": -254.492919921875,
      "logps/rejected": -242.8998565673828,
      "loss": 1273.1773,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.12537913024425507,
      "rewards/margins": 0.22105541825294495,
      "rewards/rejected": -0.3464345335960388,
      "step": 6910
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3325657428758207e-06,
      "logits/chosen": -2.238603353500366,
      "logits/rejected": -2.215906858444214,
      "logps/chosen": -252.9639434814453,
      "logps/rejected": -273.2824401855469,
      "loss": 1748.925,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09288422763347626,
      "rewards/margins": 0.19491823017597198,
      "rewards/rejected": -0.28780245780944824,
      "step": 6920
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3271796014420175e-06,
      "logits/chosen": -2.3474555015563965,
      "logits/rejected": -2.29400372505188,
      "logps/chosen": -226.4741668701172,
      "logps/rejected": -225.4298095703125,
      "loss": 1809.5197,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.14023862779140472,
      "rewards/margins": 0.16663934290409088,
      "rewards/rejected": -0.3068779408931732,
      "step": 6930
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3217891450342142e-06,
      "logits/chosen": -2.341958522796631,
      "logits/rejected": -2.0191941261291504,
      "logps/chosen": -265.12890625,
      "logps/rejected": -217.28939819335938,
      "loss": 1436.6966,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06530490517616272,
      "rewards/margins": 0.21153318881988525,
      "rewards/rejected": -0.2768380641937256,
      "step": 6940
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3163944017716733e-06,
      "logits/chosen": -2.5168886184692383,
      "logits/rejected": -2.2140870094299316,
      "logps/chosen": -224.3677520751953,
      "logps/rejected": -201.74960327148438,
      "loss": 2093.9619,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07996897399425507,
      "rewards/margins": 0.10479603707790375,
      "rewards/rejected": -0.1847650110721588,
      "step": 6950
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.310995399796017e-06,
      "logits/chosen": -2.439115524291992,
      "logits/rejected": -2.318527936935425,
      "logps/chosen": -282.1032409667969,
      "logps/rejected": -285.8451232910156,
      "loss": 2300.6125,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08117020130157471,
      "rewards/margins": 0.10543999820947647,
      "rewards/rejected": -0.18661019206047058,
      "step": 6960
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.305592167271085e-06,
      "logits/chosen": -2.317767858505249,
      "logits/rejected": -2.2276875972747803,
      "logps/chosen": -202.17050170898438,
      "logps/rejected": -208.3937225341797,
      "loss": 1673.9172,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04924345016479492,
      "rewards/margins": 0.1713528037071228,
      "rewards/rejected": -0.22059623897075653,
      "step": 6970
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.3001847323827846e-06,
      "logits/chosen": -2.304652690887451,
      "logits/rejected": -2.251988649368286,
      "logps/chosen": -272.10894775390625,
      "logps/rejected": -285.6690368652344,
      "loss": 1587.1257,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04011273384094238,
      "rewards/margins": 0.16978901624679565,
      "rewards/rejected": -0.20990173518657684,
      "step": 6980
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2947731233389447e-06,
      "logits/chosen": -2.3865530490875244,
      "logits/rejected": -2.018869161605835,
      "logps/chosen": -253.6022491455078,
      "logps/rejected": -214.45663452148438,
      "loss": 1568.9052,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.060955680906772614,
      "rewards/margins": 0.16281774640083313,
      "rewards/rejected": -0.22377343475818634,
      "step": 6990
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2893573683691706e-06,
      "logits/chosen": -2.2005510330200195,
      "logits/rejected": -2.2169203758239746,
      "logps/chosen": -209.6483154296875,
      "logps/rejected": -207.0091094970703,
      "loss": 1767.4076,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04954850673675537,
      "rewards/margins": 0.15256556868553162,
      "rewards/rejected": -0.20211410522460938,
      "step": 7000
    },
    {
      "epoch": 0.46,
      "eval_logits/chosen": -2.329406261444092,
      "eval_logits/rejected": -2.144054651260376,
      "eval_logps/chosen": -238.76791381835938,
      "eval_logps/rejected": -232.6849365234375,
      "eval_loss": 1820.3804931640625,
      "eval_rewards/accuracies": 0.6464999914169312,
      "eval_rewards/chosen": -0.06762954592704773,
      "eval_rewards/margins": 0.14310090243816376,
      "eval_rewards/rejected": -0.21073046326637268,
      "eval_runtime": 723.3553,
      "eval_samples_per_second": 2.765,
      "eval_steps_per_second": 1.382,
      "step": 7000
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2839374957246915e-06,
      "logits/chosen": -2.412297010421753,
      "logits/rejected": -2.1327645778656006,
      "logps/chosen": -276.3094482421875,
      "logps/rejected": -195.72564697265625,
      "loss": 2028.4072,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10219154506921768,
      "rewards/margins": 0.10954372584819794,
      "rewards/rejected": -0.2117352932691574,
      "step": 7010
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2785135336782187e-06,
      "logits/chosen": -2.2545413970947266,
      "logits/rejected": -2.0809073448181152,
      "logps/chosen": -245.10043334960938,
      "logps/rejected": -285.3205261230469,
      "loss": 1585.4265,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11185015738010406,
      "rewards/margins": 0.17907771468162537,
      "rewards/rejected": -0.2909278869628906,
      "step": 7020
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2730855105237952e-06,
      "logits/chosen": -2.421980619430542,
      "logits/rejected": -2.2304441928863525,
      "logps/chosen": -226.10519409179688,
      "logps/rejected": -288.8428039550781,
      "loss": 1519.8598,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.058287106454372406,
      "rewards/margins": 0.18557073175907135,
      "rewards/rejected": -0.24385783076286316,
      "step": 7030
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2676534545766486e-06,
      "logits/chosen": -2.2319369316101074,
      "logits/rejected": -2.206944465637207,
      "logps/chosen": -218.6344451904297,
      "logps/rejected": -220.7762908935547,
      "loss": 1949.5209,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06164904311299324,
      "rewards/margins": 0.12034694850444794,
      "rewards/rejected": -0.18199597299098969,
      "step": 7040
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.262217394173043e-06,
      "logits/chosen": -2.341109037399292,
      "logits/rejected": -2.059361457824707,
      "logps/chosen": -252.3672637939453,
      "logps/rejected": -258.5021057128906,
      "loss": 1971.7896,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11516623198986053,
      "rewards/margins": 0.12217897176742554,
      "rewards/rejected": -0.23734521865844727,
      "step": 7050
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2567773576701333e-06,
      "logits/chosen": -2.163533926010132,
      "logits/rejected": -1.9948575496673584,
      "logps/chosen": -260.88348388671875,
      "logps/rejected": -250.1443634033203,
      "loss": 1553.0061,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05126018449664116,
      "rewards/margins": 0.1957646906375885,
      "rewards/rejected": -0.24702484905719757,
      "step": 7060
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2513333734458154e-06,
      "logits/chosen": -2.377007246017456,
      "logits/rejected": -2.2850708961486816,
      "logps/chosen": -216.84231567382812,
      "logps/rejected": -211.0592803955078,
      "loss": 1930.9227,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.10001027584075928,
      "rewards/margins": 0.10824619233608246,
      "rewards/rejected": -0.20825648307800293,
      "step": 7070
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.245885469898576e-06,
      "logits/chosen": -2.271165609359741,
      "logits/rejected": -2.0621328353881836,
      "logps/chosen": -307.70196533203125,
      "logps/rejected": -260.340087890625,
      "loss": 1894.34,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0714680626988411,
      "rewards/margins": 0.16326221823692322,
      "rewards/rejected": -0.2347303181886673,
      "step": 7080
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2404336754473497e-06,
      "logits/chosen": -2.2628307342529297,
      "logits/rejected": -2.0071568489074707,
      "logps/chosen": -269.6414794921875,
      "logps/rejected": -222.27792358398438,
      "loss": 1584.2591,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.021345099434256554,
      "rewards/margins": 0.17406703531742096,
      "rewards/rejected": -0.19541212916374207,
      "step": 7090
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.234978018531367e-06,
      "logits/chosen": -2.57427716255188,
      "logits/rejected": -2.1523544788360596,
      "logps/chosen": -262.66851806640625,
      "logps/rejected": -215.177490234375,
      "loss": 1597.6004,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.041683003306388855,
      "rewards/margins": 0.17308592796325684,
      "rewards/rejected": -0.2147689312696457,
      "step": 7100
    },
    {
      "epoch": 0.46,
      "eval_logits/chosen": -2.309113025665283,
      "eval_logits/rejected": -2.1259310245513916,
      "eval_logps/chosen": -240.7786407470703,
      "eval_logps/rejected": -235.21725463867188,
      "eval_loss": 1808.856201171875,
      "eval_rewards/accuracies": 0.6499999761581421,
      "eval_rewards/chosen": -0.08773694932460785,
      "eval_rewards/margins": 0.14831653237342834,
      "eval_rewards/rejected": -0.2360534816980362,
      "eval_runtime": 724.9942,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.379,
      "step": 7100
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.229518527610006e-06,
      "logits/chosen": -2.396723747253418,
      "logits/rejected": -2.1092593669891357,
      "logps/chosen": -302.719482421875,
      "logps/rejected": -269.29229736328125,
      "loss": 1988.2623,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09929381310939789,
      "rewards/margins": 0.11748860031366348,
      "rewards/rejected": -0.21678242087364197,
      "step": 7110
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2240552311626465e-06,
      "logits/chosen": -2.3822884559631348,
      "logits/rejected": -2.1514086723327637,
      "logps/chosen": -248.64242553710938,
      "logps/rejected": -245.8981475830078,
      "loss": 2073.4006,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.07428698241710663,
      "rewards/margins": 0.12200691550970078,
      "rewards/rejected": -0.196293905377388,
      "step": 7120
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2185881576885193e-06,
      "logits/chosen": -2.4048800468444824,
      "logits/rejected": -2.0579912662506104,
      "logps/chosen": -225.2511749267578,
      "logps/rejected": -197.3291473388672,
      "loss": 1877.8533,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1352832019329071,
      "rewards/margins": 0.10745497047901154,
      "rewards/rejected": -0.24273820221424103,
      "step": 7130
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.213117335706557e-06,
      "logits/chosen": -2.283663511276245,
      "logits/rejected": -2.39902925491333,
      "logps/chosen": -267.2200012207031,
      "logps/rejected": -287.21697998046875,
      "loss": 2213.0275,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11175023019313812,
      "rewards/margins": 0.10790754854679108,
      "rewards/rejected": -0.2196577787399292,
      "step": 7140
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2076427937552473e-06,
      "logits/chosen": -2.3167736530303955,
      "logits/rejected": -2.0965049266815186,
      "logps/chosen": -254.1221160888672,
      "logps/rejected": -253.3069610595703,
      "loss": 1572.583,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0863238126039505,
      "rewards/margins": 0.18308484554290771,
      "rewards/rejected": -0.2694086730480194,
      "step": 7150
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2021645603924827e-06,
      "logits/chosen": -2.1734702587127686,
      "logits/rejected": -2.078382968902588,
      "logps/chosen": -147.8426513671875,
      "logps/rejected": -173.65969848632812,
      "loss": 1859.6615,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0938853994011879,
      "rewards/margins": 0.16452589631080627,
      "rewards/rejected": -0.2584112882614136,
      "step": 7160
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.196682664195412e-06,
      "logits/chosen": -2.296813488006592,
      "logits/rejected": -2.0265824794769287,
      "logps/chosen": -212.2986297607422,
      "logps/rejected": -186.99539184570312,
      "loss": 2164.1652,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08250459283590317,
      "rewards/margins": 0.09587766975164413,
      "rewards/rejected": -0.1783822774887085,
      "step": 7170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.191197133760291e-06,
      "logits/chosen": -2.542553424835205,
      "logits/rejected": -2.2270288467407227,
      "logps/chosen": -267.11199951171875,
      "logps/rejected": -215.85452270507812,
      "loss": 1465.4353,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.0029144559521228075,
      "rewards/margins": 0.22991275787353516,
      "rewards/rejected": -0.22699828445911407,
      "step": 7180
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.185707997702334e-06,
      "logits/chosen": -2.202695369720459,
      "logits/rejected": -2.0384864807128906,
      "logps/chosen": -249.30551147460938,
      "logps/rejected": -224.59207153320312,
      "loss": 2006.3016,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08540628850460052,
      "rewards/margins": 0.13138136267662048,
      "rewards/rejected": -0.216787651181221,
      "step": 7190
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1802152846555624e-06,
      "logits/chosen": -2.2406742572784424,
      "logits/rejected": -2.2176101207733154,
      "logps/chosen": -231.08535766601562,
      "logps/rejected": -232.2159881591797,
      "loss": 1694.7447,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0684235543012619,
      "rewards/margins": 0.16649962961673737,
      "rewards/rejected": -0.23492319881916046,
      "step": 7200
    },
    {
      "epoch": 0.47,
      "eval_logits/chosen": -2.3213398456573486,
      "eval_logits/rejected": -2.136472225189209,
      "eval_logps/chosen": -238.93997192382812,
      "eval_logps/rejected": -232.4331512451172,
      "eval_loss": 1807.7432861328125,
      "eval_rewards/accuracies": 0.6485000252723694,
      "eval_rewards/chosen": -0.06935019791126251,
      "eval_rewards/margins": 0.13886241614818573,
      "eval_rewards/rejected": -0.20821261405944824,
      "eval_runtime": 722.9812,
      "eval_samples_per_second": 2.766,
      "eval_steps_per_second": 1.383,
      "step": 7200
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.174719023272659e-06,
      "logits/chosen": -2.395876169204712,
      "logits/rejected": -2.441063404083252,
      "logps/chosen": -220.5709686279297,
      "logps/rejected": -281.8936462402344,
      "loss": 1723.2258,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06876830756664276,
      "rewards/margins": 0.1497349888086319,
      "rewards/rejected": -0.21850331127643585,
      "step": 7210
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.169219242224816e-06,
      "logits/chosen": -2.366623878479004,
      "logits/rejected": -2.1990063190460205,
      "logps/chosen": -251.0176544189453,
      "logps/rejected": -256.62091064453125,
      "loss": 1895.4932,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11090411245822906,
      "rewards/margins": 0.11698516458272934,
      "rewards/rejected": -0.2278892993927002,
      "step": 7220
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1637159702015837e-06,
      "logits/chosen": -2.3689377307891846,
      "logits/rejected": -2.057278871536255,
      "logps/chosen": -212.3173370361328,
      "logps/rejected": -213.333984375,
      "loss": 1277.8221,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06425191462039948,
      "rewards/margins": 0.20339460670948029,
      "rewards/rejected": -0.26764652132987976,
      "step": 7230
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1582092359107263e-06,
      "logits/chosen": -2.417332410812378,
      "logits/rejected": -2.196626901626587,
      "logps/chosen": -285.0397033691406,
      "logps/rejected": -255.1283416748047,
      "loss": 1720.6148,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06761996448040009,
      "rewards/margins": 0.1591935008764267,
      "rewards/rejected": -0.22681346535682678,
      "step": 7240
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.152699068078067e-06,
      "logits/chosen": -2.2219226360321045,
      "logits/rejected": -2.021028518676758,
      "logps/chosen": -291.9150085449219,
      "logps/rejected": -286.8653564453125,
      "loss": 1690.582,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.11624713987112045,
      "rewards/margins": 0.15491464734077454,
      "rewards/rejected": -0.2711617350578308,
      "step": 7250
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1471854954473415e-06,
      "logits/chosen": -2.366694688796997,
      "logits/rejected": -2.4141058921813965,
      "logps/chosen": -253.8938446044922,
      "logps/rejected": -267.23431396484375,
      "loss": 1545.9211,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03127194195985794,
      "rewards/margins": 0.17108400166034698,
      "rewards/rejected": -0.20235593616962433,
      "step": 7260
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1416685467800436e-06,
      "logits/chosen": -2.163619041442871,
      "logits/rejected": -2.132086992263794,
      "logps/chosen": -197.5450897216797,
      "logps/rejected": -190.2638702392578,
      "loss": 1734.2273,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11962287127971649,
      "rewards/margins": 0.15248456597328186,
      "rewards/rejected": -0.27210742235183716,
      "step": 7270
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1361482508552803e-06,
      "logits/chosen": -2.3619093894958496,
      "logits/rejected": -1.863598108291626,
      "logps/chosen": -255.34628295898438,
      "logps/rejected": -225.1408233642578,
      "loss": 1790.4477,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13395367562770844,
      "rewards/margins": 0.1318083107471466,
      "rewards/rejected": -0.26576200127601624,
      "step": 7280
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1306246364696198e-06,
      "logits/chosen": -2.48795747756958,
      "logits/rejected": -2.2847416400909424,
      "logps/chosen": -265.2281799316406,
      "logps/rejected": -259.6405029296875,
      "loss": 1982.2109,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12753859162330627,
      "rewards/margins": 0.11917465925216675,
      "rewards/rejected": -0.24671323597431183,
      "step": 7290
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1250977324369413e-06,
      "logits/chosen": -2.272779941558838,
      "logits/rejected": -2.19761323928833,
      "logps/chosen": -170.6405029296875,
      "logps/rejected": -193.0902557373047,
      "loss": 1729.9865,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11688047647476196,
      "rewards/margins": 0.15737764537334442,
      "rewards/rejected": -0.2742581069469452,
      "step": 7300
    },
    {
      "epoch": 0.48,
      "eval_logits/chosen": -2.3204145431518555,
      "eval_logits/rejected": -2.1347432136535645,
      "eval_logps/chosen": -244.92276000976562,
      "eval_logps/rejected": -239.7329864501953,
      "eval_loss": 1804.16455078125,
      "eval_rewards/accuracies": 0.6575000286102295,
      "eval_rewards/chosen": -0.12917816638946533,
      "eval_rewards/margins": 0.1520327478647232,
      "eval_rewards/rejected": -0.28121089935302734,
      "eval_runtime": 724.3424,
      "eval_samples_per_second": 2.761,
      "eval_steps_per_second": 1.381,
      "step": 7300
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1195675675882825e-06,
      "logits/chosen": -2.229360580444336,
      "logits/rejected": -2.124525547027588,
      "logps/chosen": -254.34945678710938,
      "logps/rejected": -234.5399932861328,
      "loss": 2264.6254,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1915845423936844,
      "rewards/margins": 0.10795316845178604,
      "rewards/rejected": -0.29953768849372864,
      "step": 7310
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1140341707716926e-06,
      "logits/chosen": -2.1734554767608643,
      "logits/rejected": -1.928881287574768,
      "logps/chosen": -213.058837890625,
      "logps/rejected": -185.9547882080078,
      "loss": 1667.8791,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.17251141369342804,
      "rewards/margins": 0.15912136435508728,
      "rewards/rejected": -0.3316327929496765,
      "step": 7320
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1084975708520803e-06,
      "logits/chosen": -2.422889232635498,
      "logits/rejected": -2.0278878211975098,
      "logps/chosen": -277.5769958496094,
      "logps/rejected": -226.4959259033203,
      "loss": 1500.3472,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15853142738342285,
      "rewards/margins": 0.1966545730829239,
      "rewards/rejected": -0.35518598556518555,
      "step": 7330
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1029577967110625e-06,
      "logits/chosen": -2.4203174114227295,
      "logits/rejected": -2.207869291305542,
      "logps/chosen": -232.2501983642578,
      "logps/rejected": -193.13587951660156,
      "loss": 2062.1725,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1585586816072464,
      "rewards/margins": 0.09944029152393341,
      "rewards/rejected": -0.2579990029335022,
      "step": 7340
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.097414877246814e-06,
      "logits/chosen": -2.263326406478882,
      "logits/rejected": -2.018284320831299,
      "logps/chosen": -212.86083984375,
      "logps/rejected": -198.14041137695312,
      "loss": 1925.0178,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14217372238636017,
      "rewards/margins": 0.14792440831661224,
      "rewards/rejected": -0.2900981307029724,
      "step": 7350
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0918688413739197e-06,
      "logits/chosen": -2.3319170475006104,
      "logits/rejected": -2.0233492851257324,
      "logps/chosen": -243.54483032226562,
      "logps/rejected": -193.8897247314453,
      "loss": 1834.7447,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.10861881077289581,
      "rewards/margins": 0.14617590606212616,
      "rewards/rejected": -0.254794716835022,
      "step": 7360
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0863197180232178e-06,
      "logits/chosen": -2.3774733543395996,
      "logits/rejected": -2.0132126808166504,
      "logps/chosen": -213.40408325195312,
      "logps/rejected": -212.970703125,
      "loss": 2035.8984,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13659898936748505,
      "rewards/margins": 0.11787278950214386,
      "rewards/rejected": -0.2544717490673065,
      "step": 7370
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0807675361416554e-06,
      "logits/chosen": -2.2831597328186035,
      "logits/rejected": -2.0448970794677734,
      "logps/chosen": -198.5185546875,
      "logps/rejected": -134.53500366210938,
      "loss": 1672.6324,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09638302028179169,
      "rewards/margins": 0.15050837397575378,
      "rewards/rejected": -0.24689140915870667,
      "step": 7380
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0752123246921327e-06,
      "logits/chosen": -2.4044673442840576,
      "logits/rejected": -2.1336283683776855,
      "logps/chosen": -296.2977600097656,
      "logps/rejected": -243.39669799804688,
      "loss": 1830.0312,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.18384496867656708,
      "rewards/margins": 0.14030399918556213,
      "rewards/rejected": -0.324148952960968,
      "step": 7390
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.069654112653353e-06,
      "logits/chosen": -2.4212207794189453,
      "logits/rejected": -2.220515251159668,
      "logps/chosen": -235.97140502929688,
      "logps/rejected": -212.03701782226562,
      "loss": 2149.3504,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.19487205147743225,
      "rewards/margins": 0.08617673814296722,
      "rewards/rejected": -0.28104880452156067,
      "step": 7400
    },
    {
      "epoch": 0.48,
      "eval_logits/chosen": -2.3111214637756348,
      "eval_logits/rejected": -2.126542329788208,
      "eval_logps/chosen": -248.8470001220703,
      "eval_logps/rejected": -243.38546752929688,
      "eval_loss": 1797.1915283203125,
      "eval_rewards/accuracies": 0.6520000100135803,
      "eval_rewards/chosen": -0.16842052340507507,
      "eval_rewards/margins": 0.14931510388851166,
      "eval_rewards/rejected": -0.31773561239242554,
      "eval_runtime": 724.8176,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.38,
      "step": 7400
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.064092929019673e-06,
      "logits/chosen": -2.301642656326294,
      "logits/rejected": -2.334155559539795,
      "logps/chosen": -275.03240966796875,
      "logps/rejected": -306.5906066894531,
      "loss": 1980.4135,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1748892217874527,
      "rewards/margins": 0.11683398485183716,
      "rewards/rejected": -0.29172322154045105,
      "step": 7410
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.058528802800952e-06,
      "logits/chosen": -2.3407881259918213,
      "logits/rejected": -2.1059513092041016,
      "logps/chosen": -308.2147216796875,
      "logps/rejected": -286.90826416015625,
      "loss": 1935.9912,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15580996870994568,
      "rewards/margins": 0.15266405045986176,
      "rewards/rejected": -0.30847403407096863,
      "step": 7420
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.052961763022397e-06,
      "logits/chosen": -2.486992120742798,
      "logits/rejected": -2.16389799118042,
      "logps/chosen": -200.91552734375,
      "logps/rejected": -179.10459899902344,
      "loss": 1560.0854,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1803436130285263,
      "rewards/margins": 0.16912725567817688,
      "rewards/rejected": -0.3494708836078644,
      "step": 7430
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.047391838724415e-06,
      "logits/chosen": -2.4561057090759277,
      "logits/rejected": -1.99441659450531,
      "logps/chosen": -252.029296875,
      "logps/rejected": -252.39501953125,
      "loss": 1582.4479,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.16510634124279022,
      "rewards/margins": 0.16432276368141174,
      "rewards/rejected": -0.32942909002304077,
      "step": 7440
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0418190589624587e-06,
      "logits/chosen": -2.389727830886841,
      "logits/rejected": -2.125429391860962,
      "logps/chosen": -192.25135803222656,
      "logps/rejected": -213.27542114257812,
      "loss": 1738.7389,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14629261195659637,
      "rewards/margins": 0.12671688199043274,
      "rewards/rejected": -0.2730094790458679,
      "step": 7450
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0362434528068784e-06,
      "logits/chosen": -2.372622013092041,
      "logits/rejected": -1.9482421875,
      "logps/chosen": -288.8235778808594,
      "logps/rejected": -217.744140625,
      "loss": 1948.5447,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1950044482946396,
      "rewards/margins": 0.1441466361284256,
      "rewards/rejected": -0.3391510844230652,
      "step": 7460
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0306650493427657e-06,
      "logits/chosen": -2.264923095703125,
      "logits/rejected": -2.1652779579162598,
      "logps/chosen": -249.77615356445312,
      "logps/rejected": -257.1854553222656,
      "loss": 1638.9396,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1815689206123352,
      "rewards/margins": 0.16371271014213562,
      "rewards/rejected": -0.3452816605567932,
      "step": 7470
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0250838776698077e-06,
      "logits/chosen": -2.133946418762207,
      "logits/rejected": -2.1851963996887207,
      "logps/chosen": -204.72146606445312,
      "logps/rejected": -217.3024444580078,
      "loss": 1626.3103,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2080218344926834,
      "rewards/margins": 0.15327318012714386,
      "rewards/rejected": -0.3612949848175049,
      "step": 7480
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0194999669021275e-06,
      "logits/chosen": -2.1351277828216553,
      "logits/rejected": -1.8072388172149658,
      "logps/chosen": -243.4135284423828,
      "logps/rejected": -218.5195770263672,
      "loss": 1347.7062,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16545066237449646,
      "rewards/margins": 0.22158046066761017,
      "rewards/rejected": -0.3870311379432678,
      "step": 7490
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0139133461681403e-06,
      "logits/chosen": -2.2730579376220703,
      "logits/rejected": -2.1299405097961426,
      "logps/chosen": -282.8431091308594,
      "logps/rejected": -246.6148681640625,
      "loss": 1329.7865,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.18448880314826965,
      "rewards/margins": 0.19911891222000122,
      "rewards/rejected": -0.3836076855659485,
      "step": 7500
    },
    {
      "epoch": 0.49,
      "eval_logits/chosen": -2.3219900131225586,
      "eval_logits/rejected": -2.136725664138794,
      "eval_logps/chosen": -249.3482208251953,
      "eval_logps/rejected": -244.99862670898438,
      "eval_loss": 1795.6614990234375,
      "eval_rewards/accuracies": 0.6585000157356262,
      "eval_rewards/chosen": -0.17343279719352722,
      "eval_rewards/margins": 0.16043433547019958,
      "eval_rewards/rejected": -0.3338671326637268,
      "eval_runtime": 725.297,
      "eval_samples_per_second": 2.757,
      "eval_steps_per_second": 1.379,
      "step": 7500
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0083240446103965e-06,
      "logits/chosen": -2.050713062286377,
      "logits/rejected": -2.0219483375549316,
      "logps/chosen": -202.81639099121094,
      "logps/rejected": -226.4502410888672,
      "loss": 1777.7549,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.18781790137290955,
      "rewards/margins": 0.1596471667289734,
      "rewards/rejected": -0.3474650979042053,
      "step": 7510
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0027320913854306e-06,
      "logits/chosen": -2.513092517852783,
      "logits/rejected": -2.2261266708374023,
      "logps/chosen": -304.5718994140625,
      "logps/rejected": -266.6675720214844,
      "loss": 1646.9309,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.12110300362110138,
      "rewards/margins": 0.2480504810810089,
      "rewards/rejected": -0.3691534399986267,
      "step": 7520
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.997137515663609e-06,
      "logits/chosen": -2.2597782611846924,
      "logits/rejected": -2.1838889122009277,
      "logps/chosen": -236.5629425048828,
      "logps/rejected": -221.2807159423828,
      "loss": 1442.7711,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1118551641702652,
      "rewards/margins": 0.22416965663433075,
      "rewards/rejected": -0.33602482080459595,
      "step": 7530
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.991540346628981e-06,
      "logits/chosen": -2.374667167663574,
      "logits/rejected": -2.191114664077759,
      "logps/chosen": -263.5557861328125,
      "logps/rejected": -251.18161010742188,
      "loss": 1976.7508,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23955456912517548,
      "rewards/margins": 0.1373405158519745,
      "rewards/rejected": -0.37689509987831116,
      "step": 7540
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.985940613479121e-06,
      "logits/chosen": -2.4493889808654785,
      "logits/rejected": -2.342282772064209,
      "logps/chosen": -312.0015563964844,
      "logps/rejected": -273.027099609375,
      "loss": 1377.7288,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.18151938915252686,
      "rewards/margins": 0.20397362112998962,
      "rewards/rejected": -0.38549304008483887,
      "step": 7550
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.980338345424981e-06,
      "logits/chosen": -2.3101601600646973,
      "logits/rejected": -2.007136821746826,
      "logps/chosen": -268.2823791503906,
      "logps/rejected": -233.455810546875,
      "loss": 1776.1154,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19062550365924835,
      "rewards/margins": 0.13450220227241516,
      "rewards/rejected": -0.3251277208328247,
      "step": 7560
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.974733571690735e-06,
      "logits/chosen": -2.396482467651367,
      "logits/rejected": -2.1252634525299072,
      "logps/chosen": -260.60748291015625,
      "logps/rejected": -213.5504150390625,
      "loss": 2111.8732,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2320062220096588,
      "rewards/margins": 0.1223546713590622,
      "rewards/rejected": -0.3543609082698822,
      "step": 7570
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9691263215136274e-06,
      "logits/chosen": -2.3606293201446533,
      "logits/rejected": -2.3490958213806152,
      "logps/chosen": -282.8438415527344,
      "logps/rejected": -268.9943542480469,
      "loss": 1524.6323,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1584346890449524,
      "rewards/margins": 0.17805036902427673,
      "rewards/rejected": -0.3364850878715515,
      "step": 7580
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.963516624143823e-06,
      "logits/chosen": -2.266894578933716,
      "logits/rejected": -2.1448280811309814,
      "logps/chosen": -236.0391387939453,
      "logps/rejected": -210.7563934326172,
      "loss": 2044.7648,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.23472432792186737,
      "rewards/margins": 0.12248021364212036,
      "rewards/rejected": -0.35720449686050415,
      "step": 7590
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9579045088442504e-06,
      "logits/chosen": -2.1435303688049316,
      "logits/rejected": -2.192343235015869,
      "logps/chosen": -202.0614776611328,
      "logps/rejected": -239.5817413330078,
      "loss": 1424.3902,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13556505739688873,
      "rewards/margins": 0.17771312594413757,
      "rewards/rejected": -0.3132781684398651,
      "step": 7600
    },
    {
      "epoch": 0.5,
      "eval_logits/chosen": -2.310932159423828,
      "eval_logits/rejected": -2.127431631088257,
      "eval_logps/chosen": -245.70091247558594,
      "eval_logps/rejected": -242.0367431640625,
      "eval_loss": 1812.0533447265625,
      "eval_rewards/accuracies": 0.656499981880188,
      "eval_rewards/chosen": -0.13695943355560303,
      "eval_rewards/margins": 0.1672891527414322,
      "eval_rewards/rejected": -0.3042486011981964,
      "eval_runtime": 724.3066,
      "eval_samples_per_second": 2.761,
      "eval_steps_per_second": 1.381,
      "step": 7600
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9522900048904534e-06,
      "logits/chosen": -2.2314212322235107,
      "logits/rejected": -2.1451785564422607,
      "logps/chosen": -259.7351379394531,
      "logps/rejected": -238.1786651611328,
      "loss": 2245.4512,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1999697983264923,
      "rewards/margins": 0.0846501812338829,
      "rewards/rejected": -0.2846199870109558,
      "step": 7610
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9466731415704343e-06,
      "logits/chosen": -2.3211567401885986,
      "logits/rejected": -2.1959145069122314,
      "logps/chosen": -236.20181274414062,
      "logps/rejected": -251.24313354492188,
      "loss": 1601.2445,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12415148317813873,
      "rewards/margins": 0.19408674538135529,
      "rewards/rejected": -0.31823819875717163,
      "step": 7620
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.941053948184503e-06,
      "logits/chosen": -2.3644680976867676,
      "logits/rejected": -2.2212233543395996,
      "logps/chosen": -288.4496154785156,
      "logps/rejected": -269.64007568359375,
      "loss": 1897.9051,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08510540425777435,
      "rewards/margins": 0.16102668642997742,
      "rewards/rejected": -0.24613209068775177,
      "step": 7630
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.935432454045125e-06,
      "logits/chosen": -2.1752820014953613,
      "logits/rejected": -2.2406585216522217,
      "logps/chosen": -248.2934112548828,
      "logps/rejected": -237.24526977539062,
      "loss": 2031.8443,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16329045593738556,
      "rewards/margins": 0.10159357637166977,
      "rewards/rejected": -0.2648840546607971,
      "step": 7640
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.929808688476768e-06,
      "logits/chosen": -2.3958911895751953,
      "logits/rejected": -2.271388292312622,
      "logps/chosen": -252.97970581054688,
      "logps/rejected": -251.6690216064453,
      "loss": 1745.6922,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1361730545759201,
      "rewards/margins": 0.1629178375005722,
      "rewards/rejected": -0.2990909218788147,
      "step": 7650
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.924182680815748e-06,
      "logits/chosen": -2.317483425140381,
      "logits/rejected": -2.247915267944336,
      "logps/chosen": -245.28451538085938,
      "logps/rejected": -248.97830200195312,
      "loss": 1318.8134,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10653947293758392,
      "rewards/margins": 0.26401370763778687,
      "rewards/rejected": -0.370553195476532,
      "step": 7660
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9185544604100765e-06,
      "logits/chosen": -2.10002064704895,
      "logits/rejected": -2.007351875305176,
      "logps/chosen": -217.10092163085938,
      "logps/rejected": -224.46530151367188,
      "loss": 2086.9361,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2119079828262329,
      "rewards/margins": 0.11619792133569717,
      "rewards/rejected": -0.3281059265136719,
      "step": 7670
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9129240566193083e-06,
      "logits/chosen": -2.4031784534454346,
      "logits/rejected": -2.086520195007324,
      "logps/chosen": -220.94961547851562,
      "logps/rejected": -224.2306671142578,
      "loss": 1544.9651,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1886926144361496,
      "rewards/margins": 0.16569046676158905,
      "rewards/rejected": -0.35438311100006104,
      "step": 7680
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9072914988143874e-06,
      "logits/chosen": -2.161214590072632,
      "logits/rejected": -2.080164909362793,
      "logps/chosen": -217.6607208251953,
      "logps/rejected": -232.4710235595703,
      "loss": 1151.4793,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.16230908036231995,
      "rewards/margins": 0.2328173667192459,
      "rewards/rejected": -0.3951264023780823,
      "step": 7690
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9016568163774956e-06,
      "logits/chosen": -2.3935441970825195,
      "logits/rejected": -2.1594557762145996,
      "logps/chosen": -183.3972625732422,
      "logps/rejected": -164.7067108154297,
      "loss": 1652.7855,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11167971789836884,
      "rewards/margins": 0.15362325310707092,
      "rewards/rejected": -0.26530295610427856,
      "step": 7700
    },
    {
      "epoch": 0.5,
      "eval_logits/chosen": -2.3180606365203857,
      "eval_logits/rejected": -2.1330764293670654,
      "eval_logps/chosen": -245.59124755859375,
      "eval_logps/rejected": -241.61257934570312,
      "eval_loss": 1805.7569580078125,
      "eval_rewards/accuracies": 0.6524999737739563,
      "eval_rewards/chosen": -0.13586299121379852,
      "eval_rewards/margins": 0.16414377093315125,
      "eval_rewards/rejected": -0.30000677704811096,
      "eval_runtime": 724.718,
      "eval_samples_per_second": 2.76,
      "eval_steps_per_second": 1.38,
      "step": 7700
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.8960200387018942e-06,
      "logits/chosen": -2.148181200027466,
      "logits/rejected": -2.120988368988037,
      "logps/chosen": -326.7958984375,
      "logps/rejected": -287.37103271484375,
      "loss": 2206.0355,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.19062894582748413,
      "rewards/margins": 0.09409534931182861,
      "rewards/rejected": -0.28472429513931274,
      "step": 7710
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8903811951917792e-06,
      "logits/chosen": -2.3121659755706787,
      "logits/rejected": -2.1631019115448,
      "logps/chosen": -208.6977081298828,
      "logps/rejected": -175.44943237304688,
      "loss": 1798.4217,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.104180708527565,
      "rewards/margins": 0.13893906772136688,
      "rewards/rejected": -0.2431197613477707,
      "step": 7720
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.88474031526212e-06,
      "logits/chosen": -2.2695491313934326,
      "logits/rejected": -2.2421715259552,
      "logps/chosen": -213.50888061523438,
      "logps/rejected": -238.98361206054688,
      "loss": 1972.915,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1066913828253746,
      "rewards/margins": 0.12046156078577042,
      "rewards/rejected": -0.22715294361114502,
      "step": 7730
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.879097428338509e-06,
      "logits/chosen": -2.263644218444824,
      "logits/rejected": -1.9583408832550049,
      "logps/chosen": -226.1009521484375,
      "logps/rejected": -218.6776885986328,
      "loss": 1923.6541,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10205962508916855,
      "rewards/margins": 0.1490507423877716,
      "rewards/rejected": -0.25111037492752075,
      "step": 7740
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8734525638570094e-06,
      "logits/chosen": -2.2643754482269287,
      "logits/rejected": -2.1896746158599854,
      "logps/chosen": -241.3166046142578,
      "logps/rejected": -246.44775390625,
      "loss": 1724.5227,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07122374325990677,
      "rewards/margins": 0.17504428327083588,
      "rewards/rejected": -0.24626803398132324,
      "step": 7750
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8678057512639982e-06,
      "logits/chosen": -2.2165639400482178,
      "logits/rejected": -2.12363862991333,
      "logps/chosen": -291.8692626953125,
      "logps/rejected": -293.48944091796875,
      "loss": 1396.4198,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05802815407514572,
      "rewards/margins": 0.23679597675800323,
      "rewards/rejected": -0.29482412338256836,
      "step": 7760
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8621570200160172e-06,
      "logits/chosen": -2.1056911945343018,
      "logits/rejected": -2.0133914947509766,
      "logps/chosen": -175.77249145507812,
      "logps/rejected": -190.22726440429688,
      "loss": 1617.67,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07203412801027298,
      "rewards/margins": 0.22632765769958496,
      "rewards/rejected": -0.29836180806159973,
      "step": 7770
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.856506399579615e-06,
      "logits/chosen": -2.4490838050842285,
      "logits/rejected": -2.0760135650634766,
      "logps/chosen": -235.693603515625,
      "logps/rejected": -228.88259887695312,
      "loss": 2112.6408,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1733010709285736,
      "rewards/margins": 0.1267828792333603,
      "rewards/rejected": -0.3000839650630951,
      "step": 7780
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8508539194311964e-06,
      "logits/chosen": -2.3463687896728516,
      "logits/rejected": -2.335172414779663,
      "logps/chosen": -268.44915771484375,
      "logps/rejected": -295.9856872558594,
      "loss": 1723.3611,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1471569985151291,
      "rewards/margins": 0.14706538617610931,
      "rewards/rejected": -0.294222354888916,
      "step": 7790
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8451996090568656e-06,
      "logits/chosen": -2.2608280181884766,
      "logits/rejected": -2.14410400390625,
      "logps/chosen": -199.97055053710938,
      "logps/rejected": -201.8147735595703,
      "loss": 1540.4484,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13830958306789398,
      "rewards/margins": 0.19805839657783508,
      "rewards/rejected": -0.33636802434921265,
      "step": 7800
    },
    {
      "epoch": 0.51,
      "eval_logits/chosen": -2.299730062484741,
      "eval_logits/rejected": -2.1168112754821777,
      "eval_logps/chosen": -247.72259521484375,
      "eval_logps/rejected": -242.64073181152344,
      "eval_loss": 1808.3577880859375,
      "eval_rewards/accuracies": 0.656000018119812,
      "eval_rewards/chosen": -0.15717624127864838,
      "eval_rewards/margins": 0.15311181545257568,
      "eval_rewards/rejected": -0.31028807163238525,
      "eval_runtime": 724.637,
      "eval_samples_per_second": 2.76,
      "eval_steps_per_second": 1.38,
      "step": 7800
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.839543497952276e-06,
      "logits/chosen": -2.1900064945220947,
      "logits/rejected": -2.298042058944702,
      "logps/chosen": -201.76132202148438,
      "logps/rejected": -209.26272583007812,
      "loss": 1763.2158,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.17151224613189697,
      "rewards/margins": 0.16205264627933502,
      "rewards/rejected": -0.3335648775100708,
      "step": 7810
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.833885615622474e-06,
      "logits/chosen": -2.23939847946167,
      "logits/rejected": -2.090911626815796,
      "logps/chosen": -228.0774383544922,
      "logps/rejected": -247.0799560546875,
      "loss": 2073.5559,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.2550160884857178,
      "rewards/margins": 0.09527119994163513,
      "rewards/rejected": -0.3502873182296753,
      "step": 7820
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8282259915817454e-06,
      "logits/chosen": -1.9413120746612549,
      "logits/rejected": -2.143981456756592,
      "logps/chosen": -158.98806762695312,
      "logps/rejected": -215.71817016601562,
      "loss": 1776.5148,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18043342232704163,
      "rewards/margins": 0.16518518328666687,
      "rewards/rejected": -0.3456185758113861,
      "step": 7830
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8225646553534614e-06,
      "logits/chosen": -2.1059117317199707,
      "logits/rejected": -1.999498963356018,
      "logps/chosen": -213.69155883789062,
      "logps/rejected": -221.0135040283203,
      "loss": 2001.6479,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.14375567436218262,
      "rewards/margins": 0.10613244771957397,
      "rewards/rejected": -0.24988813698291779,
      "step": 7840
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8169016364699255e-06,
      "logits/chosen": -2.2967660427093506,
      "logits/rejected": -2.025442123413086,
      "logps/chosen": -234.79638671875,
      "logps/rejected": -250.521240234375,
      "loss": 1723.0146,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.22908520698547363,
      "rewards/margins": 0.1397470086812973,
      "rewards/rejected": -0.36883223056793213,
      "step": 7850
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.811236964472217e-06,
      "logits/chosen": -2.4080090522766113,
      "logits/rejected": -2.0328783988952637,
      "logps/chosen": -324.30859375,
      "logps/rejected": -281.63848876953125,
      "loss": 1810.1021,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13902036845684052,
      "rewards/margins": 0.174498051404953,
      "rewards/rejected": -0.3135184049606323,
      "step": 7860
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.805570668910041e-06,
      "logits/chosen": -2.1317176818847656,
      "logits/rejected": -2.0996460914611816,
      "logps/chosen": -188.90228271484375,
      "logps/rejected": -266.41033935546875,
      "loss": 1824.0271,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.19262978434562683,
      "rewards/margins": 0.14007696509361267,
      "rewards/rejected": -0.3327067494392395,
      "step": 7870
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7999027793415695e-06,
      "logits/chosen": -2.5066134929656982,
      "logits/rejected": -2.0417120456695557,
      "logps/chosen": -257.93975830078125,
      "logps/rejected": -228.1025848388672,
      "loss": 1932.0182,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1302376091480255,
      "rewards/margins": 0.1508847177028656,
      "rewards/rejected": -0.2811223268508911,
      "step": 7880
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.794233325333293e-06,
      "logits/chosen": -2.1897034645080566,
      "logits/rejected": -2.0873708724975586,
      "logps/chosen": -274.3140563964844,
      "logps/rejected": -270.84112548828125,
      "loss": 1531.2451,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16032613813877106,
      "rewards/margins": 0.2020435333251953,
      "rewards/rejected": -0.36236971616744995,
      "step": 7890
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7885623364598597e-06,
      "logits/chosen": -2.408759117126465,
      "logits/rejected": -2.1068990230560303,
      "logps/chosen": -278.29742431640625,
      "logps/rejected": -260.3435974121094,
      "loss": 1297.3996,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14604783058166504,
      "rewards/margins": 0.23965105414390564,
      "rewards/rejected": -0.3856988549232483,
      "step": 7900
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.313547134399414,
      "eval_logits/rejected": -2.1292340755462646,
      "eval_logps/chosen": -247.32740783691406,
      "eval_logps/rejected": -243.37562561035156,
      "eval_loss": 1798.07177734375,
      "eval_rewards/accuracies": 0.6499999761581421,
      "eval_rewards/chosen": -0.1532244086265564,
      "eval_rewards/margins": 0.1644127070903778,
      "eval_rewards/rejected": -0.3176371157169342,
      "eval_runtime": 725.0357,
      "eval_samples_per_second": 2.758,
      "eval_steps_per_second": 1.379,
      "step": 7900
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.782889842303926e-06,
      "logits/chosen": -2.2938106060028076,
      "logits/rejected": -2.121316432952881,
      "logps/chosen": -179.6522674560547,
      "logps/rejected": -184.91526794433594,
      "loss": 2068.6041,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.19423159956932068,
      "rewards/margins": 0.10760875791311264,
      "rewards/rejected": -0.3018403649330139,
      "step": 7910
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7772158724559987e-06,
      "logits/chosen": -2.118116855621338,
      "logits/rejected": -1.9713754653930664,
      "logps/chosen": -230.3015899658203,
      "logps/rejected": -293.75885009765625,
      "loss": 1291.2383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1319253146648407,
      "rewards/margins": 0.25986504554748535,
      "rewards/rejected": -0.39179039001464844,
      "step": 7920
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7715404565142856e-06,
      "logits/chosen": -2.283179759979248,
      "logits/rejected": -2.143989324569702,
      "logps/chosen": -217.41445922851562,
      "logps/rejected": -225.4498748779297,
      "loss": 1855.3721,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1745026856660843,
      "rewards/margins": 0.14258040487766266,
      "rewards/rejected": -0.31708306074142456,
      "step": 7930
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7658636240845354e-06,
      "logits/chosen": -2.4289021492004395,
      "logits/rejected": -2.3265390396118164,
      "logps/chosen": -240.6270751953125,
      "logps/rejected": -274.0114440917969,
      "loss": 1486.8823,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17178744077682495,
      "rewards/margins": 0.18766279518604279,
      "rewards/rejected": -0.35945025086402893,
      "step": 7940
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7601854047798872e-06,
      "logits/chosen": -2.236783504486084,
      "logits/rejected": -2.273075580596924,
      "logps/chosen": -241.3152618408203,
      "logps/rejected": -273.97210693359375,
      "loss": 1882.7766,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.17391879856586456,
      "rewards/margins": 0.13241559267044067,
      "rewards/rejected": -0.3063344359397888,
      "step": 7950
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7545058282207148e-06,
      "logits/chosen": -2.384068489074707,
      "logits/rejected": -1.9915987253189087,
      "logps/chosen": -222.35446166992188,
      "logps/rejected": -206.6442413330078,
      "loss": 1766.518,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10812662541866302,
      "rewards/margins": 0.14868710935115814,
      "rewards/rejected": -0.2568137049674988,
      "step": 7960
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.748824924034471e-06,
      "logits/chosen": -2.298893928527832,
      "logits/rejected": -2.1646010875701904,
      "logps/chosen": -241.891357421875,
      "logps/rejected": -234.1492462158203,
      "loss": 2235.9057,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.22481146454811096,
      "rewards/margins": 0.11154550313949585,
      "rewards/rejected": -0.3363569676876068,
      "step": 7970
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.743142721855536e-06,
      "logits/chosen": -2.180006980895996,
      "logits/rejected": -2.1479909420013428,
      "logps/chosen": -166.78038024902344,
      "logps/rejected": -169.9340362548828,
      "loss": 1829.8742,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.14920267462730408,
      "rewards/margins": 0.11502110958099365,
      "rewards/rejected": -0.26422375440597534,
      "step": 7980
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.737459251325058e-06,
      "logits/chosen": -2.288517475128174,
      "logits/rejected": -2.2369046211242676,
      "logps/chosen": -282.2027587890625,
      "logps/rejected": -271.6640625,
      "loss": 1784.1377,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.12229214608669281,
      "rewards/margins": 0.1199268251657486,
      "rewards/rejected": -0.2422189712524414,
      "step": 7990
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.731774542090804e-06,
      "logits/chosen": -2.2692532539367676,
      "logits/rejected": -1.8465888500213623,
      "logps/chosen": -209.0124969482422,
      "logps/rejected": -196.2694091796875,
      "loss": 2462.926,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.16236457228660583,
      "rewards/margins": 0.06291927397251129,
      "rewards/rejected": -0.22528386116027832,
      "step": 8000
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.326558828353882,
      "eval_logits/rejected": -2.1409735679626465,
      "eval_logps/chosen": -246.2211456298828,
      "eval_logps/rejected": -240.75645446777344,
      "eval_loss": 1784.8865966796875,
      "eval_rewards/accuracies": 0.6554999947547913,
      "eval_rewards/chosen": -0.14216183125972748,
      "eval_rewards/margins": 0.1492837369441986,
      "eval_rewards/rejected": -0.2914455533027649,
      "eval_runtime": 726.2338,
      "eval_samples_per_second": 2.754,
      "eval_steps_per_second": 1.377,
      "step": 8000
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7260886238070034e-06,
      "logits/chosen": -2.3285775184631348,
      "logits/rejected": -2.246692657470703,
      "logps/chosen": -211.03939819335938,
      "logps/rejected": -223.37353515625,
      "loss": 1469.5437,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.14569437503814697,
      "rewards/margins": 0.17042239010334015,
      "rewards/rejected": -0.3161167502403259,
      "step": 8010
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.72040152613419e-06,
      "logits/chosen": -2.364499807357788,
      "logits/rejected": -1.8563902378082275,
      "logps/chosen": -232.80636596679688,
      "logps/rejected": -167.91128540039062,
      "loss": 1375.5312,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15931276977062225,
      "rewards/margins": 0.18330441415309906,
      "rewards/rejected": -0.3426171839237213,
      "step": 8020
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.7147132787390516e-06,
      "logits/chosen": -2.3446335792541504,
      "logits/rejected": -2.056989908218384,
      "logps/chosen": -238.309326171875,
      "logps/rejected": -240.9357452392578,
      "loss": 1498.3412,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10831235349178314,
      "rewards/margins": 0.19611985981464386,
      "rewards/rejected": -0.304432213306427,
      "step": 8030
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.709023911294273e-06,
      "logits/chosen": -2.429610013961792,
      "logits/rejected": -1.9592183828353882,
      "logps/chosen": -247.37881469726562,
      "logps/rejected": -243.9899139404297,
      "loss": 1500.2393,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.05059943348169327,
      "rewards/margins": 0.27510666847229004,
      "rewards/rejected": -0.3257060647010803,
      "step": 8040
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.7033334534783806e-06,
      "logits/chosen": -2.321929454803467,
      "logits/rejected": -2.4168248176574707,
      "logps/chosen": -209.8223419189453,
      "logps/rejected": -239.98876953125,
      "loss": 2167.8779,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10676448047161102,
      "rewards/margins": 0.1396731436252594,
      "rewards/rejected": -0.24643762409687042,
      "step": 8050
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.697641934975592e-06,
      "logits/chosen": -2.327284097671509,
      "logits/rejected": -2.1021924018859863,
      "logps/chosen": -236.8908233642578,
      "logps/rejected": -221.6531982421875,
      "loss": 1820.2213,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09104406088590622,
      "rewards/margins": 0.17316171526908875,
      "rewards/rejected": -0.26420578360557556,
      "step": 8060
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.691949385475654e-06,
      "logits/chosen": -2.3733315467834473,
      "logits/rejected": -2.116386890411377,
      "logps/chosen": -255.9573974609375,
      "logps/rejected": -245.44058227539062,
      "loss": 1706.3115,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1206657662987709,
      "rewards/margins": 0.15054434537887573,
      "rewards/rejected": -0.2712101340293884,
      "step": 8070
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6862558346736937e-06,
      "logits/chosen": -2.2894411087036133,
      "logits/rejected": -2.1502037048339844,
      "logps/chosen": -250.6189727783203,
      "logps/rejected": -265.82891845703125,
      "loss": 1447.9819,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.12266738712787628,
      "rewards/margins": 0.20411674678325653,
      "rewards/rejected": -0.3267841637134552,
      "step": 8080
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6805613122700617e-06,
      "logits/chosen": -2.3372228145599365,
      "logits/rejected": -1.9962005615234375,
      "logps/chosen": -238.9376220703125,
      "logps/rejected": -253.9717559814453,
      "loss": 1820.5197,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1536514163017273,
      "rewards/margins": 0.14091987907886505,
      "rewards/rejected": -0.29457131028175354,
      "step": 8090
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.674865847970176e-06,
      "logits/chosen": -2.2625794410705566,
      "logits/rejected": -1.9928855895996094,
      "logps/chosen": -219.21652221679688,
      "logps/rejected": -254.426025390625,
      "loss": 1812.3775,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11600162833929062,
      "rewards/margins": 0.12761160731315613,
      "rewards/rejected": -0.24361324310302734,
      "step": 8100
    },
    {
      "epoch": 0.53,
      "eval_logits/chosen": -2.320788621902466,
      "eval_logits/rejected": -2.1356489658355713,
      "eval_logps/chosen": -244.53488159179688,
      "eval_logps/rejected": -238.00411987304688,
      "eval_loss": 1789.9876708984375,
      "eval_rewards/accuracies": 0.6545000076293945,
      "eval_rewards/chosen": -0.12529927492141724,
      "eval_rewards/margins": 0.13862305879592896,
      "eval_rewards/rejected": -0.2639223337173462,
      "eval_runtime": 724.9601,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.379,
      "step": 8100
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.669169471484368e-06,
      "logits/chosen": -2.08323335647583,
      "logits/rejected": -2.1388871669769287,
      "logps/chosen": -182.0511474609375,
      "logps/rejected": -188.3828887939453,
      "loss": 2110.5072,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.16993075609207153,
      "rewards/margins": 0.08971955627202988,
      "rewards/rejected": -0.2596503496170044,
      "step": 8110
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6634722125277278e-06,
      "logits/chosen": -2.420374631881714,
      "logits/rejected": -2.0989325046539307,
      "logps/chosen": -248.5947723388672,
      "logps/rejected": -268.76654052734375,
      "loss": 1920.4986,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.16474846005439758,
      "rewards/margins": 0.11976997554302216,
      "rewards/rejected": -0.28451845049858093,
      "step": 8120
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6577741008199498e-06,
      "logits/chosen": -2.334933042526245,
      "logits/rejected": -1.9777965545654297,
      "logps/chosen": -268.9001770019531,
      "logps/rejected": -229.83541870117188,
      "loss": 1325.9265,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.12741991877555847,
      "rewards/margins": 0.2097659558057785,
      "rewards/rejected": -0.3371858596801758,
      "step": 8130
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.652075166085175e-06,
      "logits/chosen": -2.222378969192505,
      "logits/rejected": -2.175022602081299,
      "logps/chosen": -246.1367645263672,
      "logps/rejected": -289.9963684082031,
      "loss": 1941.0016,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11557464301586151,
      "rewards/margins": 0.2031438648700714,
      "rewards/rejected": -0.31871849298477173,
      "step": 8140
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6463754380518395e-06,
      "logits/chosen": -2.2017464637756348,
      "logits/rejected": -1.9937317371368408,
      "logps/chosen": -247.4430694580078,
      "logps/rejected": -205.15609741210938,
      "loss": 1995.1008,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1648540049791336,
      "rewards/margins": 0.12540122866630554,
      "rewards/rejected": -0.29025521874427795,
      "step": 8150
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6406749464525167e-06,
      "logits/chosen": -2.306833028793335,
      "logits/rejected": -2.032440423965454,
      "logps/chosen": -237.3766632080078,
      "logps/rejected": -207.55819702148438,
      "loss": 1552.0191,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03817574307322502,
      "rewards/margins": 0.17413334548473358,
      "rewards/rejected": -0.2123090922832489,
      "step": 8160
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.634973721023762e-06,
      "logits/chosen": -2.347939968109131,
      "logits/rejected": -2.1919798851013184,
      "logps/chosen": -265.0765686035156,
      "logps/rejected": -227.37551879882812,
      "loss": 2049.8635,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1061432734131813,
      "rewards/margins": 0.11737523972988129,
      "rewards/rejected": -0.22351853549480438,
      "step": 8170
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6292717915059605e-06,
      "logits/chosen": -2.382143020629883,
      "logits/rejected": -2.1191394329071045,
      "logps/chosen": -285.326171875,
      "logps/rejected": -247.8603515625,
      "loss": 1394.937,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.09696097671985626,
      "rewards/margins": 0.18727213144302368,
      "rewards/rejected": -0.28423309326171875,
      "step": 8180
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6235691876431706e-06,
      "logits/chosen": -2.1982357501983643,
      "logits/rejected": -2.253572702407837,
      "logps/chosen": -231.0131378173828,
      "logps/rejected": -249.4632110595703,
      "loss": 1836.2434,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12397567927837372,
      "rewards/margins": 0.12850920855998993,
      "rewards/rejected": -0.25248488783836365,
      "step": 8190
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6178659391829673e-06,
      "logits/chosen": -2.4205422401428223,
      "logits/rejected": -2.149384021759033,
      "logps/chosen": -242.65371704101562,
      "logps/rejected": -219.2117462158203,
      "loss": 1606.6738,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.051757581532001495,
      "rewards/margins": 0.17401257157325745,
      "rewards/rejected": -0.22577011585235596,
      "step": 8200
    },
    {
      "epoch": 0.54,
      "eval_logits/chosen": -2.302276372909546,
      "eval_logits/rejected": -2.1190202236175537,
      "eval_logps/chosen": -241.52880859375,
      "eval_logps/rejected": -235.77371215820312,
      "eval_loss": 1797.2037353515625,
      "eval_rewards/accuracies": 0.6460000276565552,
      "eval_rewards/chosen": -0.09523871541023254,
      "eval_rewards/margins": 0.146379292011261,
      "eval_rewards/rejected": -0.24161800742149353,
      "eval_runtime": 722.3242,
      "eval_samples_per_second": 2.769,
      "eval_steps_per_second": 1.384,
      "step": 8200
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6121620758762877e-06,
      "logits/chosen": -2.289379596710205,
      "logits/rejected": -2.0412538051605225,
      "logps/chosen": -200.6814422607422,
      "logps/rejected": -214.49612426757812,
      "loss": 1785.8699,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08597119152545929,
      "rewards/margins": 0.14613738656044006,
      "rewards/rejected": -0.23210859298706055,
      "step": 8210
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.606457627477277e-06,
      "logits/chosen": -2.2264838218688965,
      "logits/rejected": -2.172628402709961,
      "logps/chosen": -181.49755859375,
      "logps/rejected": -202.78858947753906,
      "loss": 1664.8721,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05868273228406906,
      "rewards/margins": 0.17938938736915588,
      "rewards/rejected": -0.23807211220264435,
      "step": 8220
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6007526237431324e-06,
      "logits/chosen": -2.3600382804870605,
      "logits/rejected": -2.3124423027038574,
      "logps/chosen": -189.7965850830078,
      "logps/rejected": -219.19058227539062,
      "loss": 1906.6195,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07767064869403839,
      "rewards/margins": 0.15275943279266357,
      "rewards/rejected": -0.23043008148670197,
      "step": 8230
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5950470944339478e-06,
      "logits/chosen": -2.135626792907715,
      "logits/rejected": -2.2016496658325195,
      "logps/chosen": -228.05728149414062,
      "logps/rejected": -234.7489776611328,
      "loss": 1881.5312,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.06255332380533218,
      "rewards/margins": 0.10187490284442902,
      "rewards/rejected": -0.1644282042980194,
      "step": 8240
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.58934106931256e-06,
      "logits/chosen": -2.2805557250976562,
      "logits/rejected": -1.9872407913208008,
      "logps/chosen": -231.88522338867188,
      "logps/rejected": -226.14285278320312,
      "loss": 2058.7807,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.11113202571868896,
      "rewards/margins": 0.13435114920139313,
      "rewards/rejected": -0.2454831600189209,
      "step": 8250
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.58363457814439e-06,
      "logits/chosen": -2.2726893424987793,
      "logits/rejected": -1.9903056621551514,
      "logps/chosen": -227.5226593017578,
      "logps/rejected": -227.06906127929688,
      "loss": 2035.3699,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.16850681602954865,
      "rewards/margins": 0.13269905745983124,
      "rewards/rejected": -0.30120590329170227,
      "step": 8260
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5779276506972924e-06,
      "logits/chosen": -2.242635488510132,
      "logits/rejected": -2.2251079082489014,
      "logps/chosen": -245.4095458984375,
      "logps/rejected": -219.79159545898438,
      "loss": 1770.1719,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.134566992521286,
      "rewards/margins": 0.1376422643661499,
      "rewards/rejected": -0.2722092568874359,
      "step": 8270
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5722203167413945e-06,
      "logits/chosen": -2.3671107292175293,
      "logits/rejected": -2.0390262603759766,
      "logps/chosen": -294.7353515625,
      "logps/rejected": -228.14974975585938,
      "loss": 1792.4943,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11437664926052094,
      "rewards/margins": 0.15335258841514587,
      "rewards/rejected": -0.267729252576828,
      "step": 8280
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5665126060489476e-06,
      "logits/chosen": -2.3390555381774902,
      "logits/rejected": -2.184868097305298,
      "logps/chosen": -205.6299285888672,
      "logps/rejected": -246.21206665039062,
      "loss": 1913.0139,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.17527888715267181,
      "rewards/margins": 0.11735512316226959,
      "rewards/rejected": -0.2926340103149414,
      "step": 8290
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.560804548394165e-06,
      "logits/chosen": -2.253669500350952,
      "logits/rejected": -2.004037857055664,
      "logps/chosen": -258.77728271484375,
      "logps/rejected": -230.3441619873047,
      "loss": 1769.8457,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11496938765048981,
      "rewards/margins": 0.16839106380939484,
      "rewards/rejected": -0.28336042165756226,
      "step": 8300
    },
    {
      "epoch": 0.54,
      "eval_logits/chosen": -2.306878089904785,
      "eval_logits/rejected": -2.1231863498687744,
      "eval_logps/chosen": -242.7214813232422,
      "eval_logps/rejected": -236.77609252929688,
      "eval_loss": 1802.872802734375,
      "eval_rewards/accuracies": 0.6449999809265137,
      "eval_rewards/chosen": -0.10716545581817627,
      "eval_rewards/margins": 0.1444765329360962,
      "eval_rewards/rejected": -0.25164198875427246,
      "eval_runtime": 726.3551,
      "eval_samples_per_second": 2.753,
      "eval_steps_per_second": 1.377,
      "step": 8300
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5550961735530734e-06,
      "logits/chosen": -2.142177104949951,
      "logits/rejected": -2.322173595428467,
      "logps/chosen": -167.76272583007812,
      "logps/rejected": -215.3518524169922,
      "loss": 1583.1814,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.047269057482481,
      "rewards/margins": 0.16998426616191864,
      "rewards/rejected": -0.21725329756736755,
      "step": 8310
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.549387511303351e-06,
      "logits/chosen": -2.2949604988098145,
      "logits/rejected": -2.3329718112945557,
      "logps/chosen": -177.2271728515625,
      "logps/rejected": -232.924072265625,
      "loss": 1899.2404,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07490333169698715,
      "rewards/margins": 0.11416280269622803,
      "rewards/rejected": -0.18906612694263458,
      "step": 8320
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5436785914241774e-06,
      "logits/chosen": -2.250805377960205,
      "logits/rejected": -2.2805335521698,
      "logps/chosen": -211.9129638671875,
      "logps/rejected": -195.1792449951172,
      "loss": 1938.2619,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12953725457191467,
      "rewards/margins": 0.15235944092273712,
      "rewards/rejected": -0.281896710395813,
      "step": 8330
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5379694436960746e-06,
      "logits/chosen": -2.4175267219543457,
      "logits/rejected": -2.2193617820739746,
      "logps/chosen": -253.90762329101562,
      "logps/rejected": -279.6899108886719,
      "loss": 1799.4357,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0789659172296524,
      "rewards/margins": 0.152174711227417,
      "rewards/rejected": -0.2311406433582306,
      "step": 8340
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5322600979007533e-06,
      "logits/chosen": -2.4399263858795166,
      "logits/rejected": -2.207584857940674,
      "logps/chosen": -223.945556640625,
      "logps/rejected": -214.1793975830078,
      "loss": 1980.3602,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.11450109630823135,
      "rewards/margins": 0.10933053493499756,
      "rewards/rejected": -0.22383160889148712,
      "step": 8350
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5265505838209592e-06,
      "logits/chosen": -2.4518284797668457,
      "logits/rejected": -2.104112148284912,
      "logps/chosen": -272.073486328125,
      "logps/rejected": -236.34579467773438,
      "loss": 2270.6391,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15109841525554657,
      "rewards/margins": 0.10685638338327408,
      "rewards/rejected": -0.25795477628707886,
      "step": 8360
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.520840931240314e-06,
      "logits/chosen": -2.4796156883239746,
      "logits/rejected": -1.9970394372940063,
      "logps/chosen": -222.74423217773438,
      "logps/rejected": -175.6988067626953,
      "loss": 1455.9042,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1293562650680542,
      "rewards/margins": 0.1667545735836029,
      "rewards/rejected": -0.2961108386516571,
      "step": 8370
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.515131169943162e-06,
      "logits/chosen": -2.024977207183838,
      "logits/rejected": -2.111084461212158,
      "logps/chosen": -274.6491394042969,
      "logps/rejected": -285.61962890625,
      "loss": 1898.059,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16554591059684753,
      "rewards/margins": 0.1794281303882599,
      "rewards/rejected": -0.34497401118278503,
      "step": 8380
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.509421329714416e-06,
      "logits/chosen": -2.1542716026306152,
      "logits/rejected": -2.1845345497131348,
      "logps/chosen": -221.7804718017578,
      "logps/rejected": -252.6263885498047,
      "loss": 1956.1996,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.15641272068023682,
      "rewards/margins": 0.09653899818658829,
      "rewards/rejected": -0.2529517412185669,
      "step": 8390
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5037114403393987e-06,
      "logits/chosen": -2.2747371196746826,
      "logits/rejected": -2.02976655960083,
      "logps/chosen": -220.5301971435547,
      "logps/rejected": -201.79635620117188,
      "loss": 1652.7957,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09549812972545624,
      "rewards/margins": 0.1504668891429901,
      "rewards/rejected": -0.24596503376960754,
      "step": 8400
    },
    {
      "epoch": 0.55,
      "eval_logits/chosen": -2.3137805461883545,
      "eval_logits/rejected": -2.1298303604125977,
      "eval_logps/chosen": -247.04515075683594,
      "eval_logps/rejected": -241.276611328125,
      "eval_loss": 1804.0986328125,
      "eval_rewards/accuracies": 0.6460000276565552,
      "eval_rewards/chosen": -0.1504022628068924,
      "eval_rewards/margins": 0.14624497294425964,
      "eval_rewards/rejected": -0.29664725065231323,
      "eval_runtime": 722.8032,
      "eval_samples_per_second": 2.767,
      "eval_steps_per_second": 1.384,
      "step": 8400
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4980015316036908e-06,
      "logits/chosen": -2.160120964050293,
      "logits/rejected": -2.2130343914031982,
      "logps/chosen": -187.4803466796875,
      "logps/rejected": -232.2801055908203,
      "loss": 1326.5803,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11670643091201782,
      "rewards/margins": 0.23213955760002136,
      "rewards/rejected": -0.34884604811668396,
      "step": 8410
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4922916332929725e-06,
      "logits/chosen": -2.4907407760620117,
      "logits/rejected": -2.2238717079162598,
      "logps/chosen": -248.92971801757812,
      "logps/rejected": -221.5033721923828,
      "loss": 2002.0156,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13505470752716064,
      "rewards/margins": 0.12188147008419037,
      "rewards/rejected": -0.2569361925125122,
      "step": 8420
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4865817751928716e-06,
      "logits/chosen": -2.228563070297241,
      "logits/rejected": -2.216607093811035,
      "logps/chosen": -210.0905303955078,
      "logps/rejected": -256.52838134765625,
      "loss": 1457.9258,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.14601817727088928,
      "rewards/margins": 0.20015673339366913,
      "rewards/rejected": -0.3461748957633972,
      "step": 8430
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4808719870888037e-06,
      "logits/chosen": -2.0969958305358887,
      "logits/rejected": -2.0262608528137207,
      "logps/chosen": -233.91207885742188,
      "logps/rejected": -217.42391967773438,
      "loss": 1678.0998,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16240623593330383,
      "rewards/margins": 0.167855903506279,
      "rewards/rejected": -0.33026212453842163,
      "step": 8440
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4751622987658206e-06,
      "logits/chosen": -2.5014309883117676,
      "logits/rejected": -2.3213982582092285,
      "logps/chosen": -249.9745330810547,
      "logps/rejected": -253.134765625,
      "loss": 1850.9133,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12024291604757309,
      "rewards/margins": 0.13900864124298096,
      "rewards/rejected": -0.25925153493881226,
      "step": 8450
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4694527400084546e-06,
      "logits/chosen": -2.286902904510498,
      "logits/rejected": -2.1964993476867676,
      "logps/chosen": -235.66690063476562,
      "logps/rejected": -243.6966552734375,
      "loss": 1702.9672,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13422232866287231,
      "rewards/margins": 0.1387438029050827,
      "rewards/rejected": -0.2729661464691162,
      "step": 8460
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4637433406005607e-06,
      "logits/chosen": -2.4860024452209473,
      "logits/rejected": -2.370863437652588,
      "logps/chosen": -323.28936767578125,
      "logps/rejected": -309.1042175292969,
      "loss": 1832.9215,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13563047349452972,
      "rewards/margins": 0.13858014345169067,
      "rewards/rejected": -0.2742106318473816,
      "step": 8470
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4580341303251628e-06,
      "logits/chosen": -2.301816940307617,
      "logits/rejected": -2.037771701812744,
      "logps/chosen": -272.97430419921875,
      "logps/rejected": -257.2006530761719,
      "loss": 1560.2741,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13023129105567932,
      "rewards/margins": 0.20625126361846924,
      "rewards/rejected": -0.33648252487182617,
      "step": 8480
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4523251389642984e-06,
      "logits/chosen": -2.1981709003448486,
      "logits/rejected": -2.0751469135284424,
      "logps/chosen": -271.48138427734375,
      "logps/rejected": -254.32119750976562,
      "loss": 1673.4539,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.16922123730182648,
      "rewards/margins": 0.19978253543376923,
      "rewards/rejected": -0.3690037429332733,
      "step": 8490
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4466163962988626e-06,
      "logits/chosen": -2.5145349502563477,
      "logits/rejected": -2.145453691482544,
      "logps/chosen": -297.2919006347656,
      "logps/rejected": -214.80752563476562,
      "loss": 2091.1088,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1669091135263443,
      "rewards/margins": 0.1623493880033493,
      "rewards/rejected": -0.3292585015296936,
      "step": 8500
    },
    {
      "epoch": 0.56,
      "eval_logits/chosen": -2.3224895000457764,
      "eval_logits/rejected": -2.1371631622314453,
      "eval_logps/chosen": -247.69615173339844,
      "eval_logps/rejected": -243.36134338378906,
      "eval_loss": 1803.540771484375,
      "eval_rewards/accuracies": 0.6510000228881836,
      "eval_rewards/chosen": -0.15691210329532623,
      "eval_rewards/margins": 0.16058222949504852,
      "eval_rewards/rejected": -0.31749436259269714,
      "eval_runtime": 723.5499,
      "eval_samples_per_second": 2.764,
      "eval_steps_per_second": 1.382,
      "step": 8500
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4409079321084543e-06,
      "logits/chosen": -2.273221492767334,
      "logits/rejected": -2.324763298034668,
      "logps/chosen": -227.0233154296875,
      "logps/rejected": -272.7955017089844,
      "loss": 1877.4422,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11903639882802963,
      "rewards/margins": 0.1583327353000641,
      "rewards/rejected": -0.2773691713809967,
      "step": 8510
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4351997761712184e-06,
      "logits/chosen": -2.522839069366455,
      "logits/rejected": -2.0727713108062744,
      "logps/chosen": -257.5574645996094,
      "logps/rejected": -212.2241973876953,
      "loss": 1262.9665,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.12169145047664642,
      "rewards/margins": 0.2137262374162674,
      "rewards/rejected": -0.33541765809059143,
      "step": 8520
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4294919582636933e-06,
      "logits/chosen": -2.3109967708587646,
      "logits/rejected": -2.169250965118408,
      "logps/chosen": -223.28262329101562,
      "logps/rejected": -228.7783966064453,
      "loss": 1521.7472,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12180079519748688,
      "rewards/margins": 0.17335185408592224,
      "rewards/rejected": -0.29515260457992554,
      "step": 8530
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.423784508160652e-06,
      "logits/chosen": -2.3863461017608643,
      "logits/rejected": -2.138375759124756,
      "logps/chosen": -272.7332458496094,
      "logps/rejected": -240.0067138671875,
      "loss": 1791.726,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.17502503097057343,
      "rewards/margins": 0.15110823512077332,
      "rewards/rejected": -0.32613325119018555,
      "step": 8540
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.418077455634951e-06,
      "logits/chosen": -2.21634578704834,
      "logits/rejected": -2.2655911445617676,
      "logps/chosen": -227.6550750732422,
      "logps/rejected": -268.90667724609375,
      "loss": 1871.1775,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11083988845348358,
      "rewards/margins": 0.13694430887699127,
      "rewards/rejected": -0.24778418242931366,
      "step": 8550
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4123708304573714e-06,
      "logits/chosen": -2.3985753059387207,
      "logits/rejected": -2.2608468532562256,
      "logps/chosen": -299.736083984375,
      "logps/rejected": -301.4363098144531,
      "loss": 1622.1048,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09968741238117218,
      "rewards/margins": 0.18398933112621307,
      "rewards/rejected": -0.28367674350738525,
      "step": 8560
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.406664662396465e-06,
      "logits/chosen": -2.191650152206421,
      "logits/rejected": -2.0365116596221924,
      "logps/chosen": -202.6273651123047,
      "logps/rejected": -198.66024780273438,
      "loss": 2089.2441,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.18245482444763184,
      "rewards/margins": 0.11715099960565567,
      "rewards/rejected": -0.2996058464050293,
      "step": 8570
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4009589812184012e-06,
      "logits/chosen": -2.357086181640625,
      "logits/rejected": -1.9754444360733032,
      "logps/chosen": -217.845947265625,
      "logps/rejected": -177.6644287109375,
      "loss": 1696.61,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14626047015190125,
      "rewards/margins": 0.12929466366767883,
      "rewards/rejected": -0.2755551338195801,
      "step": 8580
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3952538166868073e-06,
      "logits/chosen": -2.1383872032165527,
      "logits/rejected": -2.198594093322754,
      "logps/chosen": -230.4532928466797,
      "logps/rejected": -234.652587890625,
      "loss": 1386.3909,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1290692389011383,
      "rewards/margins": 0.2079249918460846,
      "rewards/rejected": -0.3369942307472229,
      "step": 8590
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.389549198562616e-06,
      "logits/chosen": -2.3295209407806396,
      "logits/rejected": -1.9044430255889893,
      "logps/chosen": -239.19851684570312,
      "logps/rejected": -229.85903930664062,
      "loss": 1515.3847,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13116881251335144,
      "rewards/margins": 0.20640698075294495,
      "rewards/rejected": -0.337575763463974,
      "step": 8600
    },
    {
      "epoch": 0.56,
      "eval_logits/chosen": -2.3367018699645996,
      "eval_logits/rejected": -2.1502957344055176,
      "eval_logps/chosen": -244.80181884765625,
      "eval_logps/rejected": -240.16551208496094,
      "eval_loss": 1784.409423828125,
      "eval_rewards/accuracies": 0.656000018119812,
      "eval_rewards/chosen": -0.12796856462955475,
      "eval_rewards/margins": 0.15756747126579285,
      "eval_rewards/rejected": -0.2855360507965088,
      "eval_runtime": 724.6062,
      "eval_samples_per_second": 2.76,
      "eval_steps_per_second": 1.38,
      "step": 8600
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3838451566039098e-06,
      "logits/chosen": -2.3467819690704346,
      "logits/rejected": -2.1745972633361816,
      "logps/chosen": -256.1773376464844,
      "logps/rejected": -255.5081329345703,
      "loss": 2093.549,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17667360603809357,
      "rewards/margins": 0.09464854001998901,
      "rewards/rejected": -0.2713221311569214,
      "step": 8610
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3781417205657662e-06,
      "logits/chosen": -2.36430025100708,
      "logits/rejected": -2.0601418018341064,
      "logps/chosen": -210.05319213867188,
      "logps/rejected": -189.75921630859375,
      "loss": 1675.3691,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12296760082244873,
      "rewards/margins": 0.17913182079792023,
      "rewards/rejected": -0.30209943652153015,
      "step": 8620
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3724389202001006e-06,
      "logits/chosen": -2.393179178237915,
      "logits/rejected": -2.12575101852417,
      "logps/chosen": -219.45242309570312,
      "logps/rejected": -213.09487915039062,
      "loss": 1820.0014,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.17128977179527283,
      "rewards/margins": 0.15339143574237823,
      "rewards/rejected": -0.32468119263648987,
      "step": 8630
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.366736785255514e-06,
      "logits/chosen": -2.276599168777466,
      "logits/rejected": -2.2039132118225098,
      "logps/chosen": -214.40023803710938,
      "logps/rejected": -216.3034210205078,
      "loss": 1842.123,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.15671400725841522,
      "rewards/margins": 0.13123196363449097,
      "rewards/rejected": -0.2879459857940674,
      "step": 8640
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3610353454771355e-06,
      "logits/chosen": -2.183971881866455,
      "logits/rejected": -2.1115612983703613,
      "logps/chosen": -205.38037109375,
      "logps/rejected": -202.2388916015625,
      "loss": 2108.0105,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.15432950854301453,
      "rewards/margins": 0.13153082132339478,
      "rewards/rejected": -0.2858603298664093,
      "step": 8650
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.355334630606467e-06,
      "logits/chosen": -2.533132791519165,
      "logits/rejected": -2.052190065383911,
      "logps/chosen": -257.5996398925781,
      "logps/rejected": -207.3619842529297,
      "loss": 1800.7424,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1730933040380478,
      "rewards/margins": 0.13307951390743256,
      "rewards/rejected": -0.30617281794548035,
      "step": 8660
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.349634670381231e-06,
      "logits/chosen": -2.1400325298309326,
      "logits/rejected": -2.0968689918518066,
      "logps/chosen": -222.1932830810547,
      "logps/rejected": -245.26327514648438,
      "loss": 1916.9623,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.14994195103645325,
      "rewards/margins": 0.1327134072780609,
      "rewards/rejected": -0.2826553285121918,
      "step": 8670
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3439354945352104e-06,
      "logits/chosen": -2.387784719467163,
      "logits/rejected": -2.3289902210235596,
      "logps/chosen": -262.81951904296875,
      "logps/rejected": -225.038818359375,
      "loss": 2254.6959,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.19295933842658997,
      "rewards/margins": 0.06617043167352676,
      "rewards/rejected": -0.2591297924518585,
      "step": 8680
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3382371327981e-06,
      "logits/chosen": -2.2475638389587402,
      "logits/rejected": -2.2487053871154785,
      "logps/chosen": -244.71591186523438,
      "logps/rejected": -249.59848022460938,
      "loss": 1358.0566,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1323966681957245,
      "rewards/margins": 0.1917734146118164,
      "rewards/rejected": -0.3241700828075409,
      "step": 8690
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3325396148953456e-06,
      "logits/chosen": -2.1244850158691406,
      "logits/rejected": -2.2361392974853516,
      "logps/chosen": -189.0052490234375,
      "logps/rejected": -257.8404846191406,
      "loss": 1773.4947,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1770378202199936,
      "rewards/margins": 0.14704987406730652,
      "rewards/rejected": -0.3240876793861389,
      "step": 8700
    },
    {
      "epoch": 0.57,
      "eval_logits/chosen": -2.3439927101135254,
      "eval_logits/rejected": -2.1572177410125732,
      "eval_logps/chosen": -246.85609436035156,
      "eval_logps/rejected": -241.23651123046875,
      "eval_loss": 1781.3251953125,
      "eval_rewards/accuracies": 0.6539999842643738,
      "eval_rewards/chosen": -0.14851143956184387,
      "eval_rewards/margins": 0.14773471653461456,
      "eval_rewards/rejected": -0.29624614119529724,
      "eval_runtime": 724.3642,
      "eval_samples_per_second": 2.761,
      "eval_steps_per_second": 1.381,
      "step": 8700
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3268429705479915e-06,
      "logits/chosen": -2.5082974433898926,
      "logits/rejected": -2.1605987548828125,
      "logps/chosen": -236.01535034179688,
      "logps/rejected": -214.6260528564453,
      "loss": 1315.2974,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11202754825353622,
      "rewards/margins": 0.19929727911949158,
      "rewards/rejected": -0.3113248348236084,
      "step": 8710
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3211472294725248e-06,
      "logits/chosen": -2.3658642768859863,
      "logits/rejected": -2.23079252243042,
      "logps/chosen": -230.264892578125,
      "logps/rejected": -236.5784149169922,
      "loss": 1565.0962,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13849703967571259,
      "rewards/margins": 0.18980672955513,
      "rewards/rejected": -0.328303724527359,
      "step": 8720
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.315452421380721e-06,
      "logits/chosen": -2.249817132949829,
      "logits/rejected": -1.7914634943008423,
      "logps/chosen": -268.02972412109375,
      "logps/rejected": -234.94607543945312,
      "loss": 1925.0975,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13085412979125977,
      "rewards/margins": 0.14909395575523376,
      "rewards/rejected": -0.27994805574417114,
      "step": 8730
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3097585759794886e-06,
      "logits/chosen": -2.3426501750946045,
      "logits/rejected": -1.9522781372070312,
      "logps/chosen": -266.3086853027344,
      "logps/rejected": -224.4796905517578,
      "loss": 1628.2115,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11967410892248154,
      "rewards/margins": 0.21562986075878143,
      "rewards/rejected": -0.3353039622306824,
      "step": 8740
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3040657229707155e-06,
      "logits/chosen": -2.352038621902466,
      "logits/rejected": -2.247140884399414,
      "logps/chosen": -187.58377075195312,
      "logps/rejected": -216.38687133789062,
      "loss": 1605.9856,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15536166727542877,
      "rewards/margins": 0.17247450351715088,
      "rewards/rejected": -0.32783618569374084,
      "step": 8750
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.2983738920511104e-06,
      "logits/chosen": -2.513679027557373,
      "logits/rejected": -2.0306098461151123,
      "logps/chosen": -281.51922607421875,
      "logps/rejected": -248.924072265625,
      "loss": 1550.2564,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1172635406255722,
      "rewards/margins": 0.17706869542598724,
      "rewards/rejected": -0.29433223605155945,
      "step": 8760
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.2926831129120523e-06,
      "logits/chosen": -2.165912389755249,
      "logits/rejected": -2.1230220794677734,
      "logps/chosen": -249.427734375,
      "logps/rejected": -236.6632537841797,
      "loss": 1452.7133,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.13636116683483124,
      "rewards/margins": 0.1559121161699295,
      "rewards/rejected": -0.29227328300476074,
      "step": 8770
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.2869934152394323e-06,
      "logits/chosen": -2.3656163215637207,
      "logits/rejected": -2.095022201538086,
      "logps/chosen": -284.83721923828125,
      "logps/rejected": -240.811767578125,
      "loss": 2238.9471,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.15871794521808624,
      "rewards/margins": 0.11576612293720245,
      "rewards/rejected": -0.2744840681552887,
      "step": 8780
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.281304828713501e-06,
      "logits/chosen": -2.2470030784606934,
      "logits/rejected": -2.1870293617248535,
      "logps/chosen": -251.87753295898438,
      "logps/rejected": -257.03125,
      "loss": 1946.543,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.18890242278575897,
      "rewards/margins": 0.12656234204769135,
      "rewards/rejected": -0.3154647648334503,
      "step": 8790
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.275617383008711e-06,
      "logits/chosen": -2.341176986694336,
      "logits/rejected": -2.240370512008667,
      "logps/chosen": -250.6907501220703,
      "logps/rejected": -261.1102600097656,
      "loss": 1795.5312,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.15203282237052917,
      "rewards/margins": 0.13087281584739685,
      "rewards/rejected": -0.282905638217926,
      "step": 8800
    },
    {
      "epoch": 0.58,
      "eval_logits/chosen": -2.3469302654266357,
      "eval_logits/rejected": -2.1595752239227295,
      "eval_logps/chosen": -243.88812255859375,
      "eval_logps/rejected": -239.34646606445312,
      "eval_loss": 1785.9962158203125,
      "eval_rewards/accuracies": 0.6585000157356262,
      "eval_rewards/chosen": -0.11883172392845154,
      "eval_rewards/margins": 0.1585141122341156,
      "eval_rewards/rejected": -0.27734580636024475,
      "eval_runtime": 724.8827,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.38,
      "step": 8800
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.269931107793567e-06,
      "logits/chosen": -2.271245002746582,
      "logits/rejected": -2.180269718170166,
      "logps/chosen": -222.31216430664062,
      "logps/rejected": -243.32568359375,
      "loss": 1879.1926,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1003180518746376,
      "rewards/margins": 0.1292106956243515,
      "rewards/rejected": -0.2295287549495697,
      "step": 8810
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2642460327304655e-06,
      "logits/chosen": -2.1947274208068848,
      "logits/rejected": -2.2527339458465576,
      "logps/chosen": -255.667236328125,
      "logps/rejected": -253.3027801513672,
      "loss": 2138.826,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.11107480525970459,
      "rewards/margins": 0.1348305344581604,
      "rewards/rejected": -0.2459053248167038,
      "step": 8820
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.258562187475543e-06,
      "logits/chosen": -2.1851754188537598,
      "logits/rejected": -2.1546683311462402,
      "logps/chosen": -238.5567626953125,
      "logps/rejected": -212.98873901367188,
      "loss": 1958.4557,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11609867960214615,
      "rewards/margins": 0.11943832784891129,
      "rewards/rejected": -0.23553700745105743,
      "step": 8830
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2528796016785196e-06,
      "logits/chosen": -2.246023654937744,
      "logits/rejected": -2.0586771965026855,
      "logps/chosen": -204.19004821777344,
      "logps/rejected": -231.16177368164062,
      "loss": 1600.6956,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.14272116124629974,
      "rewards/margins": 0.17305660247802734,
      "rewards/rejected": -0.3157777786254883,
      "step": 8840
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.247198304982548e-06,
      "logits/chosen": -2.2854185104370117,
      "logits/rejected": -2.0803418159484863,
      "logps/chosen": -174.22653198242188,
      "logps/rejected": -184.52342224121094,
      "loss": 1828.7586,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1225501298904419,
      "rewards/margins": 0.1424633413553238,
      "rewards/rejected": -0.2650134861469269,
      "step": 8850
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2415183270240533e-06,
      "logits/chosen": -2.54646635055542,
      "logits/rejected": -2.302757501602173,
      "logps/chosen": -212.83834838867188,
      "logps/rejected": -233.5038604736328,
      "loss": 1681.2141,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1350872814655304,
      "rewards/margins": 0.16359901428222656,
      "rewards/rejected": -0.29868629574775696,
      "step": 8860
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2358396974325837e-06,
      "logits/chosen": -2.35323429107666,
      "logits/rejected": -2.1477296352386475,
      "logps/chosen": -250.175048828125,
      "logps/rejected": -240.62460327148438,
      "loss": 1567.8297,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07907464355230331,
      "rewards/margins": 0.16716596484184265,
      "rewards/rejected": -0.24624061584472656,
      "step": 8870
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2301624458306525e-06,
      "logits/chosen": -2.449899196624756,
      "logits/rejected": -2.160046339035034,
      "logps/chosen": -275.97906494140625,
      "logps/rejected": -235.339599609375,
      "loss": 2116.3758,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16630972921848297,
      "rewards/margins": 0.10701718181371689,
      "rewards/rejected": -0.27332693338394165,
      "step": 8880
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2244866018335855e-06,
      "logits/chosen": -2.305452823638916,
      "logits/rejected": -2.298046350479126,
      "logps/chosen": -229.87179565429688,
      "logps/rejected": -253.76742553710938,
      "loss": 2342.0385,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12814125418663025,
      "rewards/margins": 0.09488587826490402,
      "rewards/rejected": -0.22302713990211487,
      "step": 8890
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2188121950493648e-06,
      "logits/chosen": -2.44734263420105,
      "logits/rejected": -2.075795888900757,
      "logps/chosen": -233.55419921875,
      "logps/rejected": -167.12826538085938,
      "loss": 1880.782,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1481078565120697,
      "rewards/margins": 0.11826397478580475,
      "rewards/rejected": -0.26637178659439087,
      "step": 8900
    },
    {
      "epoch": 0.58,
      "eval_logits/chosen": -2.344513177871704,
      "eval_logits/rejected": -2.157212972640991,
      "eval_logps/chosen": -246.5005340576172,
      "eval_logps/rejected": -242.08053588867188,
      "eval_loss": 1782.6387939453125,
      "eval_rewards/accuracies": 0.6545000076293945,
      "eval_rewards/chosen": -0.14495587348937988,
      "eval_rewards/margins": 0.15973049402236938,
      "eval_rewards/rejected": -0.30468639731407166,
      "eval_runtime": 723.8534,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.381,
      "step": 8900
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2131392550784766e-06,
      "logits/chosen": -2.464691638946533,
      "logits/rejected": -1.890479326248169,
      "logps/chosen": -298.20574951171875,
      "logps/rejected": -222.00830078125,
      "loss": 1379.1506,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13327114284038544,
      "rewards/margins": 0.19366781413555145,
      "rewards/rejected": -0.3269389569759369,
      "step": 8910
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2074678115137533e-06,
      "logits/chosen": -2.1455368995666504,
      "logits/rejected": -2.0546436309814453,
      "logps/chosen": -208.8870391845703,
      "logps/rejected": -233.684814453125,
      "loss": 1581.221,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1417008638381958,
      "rewards/margins": 0.19881223142147064,
      "rewards/rejected": -0.34051311016082764,
      "step": 8920
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.201797893940224e-06,
      "logits/chosen": -2.2259609699249268,
      "logits/rejected": -2.015672445297241,
      "logps/chosen": -245.16067504882812,
      "logps/rejected": -282.9098815917969,
      "loss": 1605.6056,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11983288824558258,
      "rewards/margins": 0.1739863008260727,
      "rewards/rejected": -0.2938191592693329,
      "step": 8930
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.196129531934956e-06,
      "logits/chosen": -2.27424955368042,
      "logits/rejected": -2.0044965744018555,
      "logps/chosen": -248.17770385742188,
      "logps/rejected": -248.91513061523438,
      "loss": 1630.4396,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10263688862323761,
      "rewards/margins": 0.1879998743534088,
      "rewards/rejected": -0.29063680768013,
      "step": 8940
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.190462755066902e-06,
      "logits/chosen": -2.280273199081421,
      "logits/rejected": -2.0560171604156494,
      "logps/chosen": -280.7376403808594,
      "logps/rejected": -269.51123046875,
      "loss": 1606.276,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.183364599943161,
      "rewards/margins": 0.1426433026790619,
      "rewards/rejected": -0.3260079026222229,
      "step": 8950
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.184797592896746e-06,
      "logits/chosen": -2.4171805381774902,
      "logits/rejected": -2.382488250732422,
      "logps/chosen": -245.8582763671875,
      "logps/rejected": -236.25991821289062,
      "loss": 1587.3939,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11422693729400635,
      "rewards/margins": 0.16934755444526672,
      "rewards/rejected": -0.28357452154159546,
      "step": 8960
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.17913407497675e-06,
      "logits/chosen": -2.3496878147125244,
      "logits/rejected": -2.4273006916046143,
      "logps/chosen": -185.42147827148438,
      "logps/rejected": -237.0035400390625,
      "loss": 1905.2375,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0703950971364975,
      "rewards/margins": 0.1763085126876831,
      "rewards/rejected": -0.24670365452766418,
      "step": 8970
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.173472230850596e-06,
      "logits/chosen": -2.462353229522705,
      "logits/rejected": -2.274624824523926,
      "logps/chosen": -212.4112091064453,
      "logps/rejected": -182.86685180664062,
      "loss": 2017.8213,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.12148640304803848,
      "rewards/margins": 0.10899336636066437,
      "rewards/rejected": -0.23047976195812225,
      "step": 8980
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1678120900532375e-06,
      "logits/chosen": -2.44746732711792,
      "logits/rejected": -2.1482653617858887,
      "logps/chosen": -251.8532257080078,
      "logps/rejected": -239.1935577392578,
      "loss": 1699.1285,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1593838632106781,
      "rewards/margins": 0.16600628197193146,
      "rewards/rejected": -0.32539016008377075,
      "step": 8990
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1621536821107412e-06,
      "logits/chosen": -2.332921266555786,
      "logits/rejected": -2.205714702606201,
      "logps/chosen": -214.71414184570312,
      "logps/rejected": -189.07632446289062,
      "loss": 1695.9539,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12144309282302856,
      "rewards/margins": 0.1639450341463089,
      "rewards/rejected": -0.28538811206817627,
      "step": 9000
    },
    {
      "epoch": 0.59,
      "eval_logits/chosen": -2.348705768585205,
      "eval_logits/rejected": -2.161200761795044,
      "eval_logps/chosen": -243.29458618164062,
      "eval_logps/rejected": -239.52146911621094,
      "eval_loss": 1783.7203369140625,
      "eval_rewards/accuracies": 0.6625000238418579,
      "eval_rewards/chosen": -0.11289641261100769,
      "eval_rewards/margins": 0.16619925200939178,
      "eval_rewards/rejected": -0.27909567952156067,
      "eval_runtime": 723.4332,
      "eval_samples_per_second": 2.765,
      "eval_steps_per_second": 1.382,
      "step": 9000
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1564970365401346e-06,
      "logits/chosen": -2.3792998790740967,
      "logits/rejected": -2.1225550174713135,
      "logps/chosen": -200.08950805664062,
      "logps/rejected": -175.19876098632812,
      "loss": 2113.7773,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.15938423573970795,
      "rewards/margins": 0.13117334246635437,
      "rewards/rejected": -0.2905575633049011,
      "step": 9010
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1508421828492527e-06,
      "logits/chosen": -2.501051425933838,
      "logits/rejected": -2.158234119415283,
      "logps/chosen": -235.07876586914062,
      "logps/rejected": -181.7694854736328,
      "loss": 1914.55,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09886038303375244,
      "rewards/margins": 0.14931032061576843,
      "rewards/rejected": -0.24817068874835968,
      "step": 9020
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.145189150536582e-06,
      "logits/chosen": -2.177340030670166,
      "logits/rejected": -2.081105947494507,
      "logps/chosen": -232.1053924560547,
      "logps/rejected": -197.1642303466797,
      "loss": 1794.2258,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0993586927652359,
      "rewards/margins": 0.14461085200309753,
      "rewards/rejected": -0.24396955966949463,
      "step": 9030
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.139537969091107e-06,
      "logits/chosen": -2.2126095294952393,
      "logits/rejected": -2.1825172901153564,
      "logps/chosen": -277.8831481933594,
      "logps/rejected": -227.9888458251953,
      "loss": 2110.9107,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1229945570230484,
      "rewards/margins": 0.12276256084442139,
      "rewards/rejected": -0.2457571029663086,
      "step": 9040
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1338886679921603e-06,
      "logits/chosen": -2.280548572540283,
      "logits/rejected": -2.215135097503662,
      "logps/chosen": -252.32119750976562,
      "logps/rejected": -245.0319061279297,
      "loss": 1867.8156,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1077810525894165,
      "rewards/margins": 0.13121111690998077,
      "rewards/rejected": -0.23899218440055847,
      "step": 9050
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.128241276709263e-06,
      "logits/chosen": -2.361011028289795,
      "logits/rejected": -2.3018691539764404,
      "logps/chosen": -209.04067993164062,
      "logps/rejected": -245.54940795898438,
      "loss": 1783.9053,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.05778917670249939,
      "rewards/margins": 0.1647137701511383,
      "rewards/rejected": -0.2225029468536377,
      "step": 9060
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1225958247019746e-06,
      "logits/chosen": -2.389866590499878,
      "logits/rejected": -2.525109052658081,
      "logps/chosen": -201.82632446289062,
      "logps/rejected": -233.27877807617188,
      "loss": 1902.9713,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.14684496819972992,
      "rewards/margins": 0.11312825977802277,
      "rewards/rejected": -0.2599732279777527,
      "step": 9070
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1169523414197383e-06,
      "logits/chosen": -2.182159662246704,
      "logits/rejected": -2.1855149269104004,
      "logps/chosen": -210.26461791992188,
      "logps/rejected": -243.1604766845703,
      "loss": 2167.5078,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.11458927392959595,
      "rewards/margins": 0.095741868019104,
      "rewards/rejected": -0.21033115684986115,
      "step": 9080
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1113108563017267e-06,
      "logits/chosen": -2.2826504707336426,
      "logits/rejected": -2.0654351711273193,
      "logps/chosen": -228.85623168945312,
      "logps/rejected": -214.1038818359375,
      "loss": 1925.9301,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.16072991490364075,
      "rewards/margins": 0.1587684154510498,
      "rewards/rejected": -0.31949833035469055,
      "step": 9090
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.1056713987766905e-06,
      "logits/chosen": -2.473038673400879,
      "logits/rejected": -2.1360838413238525,
      "logps/chosen": -229.9833526611328,
      "logps/rejected": -195.61868286132812,
      "loss": 1709.6678,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.14783023297786713,
      "rewards/margins": 0.14350786805152893,
      "rewards/rejected": -0.29133808612823486,
      "step": 9100
    },
    {
      "epoch": 0.6,
      "eval_logits/chosen": -2.3422412872314453,
      "eval_logits/rejected": -2.155545234680176,
      "eval_logps/chosen": -245.23779296875,
      "eval_logps/rejected": -239.1156463623047,
      "eval_loss": 1782.341796875,
      "eval_rewards/accuracies": 0.6589999794960022,
      "eval_rewards/chosen": -0.1323283314704895,
      "eval_rewards/margins": 0.14270898699760437,
      "eval_rewards/rejected": -0.27503734827041626,
      "eval_runtime": 726.4537,
      "eval_samples_per_second": 2.753,
      "eval_steps_per_second": 1.377,
      "step": 9100
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.1000339982628022e-06,
      "logits/chosen": -2.147550106048584,
      "logits/rejected": -2.2366299629211426,
      "logps/chosen": -266.4583740234375,
      "logps/rejected": -243.4515838623047,
      "loss": 1900.083,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15185512602329254,
      "rewards/margins": 0.11313346773386002,
      "rewards/rejected": -0.26498860120773315,
      "step": 9110
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0943986841675043e-06,
      "logits/chosen": -2.3581080436706543,
      "logits/rejected": -2.1352851390838623,
      "logps/chosen": -214.7187042236328,
      "logps/rejected": -210.26034545898438,
      "loss": 1624.3727,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1150769367814064,
      "rewards/margins": 0.14881470799446106,
      "rewards/rejected": -0.26389163732528687,
      "step": 9120
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.088765485887356e-06,
      "logits/chosen": -2.340144634246826,
      "logits/rejected": -2.1474156379699707,
      "logps/chosen": -257.22198486328125,
      "logps/rejected": -228.04470825195312,
      "loss": 1848.6959,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12507091462612152,
      "rewards/margins": 0.1244811862707138,
      "rewards/rejected": -0.24955210089683533,
      "step": 9130
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.083134432807879e-06,
      "logits/chosen": -2.263231039047241,
      "logits/rejected": -2.2002289295196533,
      "logps/chosen": -207.6758575439453,
      "logps/rejected": -246.3755645751953,
      "loss": 1857.1973,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1340373456478119,
      "rewards/margins": 0.1718166023492813,
      "rewards/rejected": -0.305853933095932,
      "step": 9140
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.077505554303404e-06,
      "logits/chosen": -2.3470020294189453,
      "logits/rejected": -2.311870813369751,
      "logps/chosen": -184.0131072998047,
      "logps/rejected": -202.27972412109375,
      "loss": 1668.6959,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09461130201816559,
      "rewards/margins": 0.13899998366832733,
      "rewards/rejected": -0.2336113005876541,
      "step": 9150
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.071878879736918e-06,
      "logits/chosen": -2.3313071727752686,
      "logits/rejected": -2.1438777446746826,
      "logps/chosen": -262.45343017578125,
      "logps/rejected": -354.03533935546875,
      "loss": 1832.6357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15613846480846405,
      "rewards/margins": 0.13298094272613525,
      "rewards/rejected": -0.2891194224357605,
      "step": 9160
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0662544384599136e-06,
      "logits/chosen": -2.2310731410980225,
      "logits/rejected": -2.1646816730499268,
      "logps/chosen": -212.7046356201172,
      "logps/rejected": -213.3751220703125,
      "loss": 1754.624,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10598021745681763,
      "rewards/margins": 0.17155803740024567,
      "rewards/rejected": -0.2775382399559021,
      "step": 9170
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0606322598122314e-06,
      "logits/chosen": -2.2409818172454834,
      "logits/rejected": -2.3552908897399902,
      "logps/chosen": -204.3204345703125,
      "logps/rejected": -232.7908935546875,
      "loss": 2190.9982,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.17806866765022278,
      "rewards/margins": 0.0880960077047348,
      "rewards/rejected": -0.2661646902561188,
      "step": 9180
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0550123731219085e-06,
      "logits/chosen": -2.5051026344299316,
      "logits/rejected": -2.342766284942627,
      "logps/chosen": -265.1522216796875,
      "logps/rejected": -239.1829071044922,
      "loss": 1589.7291,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11972393840551376,
      "rewards/margins": 0.15175560116767883,
      "rewards/rejected": -0.2714795470237732,
      "step": 9190
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0493948077050267e-06,
      "logits/chosen": -2.1906094551086426,
      "logits/rejected": -2.009308338165283,
      "logps/chosen": -208.6791534423828,
      "logps/rejected": -205.8546905517578,
      "loss": 1829.7031,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1542314738035202,
      "rewards/margins": 0.15265150368213654,
      "rewards/rejected": -0.30688297748565674,
      "step": 9200
    },
    {
      "epoch": 0.6,
      "eval_logits/chosen": -2.3467259407043457,
      "eval_logits/rejected": -2.1592679023742676,
      "eval_logps/chosen": -245.86827087402344,
      "eval_logps/rejected": -240.8571319580078,
      "eval_loss": 1774.454833984375,
      "eval_rewards/accuracies": 0.6610000133514404,
      "eval_rewards/chosen": -0.13863316178321838,
      "eval_rewards/margins": 0.15381911396980286,
      "eval_rewards/rejected": -0.29245227575302124,
      "eval_runtime": 723.8065,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.382,
      "step": 9200
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0437795928655596e-06,
      "logits/chosen": -2.3635621070861816,
      "logits/rejected": -2.3407657146453857,
      "logps/chosen": -296.04876708984375,
      "logps/rejected": -291.05877685546875,
      "loss": 1739.4965,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1433614194393158,
      "rewards/margins": 0.1319374144077301,
      "rewards/rejected": -0.2752988338470459,
      "step": 9210
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0381667578952184e-06,
      "logits/chosen": -2.4439473152160645,
      "logits/rejected": -2.2242536544799805,
      "logps/chosen": -229.7306671142578,
      "logps/rejected": -251.9036865234375,
      "loss": 1798.8371,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.18857814371585846,
      "rewards/margins": 0.1655600517988205,
      "rewards/rejected": -0.35413819551467896,
      "step": 9220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0325563320732995e-06,
      "logits/chosen": -2.5141654014587402,
      "logits/rejected": -2.1215217113494873,
      "logps/chosen": -277.11749267578125,
      "logps/rejected": -250.9359893798828,
      "loss": 1667.3076,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13779176771640778,
      "rewards/margins": 0.17836323380470276,
      "rewards/rejected": -0.31615501642227173,
      "step": 9230
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.026948344666532e-06,
      "logits/chosen": -2.258357048034668,
      "logits/rejected": -2.2154464721679688,
      "logps/chosen": -216.21041870117188,
      "logps/rejected": -231.43017578125,
      "loss": 1620.7397,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1708449274301529,
      "rewards/margins": 0.1566438525915146,
      "rewards/rejected": -0.3274887800216675,
      "step": 9240
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.0213428249289257e-06,
      "logits/chosen": -2.2519164085388184,
      "logits/rejected": -2.161630153656006,
      "logps/chosen": -209.21115112304688,
      "logps/rejected": -229.04519653320312,
      "loss": 1425.7735,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1067061796784401,
      "rewards/margins": 0.1802651435136795,
      "rewards/rejected": -0.2869713604450226,
      "step": 9250
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.0157398021016175e-06,
      "logits/chosen": -2.2583141326904297,
      "logits/rejected": -2.2162694931030273,
      "logps/chosen": -161.82420349121094,
      "logps/rejected": -218.95913696289062,
      "loss": 2075.3656,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.14288866519927979,
      "rewards/margins": 0.10930158197879791,
      "rewards/rejected": -0.2521902620792389,
      "step": 9260
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.010139305412719e-06,
      "logits/chosen": -2.508657455444336,
      "logits/rejected": -2.2943289279937744,
      "logps/chosen": -291.4534606933594,
      "logps/rejected": -265.9161376953125,
      "loss": 1707.8695,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.14857791364192963,
      "rewards/margins": 0.1682676374912262,
      "rewards/rejected": -0.316845566034317,
      "step": 9270
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.0045413640771644e-06,
      "logits/chosen": -2.2465755939483643,
      "logits/rejected": -2.4008095264434814,
      "logps/chosen": -269.99713134765625,
      "logps/rejected": -290.6904602050781,
      "loss": 1703.0684,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1483180820941925,
      "rewards/margins": 0.1743774116039276,
      "rewards/rejected": -0.32269546389579773,
      "step": 9280
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.998946007296558e-06,
      "logits/chosen": -2.485224962234497,
      "logits/rejected": -2.1421313285827637,
      "logps/chosen": -324.39080810546875,
      "logps/rejected": -276.89190673828125,
      "loss": 1705.784,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.157902330160141,
      "rewards/margins": 0.1593799889087677,
      "rewards/rejected": -0.3172822892665863,
      "step": 9290
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9933532642590215e-06,
      "logits/chosen": -2.228764057159424,
      "logits/rejected": -1.855963110923767,
      "logps/chosen": -197.72039794921875,
      "logps/rejected": -165.6393585205078,
      "loss": 1540.8942,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08562113344669342,
      "rewards/margins": 0.18079780042171478,
      "rewards/rejected": -0.2664188742637634,
      "step": 9300
    },
    {
      "epoch": 0.61,
      "eval_logits/chosen": -2.3514695167541504,
      "eval_logits/rejected": -2.1634879112243652,
      "eval_logps/chosen": -248.19981384277344,
      "eval_logps/rejected": -243.58885192871094,
      "eval_loss": 1782.968505859375,
      "eval_rewards/accuracies": 0.6570000052452087,
      "eval_rewards/chosen": -0.16194862127304077,
      "eval_rewards/margins": 0.1578209102153778,
      "eval_rewards/rejected": -0.3197695314884186,
      "eval_runtime": 726.8341,
      "eval_samples_per_second": 2.752,
      "eval_steps_per_second": 1.376,
      "step": 9300
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.987763164139042e-06,
      "logits/chosen": -2.383131742477417,
      "logits/rejected": -2.182943105697632,
      "logps/chosen": -220.27670288085938,
      "logps/rejected": -235.8668670654297,
      "loss": 1990.3887,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.15021634101867676,
      "rewards/margins": 0.1333579570055008,
      "rewards/rejected": -0.28357431292533875,
      "step": 9310
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.982175736097321e-06,
      "logits/chosen": -2.0686111450195312,
      "logits/rejected": -2.0829806327819824,
      "logps/chosen": -297.60162353515625,
      "logps/rejected": -323.6811218261719,
      "loss": 2291.2773,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.23060569167137146,
      "rewards/margins": 0.09059707075357437,
      "rewards/rejected": -0.32120275497436523,
      "step": 9320
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9765910092806196e-06,
      "logits/chosen": -2.271430015563965,
      "logits/rejected": -2.1739978790283203,
      "logps/chosen": -190.2344970703125,
      "logps/rejected": -182.0806884765625,
      "loss": 1859.2246,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12158989906311035,
      "rewards/margins": 0.11504407227039337,
      "rewards/rejected": -0.23663397133350372,
      "step": 9330
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9710090128216083e-06,
      "logits/chosen": -2.325185537338257,
      "logits/rejected": -2.2322769165039062,
      "logps/chosen": -241.9593505859375,
      "logps/rejected": -244.9901123046875,
      "loss": 1752.9023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19229312241077423,
      "rewards/margins": 0.19193877279758453,
      "rewards/rejected": -0.38423192501068115,
      "step": 9340
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9654297758387155e-06,
      "logits/chosen": -2.1726956367492676,
      "logits/rejected": -2.1545259952545166,
      "logps/chosen": -185.04246520996094,
      "logps/rejected": -219.32510375976562,
      "loss": 1810.7492,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.24172444641590118,
      "rewards/margins": 0.13557168841362,
      "rewards/rejected": -0.3772961497306824,
      "step": 9350
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9598533274359736e-06,
      "logits/chosen": -2.335035800933838,
      "logits/rejected": -2.242264986038208,
      "logps/chosen": -266.0108947753906,
      "logps/rejected": -279.05572509765625,
      "loss": 2427.3982,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2633058726787567,
      "rewards/margins": 0.056832779198884964,
      "rewards/rejected": -0.3201386630535126,
      "step": 9360
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9542796967028697e-06,
      "logits/chosen": -2.3382534980773926,
      "logits/rejected": -2.2298545837402344,
      "logps/chosen": -241.0985107421875,
      "logps/rejected": -235.7290496826172,
      "loss": 1784.1752,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.22857961058616638,
      "rewards/margins": 0.12873327732086182,
      "rewards/rejected": -0.3573128581047058,
      "step": 9370
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.948708912714192e-06,
      "logits/chosen": -2.254430055618286,
      "logits/rejected": -2.011399030685425,
      "logps/chosen": -270.86187744140625,
      "logps/rejected": -250.81558227539062,
      "loss": 2033.2949,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.23916606605052948,
      "rewards/margins": 0.10889426618814468,
      "rewards/rejected": -0.3480603098869324,
      "step": 9380
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9431410045298786e-06,
      "logits/chosen": -2.1085152626037598,
      "logits/rejected": -2.037166118621826,
      "logps/chosen": -237.0220184326172,
      "logps/rejected": -249.94583129882812,
      "loss": 1782.4275,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.19896671175956726,
      "rewards/margins": 0.13899557292461395,
      "rewards/rejected": -0.33796226978302,
      "step": 9390
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9375760011948654e-06,
      "logits/chosen": -2.4289746284484863,
      "logits/rejected": -2.2810416221618652,
      "logps/chosen": -217.75021362304688,
      "logps/rejected": -258.0467224121094,
      "loss": 1477.422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1807229220867157,
      "rewards/margins": 0.17976048588752747,
      "rewards/rejected": -0.36048340797424316,
      "step": 9400
    },
    {
      "epoch": 0.62,
      "eval_logits/chosen": -2.345621109008789,
      "eval_logits/rejected": -2.158308744430542,
      "eval_logps/chosen": -252.24818420410156,
      "eval_logps/rejected": -246.08737182617188,
      "eval_loss": 1785.5960693359375,
      "eval_rewards/accuracies": 0.6524999737739563,
      "eval_rewards/chosen": -0.20243246853351593,
      "eval_rewards/margins": 0.14232242107391357,
      "eval_rewards/rejected": -0.3447548449039459,
      "eval_runtime": 722.7813,
      "eval_samples_per_second": 2.767,
      "eval_steps_per_second": 1.384,
      "step": 9400
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.932013931738937e-06,
      "logits/chosen": -2.327096462249756,
      "logits/rejected": -2.1064553260803223,
      "logps/chosen": -223.3422088623047,
      "logps/rejected": -259.3307189941406,
      "loss": 1314.2292,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.1974048614501953,
      "rewards/margins": 0.23106737434864044,
      "rewards/rejected": -0.42847222089767456,
      "step": 9410
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9264548251765717e-06,
      "logits/chosen": -2.4367785453796387,
      "logits/rejected": -2.251880168914795,
      "logps/chosen": -218.90817260742188,
      "logps/rejected": -235.78543090820312,
      "loss": 1347.5708,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15105506777763367,
      "rewards/margins": 0.19989021122455597,
      "rewards/rejected": -0.35094529390335083,
      "step": 9420
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9208987105067924e-06,
      "logits/chosen": -2.2413735389709473,
      "logits/rejected": -2.1048600673675537,
      "logps/chosen": -234.02755737304688,
      "logps/rejected": -220.658935546875,
      "loss": 2059.2354,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.19470515847206116,
      "rewards/margins": 0.10519008338451385,
      "rewards/rejected": -0.2998952269554138,
      "step": 9430
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9153456167130154e-06,
      "logits/chosen": -2.3501267433166504,
      "logits/rejected": -2.341381311416626,
      "logps/chosen": -225.01980590820312,
      "logps/rejected": -263.4544372558594,
      "loss": 1804.2916,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.20949192345142365,
      "rewards/margins": 0.12235965579748154,
      "rewards/rejected": -0.331851601600647,
      "step": 9440
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9097955727628975e-06,
      "logits/chosen": -2.3747715950012207,
      "logits/rejected": -2.37276029586792,
      "logps/chosen": -212.94384765625,
      "logps/rejected": -238.12759399414062,
      "loss": 1802.3164,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16958339512348175,
      "rewards/margins": 0.13248775899410248,
      "rewards/rejected": -0.3020711839199066,
      "step": 9450
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.904248607608187e-06,
      "logits/chosen": -2.284837007522583,
      "logits/rejected": -2.315736770629883,
      "logps/chosen": -272.3593444824219,
      "logps/rejected": -240.7972869873047,
      "loss": 1520.689,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15139371156692505,
      "rewards/margins": 0.15544667840003967,
      "rewards/rejected": -0.3068404197692871,
      "step": 9460
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8987047501845714e-06,
      "logits/chosen": -2.353510618209839,
      "logits/rejected": -2.3260111808776855,
      "logps/chosen": -183.2423858642578,
      "logps/rejected": -195.4254608154297,
      "loss": 1639.7561,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16317906975746155,
      "rewards/margins": 0.18428602814674377,
      "rewards/rejected": -0.3474651277065277,
      "step": 9470
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8931640294115267e-06,
      "logits/chosen": -2.1589865684509277,
      "logits/rejected": -2.04636549949646,
      "logps/chosen": -212.6373748779297,
      "logps/rejected": -215.263671875,
      "loss": 1550.8488,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.18631470203399658,
      "rewards/margins": 0.18156476318836212,
      "rewards/rejected": -0.3678794503211975,
      "step": 9480
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8876264741921662e-06,
      "logits/chosen": -2.121523380279541,
      "logits/rejected": -2.143402576446533,
      "logps/chosen": -208.8270721435547,
      "logps/rejected": -221.06982421875,
      "loss": 1578.1198,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1870156228542328,
      "rewards/margins": 0.17678289115428925,
      "rewards/rejected": -0.3637985587120056,
      "step": 9490
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8820921134130912e-06,
      "logits/chosen": -2.3421788215637207,
      "logits/rejected": -1.9765698909759521,
      "logps/chosen": -248.6728057861328,
      "logps/rejected": -224.97183227539062,
      "loss": 1495.3285,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.18160656094551086,
      "rewards/margins": 0.21208277344703674,
      "rewards/rejected": -0.3936893343925476,
      "step": 9500
    },
    {
      "epoch": 0.62,
      "eval_logits/chosen": -2.3461034297943115,
      "eval_logits/rejected": -2.1586921215057373,
      "eval_logps/chosen": -253.2010498046875,
      "eval_logps/rejected": -247.79739379882812,
      "eval_loss": 1796.2745361328125,
      "eval_rewards/accuracies": 0.6510000228881836,
      "eval_rewards/chosen": -0.21196097135543823,
      "eval_rewards/margins": 0.14989392459392548,
      "eval_rewards/rejected": -0.3618549108505249,
      "eval_runtime": 724.698,
      "eval_samples_per_second": 2.76,
      "eval_steps_per_second": 1.38,
      "step": 9500
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8765609759442378e-06,
      "logits/chosen": -2.2057645320892334,
      "logits/rejected": -2.0962395668029785,
      "logps/chosen": -263.9498291015625,
      "logps/rejected": -265.99652099609375,
      "loss": 1741.1148,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.22628538310527802,
      "rewards/margins": 0.12858828902244568,
      "rewards/rejected": -0.3548737168312073,
      "step": 9510
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8710330906387288e-06,
      "logits/chosen": -2.3963239192962646,
      "logits/rejected": -2.344132900238037,
      "logps/chosen": -263.30889892578125,
      "logps/rejected": -298.1190185546875,
      "loss": 1954.2197,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2482127696275711,
      "rewards/margins": 0.11220197379589081,
      "rewards/rejected": -0.3604147434234619,
      "step": 9520
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8655084863327222e-06,
      "logits/chosen": -2.3089566230773926,
      "logits/rejected": -2.33139705657959,
      "logps/chosen": -202.14541625976562,
      "logps/rejected": -216.45089721679688,
      "loss": 2188.0619,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.1868743598461151,
      "rewards/margins": 0.08143356442451477,
      "rewards/rejected": -0.26830795407295227,
      "step": 9530
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8599871918452603e-06,
      "logits/chosen": -2.1625704765319824,
      "logits/rejected": -2.15848970413208,
      "logps/chosen": -240.3632049560547,
      "logps/rejected": -269.21112060546875,
      "loss": 1604.9603,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.17271386086940765,
      "rewards/margins": 0.155984565615654,
      "rewards/rejected": -0.32869842648506165,
      "step": 9540
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8544692359781192e-06,
      "logits/chosen": -2.36810040473938,
      "logits/rejected": -2.133775234222412,
      "logps/chosen": -202.4742889404297,
      "logps/rejected": -189.27862548828125,
      "loss": 1458.3448,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14889639616012573,
      "rewards/margins": 0.17751574516296387,
      "rewards/rejected": -0.3264121413230896,
      "step": 9550
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8489546475156602e-06,
      "logits/chosen": -2.518655300140381,
      "logits/rejected": -2.2955431938171387,
      "logps/chosen": -238.36276245117188,
      "logps/rejected": -234.1089630126953,
      "loss": 1833.2234,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18869927525520325,
      "rewards/margins": 0.13914178311824799,
      "rewards/rejected": -0.32784101366996765,
      "step": 9560
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8434434552246778e-06,
      "logits/chosen": -2.1662681102752686,
      "logits/rejected": -2.0950560569763184,
      "logps/chosen": -221.53689575195312,
      "logps/rejected": -230.15853881835938,
      "loss": 1530.9552,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11905352771282196,
      "rewards/margins": 0.19884634017944336,
      "rewards/rejected": -0.3178998529911041,
      "step": 9570
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.837935687854251e-06,
      "logits/chosen": -2.3633503913879395,
      "logits/rejected": -2.1214184761047363,
      "logps/chosen": -236.35519409179688,
      "logps/rejected": -226.7471466064453,
      "loss": 1417.6899,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.14808468520641327,
      "rewards/margins": 0.2219206541776657,
      "rewards/rejected": -0.3700053095817566,
      "step": 9580
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.832431374135592e-06,
      "logits/chosen": -2.4873459339141846,
      "logits/rejected": -2.0834736824035645,
      "logps/chosen": -266.6532287597656,
      "logps/rejected": -264.7237243652344,
      "loss": 1751.1215,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1821785420179367,
      "rewards/margins": 0.1694364994764328,
      "rewards/rejected": -0.3516150414943695,
      "step": 9590
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8269305427818977e-06,
      "logits/chosen": -2.4908695220947266,
      "logits/rejected": -2.3433737754821777,
      "logps/chosen": -222.4200439453125,
      "logps/rejected": -211.399658203125,
      "loss": 1647.9816,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13803806900978088,
      "rewards/margins": 0.16643409430980682,
      "rewards/rejected": -0.3044721782207489,
      "step": 9600
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -2.353161573410034,
      "eval_logits/rejected": -2.165470600128174,
      "eval_logps/chosen": -248.5635223388672,
      "eval_logps/rejected": -244.8505401611328,
      "eval_loss": 1805.82275390625,
      "eval_rewards/accuracies": 0.6539999842643738,
      "eval_rewards/chosen": -0.1655854731798172,
      "eval_rewards/margins": 0.16680093109607697,
      "eval_rewards/rejected": -0.33238640427589417,
      "eval_runtime": 724.1311,
      "eval_samples_per_second": 2.762,
      "eval_steps_per_second": 1.381,
      "step": 9600
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.821433222488199e-06,
      "logits/chosen": -2.3555164337158203,
      "logits/rejected": -2.039760112762451,
      "logps/chosen": -238.9461212158203,
      "logps/rejected": -227.8724365234375,
      "loss": 1930.6977,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1333135962486267,
      "rewards/margins": 0.166739359498024,
      "rewards/rejected": -0.3000529408454895,
      "step": 9610
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8159394419312112e-06,
      "logits/chosen": -2.3798041343688965,
      "logits/rejected": -2.197470188140869,
      "logps/chosen": -272.6890869140625,
      "logps/rejected": -249.67861938476562,
      "loss": 1383.2759,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.1395907700061798,
      "rewards/margins": 0.23528221249580383,
      "rewards/rejected": -0.37487298250198364,
      "step": 9620
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8104492297691845e-06,
      "logits/chosen": -2.3357458114624023,
      "logits/rejected": -2.1194615364074707,
      "logps/chosen": -244.08535766601562,
      "logps/rejected": -236.9788360595703,
      "loss": 1766.9283,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1756221503019333,
      "rewards/margins": 0.17006057500839233,
      "rewards/rejected": -0.3456827402114868,
      "step": 9630
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8049626146417562e-06,
      "logits/chosen": -2.1383423805236816,
      "logits/rejected": -1.9963083267211914,
      "logps/chosen": -169.82345581054688,
      "logps/rejected": -178.7533721923828,
      "loss": 2164.8064,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.14747557044029236,
      "rewards/margins": 0.1264285296201706,
      "rewards/rejected": -0.27390408515930176,
      "step": 9640
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7994796251697983e-06,
      "logits/chosen": -2.241729497909546,
      "logits/rejected": -2.105792284011841,
      "logps/chosen": -210.55819702148438,
      "logps/rejected": -267.66064453125,
      "loss": 1514.0446,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1851930171251297,
      "rewards/margins": 0.17537666857242584,
      "rewards/rejected": -0.36056965589523315,
      "step": 9650
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.794000289955269e-06,
      "logits/chosen": -2.283034563064575,
      "logits/rejected": -2.115142345428467,
      "logps/chosen": -287.27728271484375,
      "logps/rejected": -270.07666015625,
      "loss": 2013.5223,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20217004418373108,
      "rewards/margins": 0.14246216416358948,
      "rewards/rejected": -0.34463220834732056,
      "step": 9660
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7885246375810646e-06,
      "logits/chosen": -2.2674050331115723,
      "logits/rejected": -1.9900856018066406,
      "logps/chosen": -237.94100952148438,
      "logps/rejected": -250.2483367919922,
      "loss": 1496.8723,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13667115569114685,
      "rewards/margins": 0.1606934368610382,
      "rewards/rejected": -0.29736456274986267,
      "step": 9670
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7830526966108713e-06,
      "logits/chosen": -2.1287639141082764,
      "logits/rejected": -1.9397249221801758,
      "logps/chosen": -206.8074188232422,
      "logps/rejected": -198.19400024414062,
      "loss": 1426.6112,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2033451497554779,
      "rewards/margins": 0.19998101890087128,
      "rewards/rejected": -0.4033261835575104,
      "step": 9680
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7775844955890129e-06,
      "logits/chosen": -2.282334089279175,
      "logits/rejected": -2.120537757873535,
      "logps/chosen": -224.4355010986328,
      "logps/rejected": -235.31326293945312,
      "loss": 1387.2142,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12889166176319122,
      "rewards/margins": 0.22454425692558289,
      "rewards/rejected": -0.3534359335899353,
      "step": 9690
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7721200630403046e-06,
      "logits/chosen": -2.3532631397247314,
      "logits/rejected": -2.1655144691467285,
      "logps/chosen": -212.76953125,
      "logps/rejected": -249.0178680419922,
      "loss": 2050.0473,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.17007184028625488,
      "rewards/margins": 0.10702402889728546,
      "rewards/rejected": -0.27709585428237915,
      "step": 9700
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -2.3500943183898926,
      "eval_logits/rejected": -2.1625893115997314,
      "eval_logps/chosen": -249.26937866210938,
      "eval_logps/rejected": -244.1856231689453,
      "eval_loss": 1781.771728515625,
      "eval_rewards/accuracies": 0.6585000157356262,
      "eval_rewards/chosen": -0.17264403402805328,
      "eval_rewards/margins": 0.15309324860572815,
      "eval_rewards/rejected": -0.32573723793029785,
      "eval_runtime": 725.1374,
      "eval_samples_per_second": 2.758,
      "eval_steps_per_second": 1.379,
      "step": 9700
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7666594274699037e-06,
      "logits/chosen": -2.2733945846557617,
      "logits/rejected": -2.125950336456299,
      "logps/chosen": -268.30718994140625,
      "logps/rejected": -250.86489868164062,
      "loss": 1467.0482,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15616917610168457,
      "rewards/margins": 0.20761807262897491,
      "rewards/rejected": -0.3637872636318207,
      "step": 9710
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.76120261736316e-06,
      "logits/chosen": -2.3260655403137207,
      "logits/rejected": -1.9596723318099976,
      "logps/chosen": -245.9228057861328,
      "logps/rejected": -236.8369598388672,
      "loss": 1604.5218,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.19207733869552612,
      "rewards/margins": 0.1805742084980011,
      "rewards/rejected": -0.37265151739120483,
      "step": 9720
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.755749661185468e-06,
      "logits/chosen": -2.3855628967285156,
      "logits/rejected": -1.958176851272583,
      "logps/chosen": -308.1893615722656,
      "logps/rejected": -268.2945251464844,
      "loss": 1685.2773,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13739819824695587,
      "rewards/margins": 0.1893884837627411,
      "rewards/rejected": -0.32678669691085815,
      "step": 9730
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7503005873821183e-06,
      "logits/chosen": -2.33542799949646,
      "logits/rejected": -2.2756717205047607,
      "logps/chosen": -175.3501434326172,
      "logps/rejected": -215.01730346679688,
      "loss": 1826.0061,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1492181122303009,
      "rewards/margins": 0.13403229415416718,
      "rewards/rejected": -0.2832503914833069,
      "step": 9740
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.744855424378148e-06,
      "logits/chosen": -2.1913769245147705,
      "logits/rejected": -2.2056326866149902,
      "logps/chosen": -199.3675994873047,
      "logps/rejected": -243.65487670898438,
      "loss": 1348.4872,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10740645229816437,
      "rewards/margins": 0.20841753482818604,
      "rewards/rejected": -0.3158240020275116,
      "step": 9750
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7394142005781973e-06,
      "logits/chosen": -2.1301426887512207,
      "logits/rejected": -2.220323085784912,
      "logps/chosen": -277.7008972167969,
      "logps/rejected": -299.0531005859375,
      "loss": 1575.9425,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.159898579120636,
      "rewards/margins": 0.1778123378753662,
      "rewards/rejected": -0.3377109467983246,
      "step": 9760
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7339769443663528e-06,
      "logits/chosen": -2.3279261589050293,
      "logits/rejected": -2.1848256587982178,
      "logps/chosen": -161.63812255859375,
      "logps/rejected": -174.63168334960938,
      "loss": 1832.999,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1869591474533081,
      "rewards/margins": 0.12290282547473907,
      "rewards/rejected": -0.309861958026886,
      "step": 9770
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7285436841060078e-06,
      "logits/chosen": -2.510305404663086,
      "logits/rejected": -2.224672317504883,
      "logps/chosen": -288.49371337890625,
      "logps/rejected": -264.77435302734375,
      "loss": 1402.2434,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.12365376949310303,
      "rewards/margins": 0.19234833121299744,
      "rewards/rejected": -0.31600213050842285,
      "step": 9780
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7231144481397083e-06,
      "logits/chosen": -2.3978075981140137,
      "logits/rejected": -2.3237767219543457,
      "logps/chosen": -233.5013427734375,
      "logps/rejected": -224.5458984375,
      "loss": 1697.8324,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13102979958057404,
      "rewards/margins": 0.16483241319656372,
      "rewards/rejected": -0.29586222767829895,
      "step": 9790
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7176892647890092e-06,
      "logits/chosen": -2.4705073833465576,
      "logits/rejected": -2.2312042713165283,
      "logps/chosen": -252.5708770751953,
      "logps/rejected": -227.5338134765625,
      "loss": 1848.6725,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16904866695404053,
      "rewards/margins": 0.13844363391399384,
      "rewards/rejected": -0.30749231576919556,
      "step": 9800
    },
    {
      "epoch": 0.64,
      "eval_logits/chosen": -2.343329668045044,
      "eval_logits/rejected": -2.156418561935425,
      "eval_logps/chosen": -247.46359252929688,
      "eval_logps/rejected": -243.70355224609375,
      "eval_loss": 1796.389404296875,
      "eval_rewards/accuracies": 0.6610000133514404,
      "eval_rewards/chosen": -0.1545863300561905,
      "eval_rewards/margins": 0.16632995009422302,
      "eval_rewards/rejected": -0.3209163248538971,
      "eval_runtime": 726.3607,
      "eval_samples_per_second": 2.753,
      "eval_steps_per_second": 1.377,
      "step": 9800
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7122681623543239e-06,
      "logits/chosen": -2.4738147258758545,
      "logits/rejected": -2.210662364959717,
      "logps/chosen": -261.3623962402344,
      "logps/rejected": -269.6300048828125,
      "loss": 1656.6836,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.14373771846294403,
      "rewards/margins": 0.194814994931221,
      "rewards/rejected": -0.3385527431964874,
      "step": 9810
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7068511691147788e-06,
      "logits/chosen": -2.2168641090393066,
      "logits/rejected": -2.2618637084960938,
      "logps/chosen": -205.35595703125,
      "logps/rejected": -233.04379272460938,
      "loss": 1725.5117,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11224903911352158,
      "rewards/margins": 0.19318260252475739,
      "rewards/rejected": -0.30543166399002075,
      "step": 9820
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7014383133280636e-06,
      "logits/chosen": -2.452572822570801,
      "logits/rejected": -2.090609073638916,
      "logps/chosen": -269.6567687988281,
      "logps/rejected": -226.94692993164062,
      "loss": 2184.6486,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.21022340655326843,
      "rewards/margins": 0.10684118419885635,
      "rewards/rejected": -0.3170645833015442,
      "step": 9830
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.696029623230286e-06,
      "logits/chosen": -2.4171395301818848,
      "logits/rejected": -2.3338518142700195,
      "logps/chosen": -270.8750305175781,
      "logps/rejected": -304.5694274902344,
      "loss": 1857.9078,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.14952710270881653,
      "rewards/margins": 0.14915330708026886,
      "rewards/rejected": -0.2986803948879242,
      "step": 9840
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.6906251270358229e-06,
      "logits/chosen": -2.4054393768310547,
      "logits/rejected": -2.258305788040161,
      "logps/chosen": -274.5176086425781,
      "logps/rejected": -240.35842895507812,
      "loss": 1680.9318,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11663287878036499,
      "rewards/margins": 0.1558263897895813,
      "rewards/rejected": -0.2724592983722687,
      "step": 9850
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.685224852937174e-06,
      "logits/chosen": -2.216269016265869,
      "logits/rejected": -2.1068036556243896,
      "logps/chosen": -205.4796600341797,
      "logps/rejected": -273.53826904296875,
      "loss": 1176.6585,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07114885747432709,
      "rewards/margins": 0.26199641823768616,
      "rewards/rejected": -0.33314526081085205,
      "step": 9860
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6798288291048136e-06,
      "logits/chosen": -2.1753268241882324,
      "logits/rejected": -2.0786020755767822,
      "logps/chosen": -233.43582153320312,
      "logps/rejected": -225.013671875,
      "loss": 1643.7133,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10194216668605804,
      "rewards/margins": 0.2028282880783081,
      "rewards/rejected": -0.30477046966552734,
      "step": 9870
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6744370836870466e-06,
      "logits/chosen": -2.5711913108825684,
      "logits/rejected": -2.2863872051239014,
      "logps/chosen": -343.8927307128906,
      "logps/rejected": -280.05975341796875,
      "loss": 1962.7498,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09199172258377075,
      "rewards/margins": 0.19684836268424988,
      "rewards/rejected": -0.288840115070343,
      "step": 9880
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6690496448098576e-06,
      "logits/chosen": -2.2480640411376953,
      "logits/rejected": -1.9541877508163452,
      "logps/chosen": -242.25094604492188,
      "logps/rejected": -237.30178833007812,
      "loss": 1781.4516,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1321474015712738,
      "rewards/margins": 0.15909911692142487,
      "rewards/rejected": -0.2912465035915375,
      "step": 9890
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6636665405767666e-06,
      "logits/chosen": -2.3545076847076416,
      "logits/rejected": -2.1871304512023926,
      "logps/chosen": -246.6212921142578,
      "logps/rejected": -243.40444946289062,
      "loss": 1784.2059,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0885036438703537,
      "rewards/margins": 0.15673181414604187,
      "rewards/rejected": -0.24523547291755676,
      "step": 9900
    },
    {
      "epoch": 0.65,
      "eval_logits/chosen": -2.3468716144561768,
      "eval_logits/rejected": -2.1599361896514893,
      "eval_logps/chosen": -245.296630859375,
      "eval_logps/rejected": -240.30271911621094,
      "eval_loss": 1775.6644287109375,
      "eval_rewards/accuracies": 0.6625000238418579,
      "eval_rewards/chosen": -0.13291673362255096,
      "eval_rewards/margins": 0.1539914458990097,
      "eval_rewards/rejected": -0.28690820932388306,
      "eval_runtime": 725.6555,
      "eval_samples_per_second": 2.756,
      "eval_steps_per_second": 1.378,
      "step": 9900
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6582877990686827e-06,
      "logits/chosen": -2.3545796871185303,
      "logits/rejected": -2.2970447540283203,
      "logps/chosen": -134.80886840820312,
      "logps/rejected": -172.93389892578125,
      "loss": 1614.8482,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13514499366283417,
      "rewards/margins": 0.1579369157552719,
      "rewards/rejected": -0.2930819094181061,
      "step": 9910
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6529134483437562e-06,
      "logits/chosen": -2.3577752113342285,
      "logits/rejected": -2.1761374473571777,
      "logps/chosen": -214.59286499023438,
      "logps/rejected": -197.01571655273438,
      "loss": 1277.2855,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0901859924197197,
      "rewards/margins": 0.21876561641693115,
      "rewards/rejected": -0.30895155668258667,
      "step": 9920
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.647543516437233e-06,
      "logits/chosen": -2.3127143383026123,
      "logits/rejected": -2.252162456512451,
      "logps/chosen": -214.50137329101562,
      "logps/rejected": -246.9535675048828,
      "loss": 2047.5131,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.1546315997838974,
      "rewards/margins": 0.117673359811306,
      "rewards/rejected": -0.2723049521446228,
      "step": 9930
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6421780313613088e-06,
      "logits/chosen": -2.4648447036743164,
      "logits/rejected": -2.0659372806549072,
      "logps/chosen": -217.0897216796875,
      "logps/rejected": -202.35964965820312,
      "loss": 1451.7236,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12348121404647827,
      "rewards/margins": 0.18459756672382355,
      "rewards/rejected": -0.308078795671463,
      "step": 9940
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6368170211049816e-06,
      "logits/chosen": -2.371119737625122,
      "logits/rejected": -1.8730380535125732,
      "logps/chosen": -300.0697326660156,
      "logps/rejected": -254.29638671875,
      "loss": 2004.6252,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.16269531846046448,
      "rewards/margins": 0.14338134229183197,
      "rewards/rejected": -0.30607667565345764,
      "step": 9950
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6314605136339074e-06,
      "logits/chosen": -2.372769594192505,
      "logits/rejected": -2.2136123180389404,
      "logps/chosen": -213.3172149658203,
      "logps/rejected": -205.68411254882812,
      "loss": 2142.3727,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1773051619529724,
      "rewards/margins": 0.10266335308551788,
      "rewards/rejected": -0.2799685001373291,
      "step": 9960
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6261085368902526e-06,
      "logits/chosen": -2.5684804916381836,
      "logits/rejected": -2.2169997692108154,
      "logps/chosen": -282.66668701171875,
      "logps/rejected": -249.35928344726562,
      "loss": 2015.8521,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17725810408592224,
      "rewards/margins": 0.1032206267118454,
      "rewards/rejected": -0.28047871589660645,
      "step": 9970
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6207611187925503e-06,
      "logits/chosen": -2.2575650215148926,
      "logits/rejected": -2.2796449661254883,
      "logps/chosen": -221.99356079101562,
      "logps/rejected": -285.18817138671875,
      "loss": 1402.7092,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1316666156053543,
      "rewards/margins": 0.16876843571662903,
      "rewards/rejected": -0.30043506622314453,
      "step": 9980
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6154182872355512e-06,
      "logits/chosen": -2.3123984336853027,
      "logits/rejected": -2.365647554397583,
      "logps/chosen": -179.47439575195312,
      "logps/rejected": -205.01754760742188,
      "loss": 2203.9023,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.15656927227973938,
      "rewards/margins": 0.11123538017272949,
      "rewards/rejected": -0.26780465245246887,
      "step": 9990
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.610080070090084e-06,
      "logits/chosen": -2.3012657165527344,
      "logits/rejected": -2.237123489379883,
      "logps/chosen": -195.8773193359375,
      "logps/rejected": -203.55047607421875,
      "loss": 1470.263,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.14956019818782806,
      "rewards/margins": 0.16701506078243256,
      "rewards/rejected": -0.3165753185749054,
      "step": 10000
    },
    {
      "epoch": 0.65,
      "eval_logits/chosen": -2.3442938327789307,
      "eval_logits/rejected": -2.1574673652648926,
      "eval_logps/chosen": -245.648681640625,
      "eval_logps/rejected": -240.724365234375,
      "eval_loss": 1772.5384521484375,
      "eval_rewards/accuracies": 0.6579999923706055,
      "eval_rewards/chosen": -0.136437326669693,
      "eval_rewards/margins": 0.15468725562095642,
      "eval_rewards/rejected": -0.2911245822906494,
      "eval_runtime": 725.8522,
      "eval_samples_per_second": 2.755,
      "eval_steps_per_second": 1.378,
      "step": 10000
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6047464952029034e-06,
      "logits/chosen": -2.4696366786956787,
      "logits/rejected": -2.3387560844421387,
      "logps/chosen": -264.2880554199219,
      "logps/rejected": -293.85455322265625,
      "loss": 1207.3434,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.10000646114349365,
      "rewards/margins": 0.21340465545654297,
      "rewards/rejected": -0.31341108679771423,
      "step": 10010
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5994175903965486e-06,
      "logits/chosen": -2.204178810119629,
      "logits/rejected": -2.0907106399536133,
      "logps/chosen": -264.9002685546875,
      "logps/rejected": -285.266357421875,
      "loss": 1572.2071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.14569909870624542,
      "rewards/margins": 0.18611545860767365,
      "rewards/rejected": -0.33181455731391907,
      "step": 10020
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5940933834691977e-06,
      "logits/chosen": -2.6393990516662598,
      "logits/rejected": -1.9954341650009155,
      "logps/chosen": -318.2119445800781,
      "logps/rejected": -229.7041015625,
      "loss": 1494.415,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.12946000695228577,
      "rewards/margins": 0.1938951015472412,
      "rewards/rejected": -0.32335513830184937,
      "step": 10030
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.588773902194522e-06,
      "logits/chosen": -2.153494358062744,
      "logits/rejected": -1.8746439218521118,
      "logps/chosen": -221.29696655273438,
      "logps/rejected": -243.8973388671875,
      "loss": 2618.1234,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1721726804971695,
      "rewards/margins": 0.2684074342250824,
      "rewards/rejected": -0.4405801296234131,
      "step": 10040
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.583459174321541e-06,
      "logits/chosen": -2.1277883052825928,
      "logits/rejected": -1.9743620157241821,
      "logps/chosen": -226.68612670898438,
      "logps/rejected": -216.8396453857422,
      "loss": 1734.6359,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17068837583065033,
      "rewards/margins": 0.17247551679611206,
      "rewards/rejected": -0.3431639075279236,
      "step": 10050
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5781492275744797e-06,
      "logits/chosen": -2.529752492904663,
      "logits/rejected": -2.1446166038513184,
      "logps/chosen": -305.91815185546875,
      "logps/rejected": -305.4971618652344,
      "loss": 1542.6445,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.11673681437969208,
      "rewards/margins": 0.25796908140182495,
      "rewards/rejected": -0.3747059106826782,
      "step": 10060
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5728440896526215e-06,
      "logits/chosen": -2.2576258182525635,
      "logits/rejected": -2.0778629779815674,
      "logps/chosen": -286.76837158203125,
      "logps/rejected": -255.7520751953125,
      "loss": 1687.0375,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13366422057151794,
      "rewards/margins": 0.151865154504776,
      "rewards/rejected": -0.28552937507629395,
      "step": 10070
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5675437882301633e-06,
      "logits/chosen": -2.3388214111328125,
      "logits/rejected": -2.1428470611572266,
      "logps/chosen": -230.275146484375,
      "logps/rejected": -207.76254272460938,
      "loss": 1935.9998,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11230950057506561,
      "rewards/margins": 0.12283160537481308,
      "rewards/rejected": -0.2351410835981369,
      "step": 10080
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5622483509560748e-06,
      "logits/chosen": -2.2283027172088623,
      "logits/rejected": -2.2405989170074463,
      "logps/chosen": -184.2806854248047,
      "logps/rejected": -231.23104858398438,
      "loss": 1743.1004,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1355154663324356,
      "rewards/margins": 0.16768063604831696,
      "rewards/rejected": -0.30319613218307495,
      "step": 10090
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5569578054539506e-06,
      "logits/chosen": -2.279250383377075,
      "logits/rejected": -1.950156569480896,
      "logps/chosen": -290.9871520996094,
      "logps/rejected": -232.88156127929688,
      "loss": 1144.0941,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.11790060997009277,
      "rewards/margins": 0.25757458806037903,
      "rewards/rejected": -0.3754751682281494,
      "step": 10100
    },
    {
      "epoch": 0.66,
      "eval_logits/chosen": -2.339332103729248,
      "eval_logits/rejected": -2.1528501510620117,
      "eval_logps/chosen": -244.95468139648438,
      "eval_logps/rejected": -240.91522216796875,
      "eval_loss": 1776.5482177734375,
      "eval_rewards/accuracies": 0.6614999771118164,
      "eval_rewards/chosen": -0.12949733436107635,
      "eval_rewards/margins": 0.1635356992483139,
      "eval_rewards/rejected": -0.29303303360939026,
      "eval_runtime": 725.4527,
      "eval_samples_per_second": 2.757,
      "eval_steps_per_second": 1.378,
      "step": 10100
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.551672179321867e-06,
      "logits/chosen": -2.2603707313537598,
      "logits/rejected": -2.33063006401062,
      "logps/chosen": -225.4585723876953,
      "logps/rejected": -221.43777465820312,
      "loss": 1686.2473,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09601660072803497,
      "rewards/margins": 0.17150314152240753,
      "rewards/rejected": -0.2675197124481201,
      "step": 10110
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5463915001322398e-06,
      "logits/chosen": -2.2999863624572754,
      "logits/rejected": -2.158628463745117,
      "logps/chosen": -259.41180419921875,
      "logps/rejected": -259.23175048828125,
      "loss": 2175.3219,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13755428791046143,
      "rewards/margins": 0.14187631011009216,
      "rewards/rejected": -0.279430627822876,
      "step": 10120
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5411157954316784e-06,
      "logits/chosen": -2.2974162101745605,
      "logits/rejected": -2.2118053436279297,
      "logps/chosen": -212.8948211669922,
      "logps/rejected": -214.84414672851562,
      "loss": 1701.4238,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11017285287380219,
      "rewards/margins": 0.15639138221740723,
      "rewards/rejected": -0.2665642201900482,
      "step": 10130
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.535845092740843e-06,
      "logits/chosen": -2.470548629760742,
      "logits/rejected": -2.306570053100586,
      "logps/chosen": -249.66329956054688,
      "logps/rejected": -272.49053955078125,
      "loss": 2000.9893,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12342099845409393,
      "rewards/margins": 0.10431381314992905,
      "rewards/rejected": -0.22773483395576477,
      "step": 10140
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5305794195543005e-06,
      "logits/chosen": -2.3118038177490234,
      "logits/rejected": -2.3743338584899902,
      "logps/chosen": -220.2370147705078,
      "logps/rejected": -222.10546875,
      "loss": 1502.9931,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1305791139602661,
      "rewards/margins": 0.1879984438419342,
      "rewards/rejected": -0.3185775578022003,
      "step": 10150
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5253188033403816e-06,
      "logits/chosen": -2.330814838409424,
      "logits/rejected": -2.433004379272461,
      "logps/chosen": -186.08499145507812,
      "logps/rejected": -211.61630249023438,
      "loss": 2086.2031,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.15350119769573212,
      "rewards/margins": 0.08872922509908676,
      "rewards/rejected": -0.24223044514656067,
      "step": 10160
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.520063271541037e-06,
      "logits/chosen": -2.319892406463623,
      "logits/rejected": -2.222487688064575,
      "logps/chosen": -198.03294372558594,
      "logps/rejected": -202.96609497070312,
      "loss": 1855.0166,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.19528180360794067,
      "rewards/margins": 0.1623615324497223,
      "rewards/rejected": -0.35764333605766296,
      "step": 10170
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5148128515716954e-06,
      "logits/chosen": -2.5416927337646484,
      "logits/rejected": -1.908489465713501,
      "logps/chosen": -280.6117248535156,
      "logps/rejected": -228.46212768554688,
      "loss": 1646.4756,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.14082498848438263,
      "rewards/margins": 0.20224598050117493,
      "rewards/rejected": -0.343070924282074,
      "step": 10180
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5095675708211197e-06,
      "logits/chosen": -2.362449884414673,
      "logits/rejected": -2.357105016708374,
      "logps/chosen": -216.79443359375,
      "logps/rejected": -246.13461303710938,
      "loss": 2490.6564,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.25633513927459717,
      "rewards/margins": 0.05996602773666382,
      "rewards/rejected": -0.316301167011261,
      "step": 10190
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.504327456651263e-06,
      "logits/chosen": -2.305445432662964,
      "logits/rejected": -2.234525203704834,
      "logps/chosen": -286.28363037109375,
      "logps/rejected": -270.63360595703125,
      "loss": 1890.1879,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.24880632758140564,
      "rewards/margins": 0.11314669996500015,
      "rewards/rejected": -0.3619530200958252,
      "step": 10200
    },
    {
      "epoch": 0.67,
      "eval_logits/chosen": -2.3296923637390137,
      "eval_logits/rejected": -2.144120454788208,
      "eval_logps/chosen": -250.76393127441406,
      "eval_logps/rejected": -244.91712951660156,
      "eval_loss": 1785.0318603515625,
      "eval_rewards/accuracies": 0.652999997138977,
      "eval_rewards/chosen": -0.1875898540019989,
      "eval_rewards/margins": 0.14546243846416473,
      "eval_rewards/rejected": -0.33305224776268005,
      "eval_runtime": 722.9768,
      "eval_samples_per_second": 2.766,
      "eval_steps_per_second": 1.383,
      "step": 10200
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4990925363971284e-06,
      "logits/chosen": -2.388564348220825,
      "logits/rejected": -1.984683632850647,
      "logps/chosen": -311.1983642578125,
      "logps/rejected": -266.94000244140625,
      "loss": 1314.9645,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21275189518928528,
      "rewards/margins": 0.24230685830116272,
      "rewards/rejected": -0.4550587236881256,
      "step": 10210
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4938628373666236e-06,
      "logits/chosen": -2.25669264793396,
      "logits/rejected": -2.3107972145080566,
      "logps/chosen": -190.91799926757812,
      "logps/rejected": -204.0850372314453,
      "loss": 2016.073,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.18338727951049805,
      "rewards/margins": 0.11294351518154144,
      "rewards/rejected": -0.29633083939552307,
      "step": 10220
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4886383868404203e-06,
      "logits/chosen": -2.1663084030151367,
      "logits/rejected": -2.0342824459075928,
      "logps/chosen": -180.13150024414062,
      "logps/rejected": -188.49063110351562,
      "loss": 1373.7174,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18909737467765808,
      "rewards/margins": 0.1838906705379486,
      "rewards/rejected": -0.3729880750179291,
      "step": 10230
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.483419212071813e-06,
      "logits/chosen": -2.12137770652771,
      "logits/rejected": -1.9535928964614868,
      "logps/chosen": -203.91366577148438,
      "logps/rejected": -209.3536376953125,
      "loss": 1905.7352,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15449532866477966,
      "rewards/margins": 0.1262134462594986,
      "rewards/rejected": -0.28070876002311707,
      "step": 10240
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.478205340286573e-06,
      "logits/chosen": -2.257779121398926,
      "logits/rejected": -2.225194215774536,
      "logps/chosen": -225.85525512695312,
      "logps/rejected": -220.8550567626953,
      "loss": 1872.184,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.222463458776474,
      "rewards/margins": 0.13004586100578308,
      "rewards/rejected": -0.35250934958457947,
      "step": 10250
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4729967986828104e-06,
      "logits/chosen": -2.408285140991211,
      "logits/rejected": -2.1716907024383545,
      "logps/chosen": -335.02398681640625,
      "logps/rejected": -307.013427734375,
      "loss": 1414.1995,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1304025650024414,
      "rewards/margins": 0.19957467913627625,
      "rewards/rejected": -0.32997727394104004,
      "step": 10260
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4677936144308286e-06,
      "logits/chosen": -2.3844265937805176,
      "logits/rejected": -2.093235731124878,
      "logps/chosen": -239.69839477539062,
      "logps/rejected": -229.95849609375,
      "loss": 1554.3381,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15453621745109558,
      "rewards/margins": 0.18323197960853577,
      "rewards/rejected": -0.33776822686195374,
      "step": 10270
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4625958146729864e-06,
      "logits/chosen": -2.401050329208374,
      "logits/rejected": -2.231168031692505,
      "logps/chosen": -234.28488159179688,
      "logps/rejected": -232.4915771484375,
      "loss": 1820.0857,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.14021024107933044,
      "rewards/margins": 0.14154712855815887,
      "rewards/rejected": -0.2817573845386505,
      "step": 10280
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4574034265235523e-06,
      "logits/chosen": -2.496002197265625,
      "logits/rejected": -1.9339748620986938,
      "logps/chosen": -269.0251770019531,
      "logps/rejected": -190.8519287109375,
      "loss": 1758.001,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1319662183523178,
      "rewards/margins": 0.1958966851234436,
      "rewards/rejected": -0.3278628885746002,
      "step": 10290
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.452216477068568e-06,
      "logits/chosen": -2.3668737411499023,
      "logits/rejected": -1.8553102016448975,
      "logps/chosen": -242.12954711914062,
      "logps/rejected": -170.4160614013672,
      "loss": 1441.0404,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1208452582359314,
      "rewards/margins": 0.21201328933238983,
      "rewards/rejected": -0.3328585624694824,
      "step": 10300
    },
    {
      "epoch": 0.67,
      "eval_logits/chosen": -2.336395502090454,
      "eval_logits/rejected": -2.1501612663269043,
      "eval_logps/chosen": -247.1606903076172,
      "eval_logps/rejected": -242.07144165039062,
      "eval_loss": 1778.0972900390625,
      "eval_rewards/accuracies": 0.6549999713897705,
      "eval_rewards/chosen": -0.1515573114156723,
      "eval_rewards/margins": 0.1530378758907318,
      "eval_rewards/rejected": -0.30459514260292053,
      "eval_runtime": 726.5012,
      "eval_samples_per_second": 2.753,
      "eval_steps_per_second": 1.376,
      "step": 10300
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4470349933657004e-06,
      "logits/chosen": -2.5289347171783447,
      "logits/rejected": -2.299401044845581,
      "logps/chosen": -235.0850830078125,
      "logps/rejected": -227.1444091796875,
      "loss": 1535.2778,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14355462789535522,
      "rewards/margins": 0.16038207709789276,
      "rewards/rejected": -0.3039367198944092,
      "step": 10310
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4418590024441096e-06,
      "logits/chosen": -2.4282679557800293,
      "logits/rejected": -2.0404889583587646,
      "logps/chosen": -264.26629638671875,
      "logps/rejected": -212.7019500732422,
      "loss": 1732.6947,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10806041955947876,
      "rewards/margins": 0.14920981228351593,
      "rewards/rejected": -0.25727027654647827,
      "step": 10320
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.436688531304297e-06,
      "logits/chosen": -2.422285556793213,
      "logits/rejected": -2.0882749557495117,
      "logps/chosen": -230.7120361328125,
      "logps/rejected": -241.85488891601562,
      "loss": 1907.8205,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1238451600074768,
      "rewards/margins": 0.16632047295570374,
      "rewards/rejected": -0.29016566276550293,
      "step": 10330
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.431523606917974e-06,
      "logits/chosen": -2.2472996711730957,
      "logits/rejected": -2.2454934120178223,
      "logps/chosen": -224.62149047851562,
      "logps/rejected": -243.8383026123047,
      "loss": 2174.4451,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19522175192832947,
      "rewards/margins": 0.12602083384990692,
      "rewards/rejected": -0.3212426006793976,
      "step": 10340
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4263642562279162e-06,
      "logits/chosen": -2.0353946685791016,
      "logits/rejected": -2.0154049396514893,
      "logps/chosen": -265.7103271484375,
      "logps/rejected": -288.4093322753906,
      "loss": 1817.1635,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.16222988069057465,
      "rewards/margins": 0.14208927750587463,
      "rewards/rejected": -0.3043191730976105,
      "step": 10350
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4212105061478257e-06,
      "logits/chosen": -2.087690830230713,
      "logits/rejected": -2.112273693084717,
      "logps/chosen": -246.8751220703125,
      "logps/rejected": -272.3307189941406,
      "loss": 1790.5391,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16984596848487854,
      "rewards/margins": 0.1763208955526352,
      "rewards/rejected": -0.34616684913635254,
      "step": 10360
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4160623835621848e-06,
      "logits/chosen": -2.43872332572937,
      "logits/rejected": -2.3132472038269043,
      "logps/chosen": -249.39242553710938,
      "logps/rejected": -254.17178344726562,
      "loss": 1500.7467,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1208123192191124,
      "rewards/margins": 0.16257521510124207,
      "rewards/rejected": -0.28338757157325745,
      "step": 10370
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4109199153261249e-06,
      "logits/chosen": -2.226412773132324,
      "logits/rejected": -2.1237289905548096,
      "logps/chosen": -289.9986267089844,
      "logps/rejected": -275.4288024902344,
      "loss": 1812.118,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.14004027843475342,
      "rewards/margins": 0.1574375331401825,
      "rewards/rejected": -0.2974777817726135,
      "step": 10380
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.405783128265278e-06,
      "logits/chosen": -2.304079294204712,
      "logits/rejected": -2.284876823425293,
      "logps/chosen": -222.9115753173828,
      "logps/rejected": -230.6747589111328,
      "loss": 1979.2328,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1877807080745697,
      "rewards/margins": 0.11029338836669922,
      "rewards/rejected": -0.2980740964412689,
      "step": 10390
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4006520491756427e-06,
      "logits/chosen": -2.4129700660705566,
      "logits/rejected": -2.1868672370910645,
      "logps/chosen": -209.59365844726562,
      "logps/rejected": -160.5478973388672,
      "loss": 1606.4429,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13369429111480713,
      "rewards/margins": 0.15595155954360962,
      "rewards/rejected": -0.28964588046073914,
      "step": 10400
    },
    {
      "epoch": 0.68,
      "eval_logits/chosen": -2.336988925933838,
      "eval_logits/rejected": -2.1507012844085693,
      "eval_logps/chosen": -243.77908325195312,
      "eval_logps/rejected": -238.7640380859375,
      "eval_loss": 1776.54150390625,
      "eval_rewards/accuracies": 0.6604999899864197,
      "eval_rewards/chosen": -0.11774113774299622,
      "eval_rewards/margins": 0.15378008782863617,
      "eval_rewards/rejected": -0.2715212106704712,
      "eval_runtime": 723.8472,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.382,
      "step": 10400
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.39552670482344e-06,
      "logits/chosen": -2.234830856323242,
      "logits/rejected": -2.3060553073883057,
      "logps/chosen": -187.51327514648438,
      "logps/rejected": -200.25709533691406,
      "loss": 1896.8854,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12913690507411957,
      "rewards/margins": 0.1285235583782196,
      "rewards/rejected": -0.25766047835350037,
      "step": 10410
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3904071219449776e-06,
      "logits/chosen": -2.319739818572998,
      "logits/rejected": -1.888177514076233,
      "logps/chosen": -208.35977172851562,
      "logps/rejected": -139.51470947265625,
      "loss": 1721.7918,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0845813974738121,
      "rewards/margins": 0.15815046429634094,
      "rewards/rejected": -0.24273185431957245,
      "step": 10420
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3852933272465068e-06,
      "logits/chosen": -2.4715352058410645,
      "logits/rejected": -2.241602659225464,
      "logps/chosen": -245.01632690429688,
      "logps/rejected": -224.55087280273438,
      "loss": 1806.8795,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07156185060739517,
      "rewards/margins": 0.14758458733558655,
      "rewards/rejected": -0.21914644539356232,
      "step": 10430
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3801853474040873e-06,
      "logits/chosen": -2.273109197616577,
      "logits/rejected": -2.209038734436035,
      "logps/chosen": -250.9441375732422,
      "logps/rejected": -252.0476837158203,
      "loss": 1534.3904,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12422917783260345,
      "rewards/margins": 0.17563822865486145,
      "rewards/rejected": -0.2998674213886261,
      "step": 10440
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3750832090634417e-06,
      "logits/chosen": -2.4183297157287598,
      "logits/rejected": -2.111826181411743,
      "logps/chosen": -193.4394073486328,
      "logps/rejected": -192.63906860351562,
      "loss": 1852.5627,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07266728579998016,
      "rewards/margins": 0.1385105401277542,
      "rewards/rejected": -0.211177796125412,
      "step": 10450
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3699869388398245e-06,
      "logits/chosen": -2.278883218765259,
      "logits/rejected": -2.1369309425354004,
      "logps/chosen": -225.41000366210938,
      "logps/rejected": -223.36270141601562,
      "loss": 1772.0594,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12388720363378525,
      "rewards/margins": 0.1640610694885254,
      "rewards/rejected": -0.28794828057289124,
      "step": 10460
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3648965633178772e-06,
      "logits/chosen": -2.3005356788635254,
      "logits/rejected": -2.2080466747283936,
      "logps/chosen": -210.2812042236328,
      "logps/rejected": -241.01046752929688,
      "loss": 1483.6773,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09549007564783096,
      "rewards/margins": 0.19317705929279327,
      "rewards/rejected": -0.28866714239120483,
      "step": 10470
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3598121090514938e-06,
      "logits/chosen": -2.290022850036621,
      "logits/rejected": -2.1660006046295166,
      "logps/chosen": -193.58340454101562,
      "logps/rejected": -185.4873809814453,
      "loss": 1735.5125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10575524717569351,
      "rewards/margins": 0.16027264297008514,
      "rewards/rejected": -0.26602789759635925,
      "step": 10480
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3547336025636753e-06,
      "logits/chosen": -2.218031167984009,
      "logits/rejected": -2.0019259452819824,
      "logps/chosen": -283.0633544921875,
      "logps/rejected": -257.22479248046875,
      "loss": 1576.8256,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07766801863908768,
      "rewards/margins": 0.1569044291973114,
      "rewards/rejected": -0.23457245528697968,
      "step": 10490
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3496610703464022e-06,
      "logits/chosen": -2.36081600189209,
      "logits/rejected": -2.132767915725708,
      "logps/chosen": -239.56436157226562,
      "logps/rejected": -205.96337890625,
      "loss": 2053.1404,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10349766165018082,
      "rewards/margins": 0.11920388042926788,
      "rewards/rejected": -0.2227015495300293,
      "step": 10500
    },
    {
      "epoch": 0.69,
      "eval_logits/chosen": -2.3365676403045654,
      "eval_logits/rejected": -2.1502785682678223,
      "eval_logps/chosen": -241.38241577148438,
      "eval_logps/rejected": -236.80250549316406,
      "eval_loss": 1771.5421142578125,
      "eval_rewards/accuracies": 0.6620000004768372,
      "eval_rewards/chosen": -0.09377462416887283,
      "eval_rewards/margins": 0.15813127160072327,
      "eval_rewards/rejected": -0.2519059181213379,
      "eval_runtime": 728.0246,
      "eval_samples_per_second": 2.747,
      "eval_steps_per_second": 1.374,
      "step": 10500
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3445945388604848e-06,
      "logits/chosen": -2.2319271564483643,
      "logits/rejected": -1.9931703805923462,
      "logps/chosen": -253.85574340820312,
      "logps/rejected": -228.87545776367188,
      "loss": 1665.5861,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15632253885269165,
      "rewards/margins": 0.18180391192436218,
      "rewards/rejected": -0.33812645077705383,
      "step": 10510
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3395340345354358e-06,
      "logits/chosen": -2.257664203643799,
      "logits/rejected": -2.386795997619629,
      "logps/chosen": -237.11557006835938,
      "logps/rejected": -264.43096923828125,
      "loss": 2022.6344,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13417726755142212,
      "rewards/margins": 0.1006779819726944,
      "rewards/rejected": -0.2348552644252777,
      "step": 10520
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.334479583769322e-06,
      "logits/chosen": -2.4861791133880615,
      "logits/rejected": -2.179697036743164,
      "logps/chosen": -262.93231201171875,
      "logps/rejected": -225.8566131591797,
      "loss": 1849.49,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08093537390232086,
      "rewards/margins": 0.128939688205719,
      "rewards/rejected": -0.20987506210803986,
      "step": 10530
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3294312129286366e-06,
      "logits/chosen": -2.3211584091186523,
      "logits/rejected": -2.2067387104034424,
      "logps/chosen": -271.5013732910156,
      "logps/rejected": -275.35357666015625,
      "loss": 1424.4191,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0477275513112545,
      "rewards/margins": 0.17523090541362762,
      "rewards/rejected": -0.22295847535133362,
      "step": 10540
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.324388948348153e-06,
      "logits/chosen": -2.482022762298584,
      "logits/rejected": -2.0950958728790283,
      "logps/chosen": -292.1917419433594,
      "logps/rejected": -226.5915985107422,
      "loss": 1598.9847,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06953771412372589,
      "rewards/margins": 0.15766309201717377,
      "rewards/rejected": -0.22720082104206085,
      "step": 10550
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.319352816330796e-06,
      "logits/chosen": -2.5680453777313232,
      "logits/rejected": -2.056453227996826,
      "logps/chosen": -293.2705383300781,
      "logps/rejected": -205.46707153320312,
      "loss": 1670.0934,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10073323547840118,
      "rewards/margins": 0.1865191012620926,
      "rewards/rejected": -0.2872523367404938,
      "step": 10560
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.314322843147494e-06,
      "logits/chosen": -2.173163652420044,
      "logits/rejected": -2.2709197998046875,
      "logps/chosen": -191.83360290527344,
      "logps/rejected": -255.783447265625,
      "loss": 2125.8488,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21507766842842102,
      "rewards/margins": 0.0908210277557373,
      "rewards/rejected": -0.3058987259864807,
      "step": 10570
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3092990550370526e-06,
      "logits/chosen": -2.4326562881469727,
      "logits/rejected": -2.095061779022217,
      "logps/chosen": -358.76983642578125,
      "logps/rejected": -290.0596008300781,
      "loss": 1846.373,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13011455535888672,
      "rewards/margins": 0.16640953719615936,
      "rewards/rejected": -0.2965241074562073,
      "step": 10580
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3042814782060131e-06,
      "logits/chosen": -2.3822150230407715,
      "logits/rejected": -2.0831844806671143,
      "logps/chosen": -187.81179809570312,
      "logps/rejected": -182.1324462890625,
      "loss": 1811.2422,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06308932602405548,
      "rewards/margins": 0.1823693811893463,
      "rewards/rejected": -0.245458722114563,
      "step": 10590
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.2992701388285112e-06,
      "logits/chosen": -2.388526439666748,
      "logits/rejected": -2.132236957550049,
      "logps/chosen": -276.08428955078125,
      "logps/rejected": -247.83102416992188,
      "loss": 1666.0459,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06735239177942276,
      "rewards/margins": 0.16218677163124084,
      "rewards/rejected": -0.229539155960083,
      "step": 10600
    },
    {
      "epoch": 0.69,
      "eval_logits/chosen": -2.3377394676208496,
      "eval_logits/rejected": -2.1511542797088623,
      "eval_logps/chosen": -242.1644287109375,
      "eval_logps/rejected": -237.53196716308594,
      "eval_loss": 1766.10595703125,
      "eval_rewards/accuracies": 0.6614999771118164,
      "eval_rewards/chosen": -0.10159474611282349,
      "eval_rewards/margins": 0.15760593116283417,
      "eval_rewards/rejected": -0.25920066237449646,
      "eval_runtime": 723.0366,
      "eval_samples_per_second": 2.766,
      "eval_steps_per_second": 1.383,
      "step": 10600
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.29426506304615e-06,
      "logits/chosen": -2.2357826232910156,
      "logits/rejected": -2.1712448596954346,
      "logps/chosen": -238.3498992919922,
      "logps/rejected": -226.3423309326172,
      "loss": 2526.1244,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.15626461803913116,
      "rewards/margins": 0.0747048556804657,
      "rewards/rejected": -0.23096947371959686,
      "step": 10610
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.289266276967855e-06,
      "logits/chosen": -2.375516653060913,
      "logits/rejected": -2.2651820182800293,
      "logps/chosen": -345.6637878417969,
      "logps/rejected": -274.7169189453125,
      "loss": 1486.8291,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07081923633813858,
      "rewards/margins": 0.1921137273311615,
      "rewards/rejected": -0.2629329562187195,
      "step": 10620
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.284273806669745e-06,
      "logits/chosen": -2.3371615409851074,
      "logits/rejected": -2.1077303886413574,
      "logps/chosen": -256.08428955078125,
      "logps/rejected": -286.40386962890625,
      "loss": 1893.0041,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16759343445301056,
      "rewards/margins": 0.12778960168361664,
      "rewards/rejected": -0.2953830361366272,
      "step": 10630
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2792876781949884e-06,
      "logits/chosen": -2.083141326904297,
      "logits/rejected": -1.8333759307861328,
      "logps/chosen": -220.6055450439453,
      "logps/rejected": -211.8660430908203,
      "loss": 2060.8947,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.14532899856567383,
      "rewards/margins": 0.10578174889087677,
      "rewards/rejected": -0.2511107325553894,
      "step": 10640
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.274307917553676e-06,
      "logits/chosen": -2.3342247009277344,
      "logits/rejected": -2.2580673694610596,
      "logps/chosen": -208.902587890625,
      "logps/rejected": -252.3454132080078,
      "loss": 1515.762,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13243810832500458,
      "rewards/margins": 0.18843302130699158,
      "rewards/rejected": -0.32087117433547974,
      "step": 10650
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2693345507226767e-06,
      "logits/chosen": -2.127349376678467,
      "logits/rejected": -2.2174124717712402,
      "logps/chosen": -243.489990234375,
      "logps/rejected": -262.34393310546875,
      "loss": 1637.8714,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13499729335308075,
      "rewards/margins": 0.1674044132232666,
      "rewards/rejected": -0.30240169167518616,
      "step": 10660
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2643676036455099e-06,
      "logits/chosen": -2.3676364421844482,
      "logits/rejected": -2.281890869140625,
      "logps/chosen": -299.4869079589844,
      "logps/rejected": -263.0523986816406,
      "loss": 1848.6758,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11634413152933121,
      "rewards/margins": 0.1430656611919403,
      "rewards/rejected": -0.2594097852706909,
      "step": 10670
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.259407102232203e-06,
      "logits/chosen": -2.471794366836548,
      "logits/rejected": -2.0974783897399902,
      "logps/chosen": -290.4842224121094,
      "logps/rejected": -236.22451782226562,
      "loss": 1389.6499,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1284985989332199,
      "rewards/margins": 0.18684282898902893,
      "rewards/rejected": -0.3153414726257324,
      "step": 10680
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.254453072359163e-06,
      "logits/chosen": -2.410886287689209,
      "logits/rejected": -2.183062791824341,
      "logps/chosen": -247.2122039794922,
      "logps/rejected": -240.69680786132812,
      "loss": 1719.7668,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13192158937454224,
      "rewards/margins": 0.1535084843635559,
      "rewards/rejected": -0.28543007373809814,
      "step": 10690
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2495055398690337e-06,
      "logits/chosen": -2.4857728481292725,
      "logits/rejected": -2.243335723876953,
      "logps/chosen": -241.6734161376953,
      "logps/rejected": -244.768798828125,
      "loss": 2062.3629,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11819012463092804,
      "rewards/margins": 0.12378589808940887,
      "rewards/rejected": -0.24197602272033691,
      "step": 10700
    },
    {
      "epoch": 0.7,
      "eval_logits/chosen": -2.340343475341797,
      "eval_logits/rejected": -2.1535558700561523,
      "eval_logps/chosen": -244.5763702392578,
      "eval_logps/rejected": -240.011962890625,
      "eval_loss": 1763.6278076171875,
      "eval_rewards/accuracies": 0.6635000109672546,
      "eval_rewards/chosen": -0.12571415305137634,
      "eval_rewards/margins": 0.15828652679920197,
      "eval_rewards/rejected": -0.2840006947517395,
      "eval_runtime": 722.9166,
      "eval_samples_per_second": 2.767,
      "eval_steps_per_second": 1.383,
      "step": 10700
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2445645305705718e-06,
      "logits/chosen": -2.511423110961914,
      "logits/rejected": -2.175668239593506,
      "logps/chosen": -220.97122192382812,
      "logps/rejected": -212.86807250976562,
      "loss": 2147.9316,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1465025693178177,
      "rewards/margins": 0.12539730966091156,
      "rewards/rejected": -0.27189984917640686,
      "step": 10710
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2396300702384995e-06,
      "logits/chosen": -2.497962474822998,
      "logits/rejected": -2.2529397010803223,
      "logps/chosen": -274.5304870605469,
      "logps/rejected": -246.68466186523438,
      "loss": 2249.2039,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.16722413897514343,
      "rewards/margins": 0.09819474071264267,
      "rewards/rejected": -0.2654188871383667,
      "step": 10720
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.234702184613381e-06,
      "logits/chosen": -2.1192617416381836,
      "logits/rejected": -2.2030041217803955,
      "logps/chosen": -228.84054565429688,
      "logps/rejected": -241.6710968017578,
      "loss": 1799.9865,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1335824429988861,
      "rewards/margins": 0.13166043162345886,
      "rewards/rejected": -0.26524287462234497,
      "step": 10730
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2297808994014793e-06,
      "logits/chosen": -2.4309096336364746,
      "logits/rejected": -2.181896209716797,
      "logps/chosen": -296.35955810546875,
      "logps/rejected": -274.7001647949219,
      "loss": 1733.6844,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10424685478210449,
      "rewards/margins": 0.1323317587375641,
      "rewards/rejected": -0.23657865822315216,
      "step": 10740
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2248662402746314e-06,
      "logits/chosen": -2.2782938480377197,
      "logits/rejected": -2.1423420906066895,
      "logps/chosen": -205.86904907226562,
      "logps/rejected": -215.0177001953125,
      "loss": 1634.1285,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15952368080615997,
      "rewards/margins": 0.15264497697353363,
      "rewards/rejected": -0.3121686577796936,
      "step": 10750
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2199582328701045e-06,
      "logits/chosen": -2.4306654930114746,
      "logits/rejected": -1.937919020652771,
      "logps/chosen": -302.0787658691406,
      "logps/rejected": -277.5871887207031,
      "loss": 1759.1566,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11445232480764389,
      "rewards/margins": 0.15799331665039062,
      "rewards/rejected": -0.2724456489086151,
      "step": 10760
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2150569027904712e-06,
      "logits/chosen": -2.3267223834991455,
      "logits/rejected": -2.2170944213867188,
      "logps/chosen": -257.29571533203125,
      "logps/rejected": -268.56219482421875,
      "loss": 1791.2227,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1131223812699318,
      "rewards/margins": 0.17569416761398315,
      "rewards/rejected": -0.28881654143333435,
      "step": 10770
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.2101622756034688e-06,
      "logits/chosen": -2.318678379058838,
      "logits/rejected": -2.3083908557891846,
      "logps/chosen": -230.8387908935547,
      "logps/rejected": -215.8868408203125,
      "loss": 1718.6527,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0808473750948906,
      "rewards/margins": 0.15413309633731842,
      "rewards/rejected": -0.2349804937839508,
      "step": 10780
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.2052743768418715e-06,
      "logits/chosen": -2.3797993659973145,
      "logits/rejected": -2.1624557971954346,
      "logps/chosen": -256.75262451171875,
      "logps/rejected": -237.8788604736328,
      "loss": 1739.4143,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08943568170070648,
      "rewards/margins": 0.1513310670852661,
      "rewards/rejected": -0.2407667636871338,
      "step": 10790
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.2003932320033523e-06,
      "logits/chosen": -2.4958038330078125,
      "logits/rejected": -2.1804280281066895,
      "logps/chosen": -237.180908203125,
      "logps/rejected": -261.7378234863281,
      "loss": 1241.0871,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10678191483020782,
      "rewards/margins": 0.2392505705356598,
      "rewards/rejected": -0.3460325300693512,
      "step": 10800
    },
    {
      "epoch": 0.71,
      "eval_logits/chosen": -2.347846031188965,
      "eval_logits/rejected": -2.160001039505005,
      "eval_logps/chosen": -243.9580841064453,
      "eval_logps/rejected": -239.3552703857422,
      "eval_loss": 1766.05029296875,
      "eval_rewards/accuracies": 0.6635000109672546,
      "eval_rewards/chosen": -0.11953125149011612,
      "eval_rewards/margins": 0.1579025387763977,
      "eval_rewards/rejected": -0.2774338126182556,
      "eval_runtime": 724.7871,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.38,
      "step": 10800
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1955188665503553e-06,
      "logits/chosen": -2.1565940380096436,
      "logits/rejected": -2.215562105178833,
      "logps/chosen": -219.93881225585938,
      "logps/rejected": -212.1143035888672,
      "loss": 2009.3404,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.14815133810043335,
      "rewards/margins": 0.12965616583824158,
      "rewards/rejected": -0.2778075337409973,
      "step": 10810
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1906513059099566e-06,
      "logits/chosen": -2.3357901573181152,
      "logits/rejected": -2.0216660499572754,
      "logps/chosen": -244.9086456298828,
      "logps/rejected": -254.49697875976562,
      "loss": 1451.7775,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.13419561088085175,
      "rewards/margins": 0.1832888424396515,
      "rewards/rejected": -0.31748443841934204,
      "step": 10820
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.185790575473738e-06,
      "logits/chosen": -2.277193069458008,
      "logits/rejected": -2.0780417919158936,
      "logps/chosen": -241.35208129882812,
      "logps/rejected": -210.4105987548828,
      "loss": 1686.5461,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12218759208917618,
      "rewards/margins": 0.1633271425962448,
      "rewards/rejected": -0.2855147123336792,
      "step": 10830
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1809367005976516e-06,
      "logits/chosen": -2.3439080715179443,
      "logits/rejected": -2.101081609725952,
      "logps/chosen": -293.42987060546875,
      "logps/rejected": -234.614501953125,
      "loss": 1877.9219,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.10935065895318985,
      "rewards/margins": 0.12232010066509247,
      "rewards/rejected": -0.23167076706886292,
      "step": 10840
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1760897066018842e-06,
      "logits/chosen": -2.271735668182373,
      "logits/rejected": -2.1386935710906982,
      "logps/chosen": -234.00830078125,
      "logps/rejected": -246.95761108398438,
      "loss": 1651.3365,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.109672412276268,
      "rewards/margins": 0.18946237862110138,
      "rewards/rejected": -0.2991347908973694,
      "step": 10850
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1712496187707327e-06,
      "logits/chosen": -2.307668447494507,
      "logits/rejected": -2.053638219833374,
      "logps/chosen": -254.58633422851562,
      "logps/rejected": -263.89300537109375,
      "loss": 1443.3568,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.14282092452049255,
      "rewards/margins": 0.2337246686220169,
      "rewards/rejected": -0.37654557824134827,
      "step": 10860
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1664164623524646e-06,
      "logits/chosen": -2.286165237426758,
      "logits/rejected": -2.126422882080078,
      "logps/chosen": -228.3876953125,
      "logps/rejected": -217.5677490234375,
      "loss": 1624.3861,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10012678802013397,
      "rewards/margins": 0.17437371611595154,
      "rewards/rejected": -0.2745005190372467,
      "step": 10870
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1615902625591926e-06,
      "logits/chosen": -2.244960308074951,
      "logits/rejected": -2.16121244430542,
      "logps/chosen": -243.59188842773438,
      "logps/rejected": -248.5153350830078,
      "loss": 1472.7349,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1837068498134613,
      "rewards/margins": 0.14337095618247986,
      "rewards/rejected": -0.32707780599594116,
      "step": 10880
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.156771044566738e-06,
      "logits/chosen": -2.3289358615875244,
      "logits/rejected": -2.2567138671875,
      "logps/chosen": -279.33380126953125,
      "logps/rejected": -250.97756958007812,
      "loss": 1849.6008,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.14719167351722717,
      "rewards/margins": 0.14861159026622772,
      "rewards/rejected": -0.2958032786846161,
      "step": 10890
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1519588335145037e-06,
      "logits/chosen": -2.284789800643921,
      "logits/rejected": -2.4238369464874268,
      "logps/chosen": -221.6291961669922,
      "logps/rejected": -253.0560302734375,
      "loss": 1870.9098,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14050407707691193,
      "rewards/margins": 0.12727408111095428,
      "rewards/rejected": -0.2677781581878662,
      "step": 10900
    },
    {
      "epoch": 0.71,
      "eval_logits/chosen": -2.345259189605713,
      "eval_logits/rejected": -2.1578006744384766,
      "eval_logps/chosen": -244.84068298339844,
      "eval_logps/rejected": -240.75035095214844,
      "eval_loss": 1764.2947998046875,
      "eval_rewards/accuracies": 0.6620000004768372,
      "eval_rewards/chosen": -0.12835733592510223,
      "eval_rewards/margins": 0.16302713751792908,
      "eval_rewards/rejected": -0.29138442873954773,
      "eval_runtime": 723.9578,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.381,
      "step": 10900
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1471536545053382e-06,
      "logits/chosen": -2.3750205039978027,
      "logits/rejected": -2.3602657318115234,
      "logps/chosen": -217.36862182617188,
      "logps/rejected": -239.7703857421875,
      "loss": 1909.0926,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08333705365657806,
      "rewards/margins": 0.14849582314491272,
      "rewards/rejected": -0.23183289170265198,
      "step": 10910
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1423555326054112e-06,
      "logits/chosen": -2.281400442123413,
      "logits/rejected": -2.011650562286377,
      "logps/chosen": -295.34649658203125,
      "logps/rejected": -246.96923828125,
      "loss": 1257.3419,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.12511949241161346,
      "rewards/margins": 0.2505880296230316,
      "rewards/rejected": -0.3757075369358063,
      "step": 10920
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1375644928440743e-06,
      "logits/chosen": -2.4245126247406006,
      "logits/rejected": -1.997741937637329,
      "logps/chosen": -246.53213500976562,
      "logps/rejected": -196.3480987548828,
      "loss": 1607.6604,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0984887033700943,
      "rewards/margins": 0.17319846153259277,
      "rewards/rejected": -0.2716871201992035,
      "step": 10930
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1327805602137396e-06,
      "logits/chosen": -2.355712413787842,
      "logits/rejected": -2.1805901527404785,
      "logps/chosen": -270.1285400390625,
      "logps/rejected": -235.70108032226562,
      "loss": 1566.6525,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.11384463310241699,
      "rewards/margins": 0.18550507724285126,
      "rewards/rejected": -0.29934969544410706,
      "step": 10940
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1280037596697426e-06,
      "logits/chosen": -2.211615562438965,
      "logits/rejected": -2.1793317794799805,
      "logps/chosen": -237.2572784423828,
      "logps/rejected": -293.44342041015625,
      "loss": 1663.5357,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.16374829411506653,
      "rewards/margins": 0.1925935000181198,
      "rewards/rejected": -0.35634177923202515,
      "step": 10950
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.123234116130216e-06,
      "logits/chosen": -2.2696049213409424,
      "logits/rejected": -2.2383406162261963,
      "logps/chosen": -202.56617736816406,
      "logps/rejected": -223.7290802001953,
      "loss": 2012.6854,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16677790880203247,
      "rewards/margins": 0.15108618140220642,
      "rewards/rejected": -0.3178640902042389,
      "step": 10960
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1184716544759553e-06,
      "logits/chosen": -2.1331257820129395,
      "logits/rejected": -2.110107183456421,
      "logps/chosen": -176.7662811279297,
      "logps/rejected": -200.56005859375,
      "loss": 1832.0801,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11402100324630737,
      "rewards/margins": 0.11379198729991913,
      "rewards/rejected": -0.2278130054473877,
      "step": 10970
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1137163995502948e-06,
      "logits/chosen": -2.558546781539917,
      "logits/rejected": -2.3269588947296143,
      "logps/chosen": -231.333740234375,
      "logps/rejected": -213.76669311523438,
      "loss": 1781.4242,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09798694401979446,
      "rewards/margins": 0.15134866535663605,
      "rewards/rejected": -0.24933557212352753,
      "step": 10980
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1089683761589717e-06,
      "logits/chosen": -2.1864194869995117,
      "logits/rejected": -2.0251612663269043,
      "logps/chosen": -249.9845733642578,
      "logps/rejected": -255.09475708007812,
      "loss": 1544.3908,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09778235107660294,
      "rewards/margins": 0.2402438372373581,
      "rewards/rejected": -0.338026225566864,
      "step": 10990
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1042276090700044e-06,
      "logits/chosen": -2.3458712100982666,
      "logits/rejected": -2.284625291824341,
      "logps/chosen": -229.9285125732422,
      "logps/rejected": -269.467529296875,
      "loss": 2322.8574,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.1962636113166809,
      "rewards/margins": 0.0936969667673111,
      "rewards/rejected": -0.2899605631828308,
      "step": 11000
    },
    {
      "epoch": 0.72,
      "eval_logits/chosen": -2.339794874191284,
      "eval_logits/rejected": -2.152827262878418,
      "eval_logps/chosen": -245.14317321777344,
      "eval_logps/rejected": -240.09378051757812,
      "eval_loss": 1763.4559326171875,
      "eval_rewards/accuracies": 0.6639999747276306,
      "eval_rewards/chosen": -0.1313822716474533,
      "eval_rewards/margins": 0.15343663096427917,
      "eval_rewards/rejected": -0.2848189175128937,
      "eval_runtime": 724.1879,
      "eval_samples_per_second": 2.762,
      "eval_steps_per_second": 1.381,
      "step": 11000
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0994941230135536e-06,
      "logits/chosen": -2.305234909057617,
      "logits/rejected": -2.010282516479492,
      "logps/chosen": -241.6505889892578,
      "logps/rejected": -231.662841796875,
      "loss": 1076.5374,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08800555020570755,
      "rewards/margins": 0.26072031259536743,
      "rewards/rejected": -0.3487258553504944,
      "step": 11010
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.094767942681804e-06,
      "logits/chosen": -2.582350254058838,
      "logits/rejected": -2.1599652767181396,
      "logps/chosen": -269.214599609375,
      "logps/rejected": -243.85092163085938,
      "loss": 1959.1641,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2134559601545334,
      "rewards/margins": 0.1383691430091858,
      "rewards/rejected": -0.35182514786720276,
      "step": 11020
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0900490927288248e-06,
      "logits/chosen": -2.12015700340271,
      "logits/rejected": -2.0761804580688477,
      "logps/chosen": -274.0080261230469,
      "logps/rejected": -238.61367797851562,
      "loss": 1776.8357,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13539735972881317,
      "rewards/margins": 0.1408197581768036,
      "rewards/rejected": -0.27621710300445557,
      "step": 11030
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0853375977704511e-06,
      "logits/chosen": -2.343122959136963,
      "logits/rejected": -2.161822557449341,
      "logps/chosen": -256.9599304199219,
      "logps/rejected": -207.81494140625,
      "loss": 1541.3028,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.16535378992557526,
      "rewards/margins": 0.14865735173225403,
      "rewards/rejected": -0.3140111565589905,
      "step": 11040
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0806334823841466e-06,
      "logits/chosen": -2.193089246749878,
      "logits/rejected": -2.2982065677642822,
      "logps/chosen": -257.24639892578125,
      "logps/rejected": -295.80499267578125,
      "loss": 2358.9592,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.17630498111248016,
      "rewards/margins": 0.09155434370040894,
      "rewards/rejected": -0.2678592801094055,
      "step": 11050
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0759367711088825e-06,
      "logits/chosen": -2.201061487197876,
      "logits/rejected": -2.335559368133545,
      "logps/chosen": -199.49998474121094,
      "logps/rejected": -242.69033813476562,
      "loss": 1907.6775,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15667478740215302,
      "rewards/margins": 0.09713147580623627,
      "rewards/rejected": -0.25380629301071167,
      "step": 11060
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0712474884450056e-06,
      "logits/chosen": -2.3094496726989746,
      "logits/rejected": -2.110459566116333,
      "logps/chosen": -214.3277587890625,
      "logps/rejected": -203.27700805664062,
      "loss": 1435.9635,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11494525521993637,
      "rewards/margins": 0.17736020684242249,
      "rewards/rejected": -0.29230546951293945,
      "step": 11070
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.066565658854112e-06,
      "logits/chosen": -2.2599921226501465,
      "logits/rejected": -2.2243447303771973,
      "logps/chosen": -142.46395874023438,
      "logps/rejected": -153.2086181640625,
      "loss": 1846.8561,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.14632046222686768,
      "rewards/margins": 0.1397717297077179,
      "rewards/rejected": -0.2860921621322632,
      "step": 11080
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0618913067589165e-06,
      "logits/chosen": -2.3570351600646973,
      "logits/rejected": -2.141935348510742,
      "logps/chosen": -228.3729248046875,
      "logps/rejected": -207.08169555664062,
      "loss": 1559.0029,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12558218836784363,
      "rewards/margins": 0.16168659925460815,
      "rewards/rejected": -0.2872687876224518,
      "step": 11090
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0572244565431313e-06,
      "logits/chosen": -2.2187297344207764,
      "logits/rejected": -2.1026558876037598,
      "logps/chosen": -161.37313842773438,
      "logps/rejected": -179.53013610839844,
      "loss": 1666.5447,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.15956351161003113,
      "rewards/margins": 0.12995785474777222,
      "rewards/rejected": -0.28952136635780334,
      "step": 11100
    },
    {
      "epoch": 0.73,
      "eval_logits/chosen": -2.3374075889587402,
      "eval_logits/rejected": -2.1504600048065186,
      "eval_logps/chosen": -245.01600646972656,
      "eval_logps/rejected": -240.04672241210938,
      "eval_loss": 1763.682861328125,
      "eval_rewards/accuracies": 0.6589999794960022,
      "eval_rewards/chosen": -0.13011065125465393,
      "eval_rewards/margins": 0.15423738956451416,
      "eval_rewards/rejected": -0.2843480408191681,
      "eval_runtime": 725.2445,
      "eval_samples_per_second": 2.758,
      "eval_steps_per_second": 1.379,
      "step": 11100
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0525651325513317e-06,
      "logits/chosen": -2.3347012996673584,
      "logits/rejected": -2.3130810260772705,
      "logps/chosen": -344.20343017578125,
      "logps/rejected": -334.12054443359375,
      "loss": 2193.3535,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12933756411075592,
      "rewards/margins": 0.11072547733783722,
      "rewards/rejected": -0.24006302654743195,
      "step": 11110
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0479133590888351e-06,
      "logits/chosen": -2.378882884979248,
      "logits/rejected": -2.0841665267944336,
      "logps/chosen": -268.0919494628906,
      "logps/rejected": -256.08404541015625,
      "loss": 1605.4642,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12636981904506683,
      "rewards/margins": 0.149643212556839,
      "rewards/rejected": -0.276013046503067,
      "step": 11120
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0432691604215695e-06,
      "logits/chosen": -2.2541301250457764,
      "logits/rejected": -2.172335624694824,
      "logps/chosen": -244.57998657226562,
      "logps/rejected": -226.08059692382812,
      "loss": 1816.2936,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09533033519983292,
      "rewards/margins": 0.1321374773979187,
      "rewards/rejected": -0.22746777534484863,
      "step": 11130
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0386325607759515e-06,
      "logits/chosen": -2.268679141998291,
      "logits/rejected": -2.209555149078369,
      "logps/chosen": -199.47750854492188,
      "logps/rejected": -195.68783569335938,
      "loss": 1598.9104,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08087374269962311,
      "rewards/margins": 0.1628076583147049,
      "rewards/rejected": -0.2436813861131668,
      "step": 11140
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0340035843387544e-06,
      "logits/chosen": -2.3822929859161377,
      "logits/rejected": -2.054208278656006,
      "logps/chosen": -191.0309600830078,
      "logps/rejected": -177.69284057617188,
      "loss": 1895.2246,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11295398324728012,
      "rewards/margins": 0.14244724810123444,
      "rewards/rejected": -0.25540122389793396,
      "step": 11150
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0293822552569887e-06,
      "logits/chosen": -2.488461971282959,
      "logits/rejected": -2.1910617351531982,
      "logps/chosen": -264.654541015625,
      "logps/rejected": -224.4275360107422,
      "loss": 1848.1217,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.127706840634346,
      "rewards/margins": 0.15603476762771606,
      "rewards/rejected": -0.2837415933609009,
      "step": 11160
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0247685976377688e-06,
      "logits/chosen": -2.253060817718506,
      "logits/rejected": -2.084641695022583,
      "logps/chosen": -196.15762329101562,
      "logps/rejected": -173.33316040039062,
      "loss": 1799.7363,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.14472590386867523,
      "rewards/margins": 0.16006897389888763,
      "rewards/rejected": -0.30479490756988525,
      "step": 11170
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0201626355481939e-06,
      "logits/chosen": -2.4443888664245605,
      "logits/rejected": -2.1537623405456543,
      "logps/chosen": -229.23739624023438,
      "logps/rejected": -192.5585174560547,
      "loss": 1444.4172,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.13116887211799622,
      "rewards/margins": 0.16881008446216583,
      "rewards/rejected": -0.29997897148132324,
      "step": 11180
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0155643930152192e-06,
      "logits/chosen": -2.4378559589385986,
      "logits/rejected": -2.357971668243408,
      "logps/chosen": -286.31646728515625,
      "logps/rejected": -247.3133087158203,
      "loss": 1755.0889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13709110021591187,
      "rewards/margins": 0.1564275175333023,
      "rewards/rejected": -0.293518602848053,
      "step": 11190
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0109738940255286e-06,
      "logits/chosen": -2.270477056503296,
      "logits/rejected": -2.002401351928711,
      "logps/chosen": -227.4958953857422,
      "logps/rejected": -205.7374267578125,
      "loss": 1670.8051,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10513322055339813,
      "rewards/margins": 0.15522795915603638,
      "rewards/rejected": -0.2603611946105957,
      "step": 11200
    },
    {
      "epoch": 0.73,
      "eval_logits/chosen": -2.339367628097534,
      "eval_logits/rejected": -2.152101755142212,
      "eval_logps/chosen": -244.8194580078125,
      "eval_logps/rejected": -239.89862060546875,
      "eval_loss": 1761.804931640625,
      "eval_rewards/accuracies": 0.6635000109672546,
      "eval_rewards/chosen": -0.12814511358737946,
      "eval_rewards/margins": 0.1547221690416336,
      "eval_rewards/rejected": -0.28286728262901306,
      "eval_runtime": 724.1801,
      "eval_samples_per_second": 2.762,
      "eval_steps_per_second": 1.381,
      "step": 11200
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0063911625254155e-06,
      "logits/chosen": -2.3362460136413574,
      "logits/rejected": -2.194571018218994,
      "logps/chosen": -242.2017059326172,
      "logps/rejected": -260.4835510253906,
      "loss": 1616.4728,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0627903938293457,
      "rewards/margins": 0.17652447521686554,
      "rewards/rejected": -0.23931488394737244,
      "step": 11210
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0018162224206502e-06,
      "logits/chosen": -2.2498667240142822,
      "logits/rejected": -2.1582674980163574,
      "logps/chosen": -190.2393341064453,
      "logps/rejected": -212.4859161376953,
      "loss": 1616.1617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1576453149318695,
      "rewards/margins": 0.18536503612995148,
      "rewards/rejected": -0.3430103659629822,
      "step": 11220
    },
    {
      "epoch": 0.73,
      "learning_rate": 9.97249097576363e-07,
      "logits/chosen": -2.490760564804077,
      "logits/rejected": -2.2085318565368652,
      "logps/chosen": -248.25839233398438,
      "logps/rejected": -224.3931427001953,
      "loss": 1423.3826,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15201839804649353,
      "rewards/margins": 0.19626560807228088,
      "rewards/rejected": -0.3482840359210968,
      "step": 11230
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.92689811816913e-07,
      "logits/chosen": -2.3705766201019287,
      "logits/rejected": -2.1356749534606934,
      "logps/chosen": -227.1351776123047,
      "logps/rejected": -194.965087890625,
      "loss": 1767.498,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.14614000916481018,
      "rewards/margins": 0.15232881903648376,
      "rewards/rejected": -0.29846885800361633,
      "step": 11240
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.881383889257691e-07,
      "logits/chosen": -2.307352066040039,
      "logits/rejected": -2.3431947231292725,
      "logps/chosen": -183.90199279785156,
      "logps/rejected": -261.51580810546875,
      "loss": 2043.1543,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.14605112373828888,
      "rewards/margins": 0.1353534311056137,
      "rewards/rejected": -0.2814045548439026,
      "step": 11250
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.835948526453817e-07,
      "logits/chosen": -2.1781134605407715,
      "logits/rejected": -2.3438239097595215,
      "logps/chosen": -197.26766967773438,
      "logps/rejected": -252.9156036376953,
      "loss": 1805.983,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1683829426765442,
      "rewards/margins": 0.11976154148578644,
      "rewards/rejected": -0.28814446926116943,
      "step": 11260
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.790592266770633e-07,
      "logits/chosen": -2.52354097366333,
      "logits/rejected": -2.236750364303589,
      "logps/chosen": -280.80609130859375,
      "logps/rejected": -268.2882080078125,
      "loss": 1571.1008,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11461339890956879,
      "rewards/margins": 0.15720847249031067,
      "rewards/rejected": -0.27182191610336304,
      "step": 11270
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.745315346808584e-07,
      "logits/chosen": -2.21311616897583,
      "logits/rejected": -2.0581233501434326,
      "logps/chosen": -229.4313201904297,
      "logps/rejected": -219.256103515625,
      "loss": 1922.0895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10355854034423828,
      "rewards/margins": 0.12207909673452377,
      "rewards/rejected": -0.22563764452934265,
      "step": 11280
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.70011800275428e-07,
      "logits/chosen": -2.2656772136688232,
      "logits/rejected": -2.193603038787842,
      "logps/chosen": -253.6857452392578,
      "logps/rejected": -281.72552490234375,
      "loss": 1775.9613,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16160854697227478,
      "rewards/margins": 0.16593758761882782,
      "rewards/rejected": -0.3275461792945862,
      "step": 11290
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.655000470379206e-07,
      "logits/chosen": -2.1882381439208984,
      "logits/rejected": -2.108506679534912,
      "logps/chosen": -224.51443481445312,
      "logps/rejected": -231.692626953125,
      "loss": 1693.2752,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.14166946709156036,
      "rewards/margins": 0.17860746383666992,
      "rewards/rejected": -0.3202769160270691,
      "step": 11300
    },
    {
      "epoch": 0.74,
      "eval_logits/chosen": -2.3482959270477295,
      "eval_logits/rejected": -2.1602466106414795,
      "eval_logps/chosen": -244.63011169433594,
      "eval_logps/rejected": -239.97296142578125,
      "eval_loss": 1757.552001953125,
      "eval_rewards/accuracies": 0.6650000214576721,
      "eval_rewards/chosen": -0.12625156342983246,
      "eval_rewards/margins": 0.1573590487241745,
      "eval_rewards/rejected": -0.28361061215400696,
      "eval_runtime": 722.233,
      "eval_samples_per_second": 2.769,
      "eval_steps_per_second": 1.385,
      "step": 11300
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.609962985038517e-07,
      "logits/chosen": -2.4610273838043213,
      "logits/rejected": -2.0612757205963135,
      "logps/chosen": -228.94839477539062,
      "logps/rejected": -225.9475555419922,
      "loss": 1165.8479,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1358734667301178,
      "rewards/margins": 0.22124087810516357,
      "rewards/rejected": -0.35711434483528137,
      "step": 11310
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.565005781669786e-07,
      "logits/chosen": -2.5059690475463867,
      "logits/rejected": -2.1450695991516113,
      "logps/chosen": -270.55938720703125,
      "logps/rejected": -235.69613647460938,
      "loss": 1448.8127,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.09968758374452591,
      "rewards/margins": 0.18449628353118896,
      "rewards/rejected": -0.2841838300228119,
      "step": 11320
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.520129094791822e-07,
      "logits/chosen": -2.3138279914855957,
      "logits/rejected": -2.1456973552703857,
      "logps/chosen": -196.9590301513672,
      "logps/rejected": -200.0696258544922,
      "loss": 1985.784,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.17463277280330658,
      "rewards/margins": 0.13313862681388855,
      "rewards/rejected": -0.30777138471603394,
      "step": 11330
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.475333158503389e-07,
      "logits/chosen": -2.304564952850342,
      "logits/rejected": -2.009404420852661,
      "logps/chosen": -224.3502960205078,
      "logps/rejected": -195.86581420898438,
      "loss": 1605.6322,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06528019905090332,
      "rewards/margins": 0.16674073040485382,
      "rewards/rejected": -0.23202089965343475,
      "step": 11340
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.430618206482053e-07,
      "logits/chosen": -2.279343843460083,
      "logits/rejected": -2.2244575023651123,
      "logps/chosen": -149.1036376953125,
      "logps/rejected": -159.86256408691406,
      "loss": 2122.7781,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08680722862482071,
      "rewards/margins": 0.10163422673940659,
      "rewards/rejected": -0.1884414702653885,
      "step": 11350
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.385984471982892e-07,
      "logits/chosen": -2.270360231399536,
      "logits/rejected": -1.9002015590667725,
      "logps/chosen": -225.63143920898438,
      "logps/rejected": -198.7368927001953,
      "loss": 1430.2678,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1080559641122818,
      "rewards/margins": 0.23840036988258362,
      "rewards/rejected": -0.3464563190937042,
      "step": 11360
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.341432187837343e-07,
      "logits/chosen": -2.315551280975342,
      "logits/rejected": -2.2298500537872314,
      "logps/chosen": -200.61843872070312,
      "logps/rejected": -233.1428680419922,
      "loss": 1634.4435,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.10314662754535675,
      "rewards/margins": 0.17656059563159943,
      "rewards/rejected": -0.27970725297927856,
      "step": 11370
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.29696158645193e-07,
      "logits/chosen": -2.2632083892822266,
      "logits/rejected": -2.3652756214141846,
      "logps/chosen": -232.05648803710938,
      "logps/rejected": -283.6464538574219,
      "loss": 1160.1338,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10954240709543228,
      "rewards/margins": 0.2260010689496994,
      "rewards/rejected": -0.33554351329803467,
      "step": 11380
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.252572899807111e-07,
      "logits/chosen": -2.323512554168701,
      "logits/rejected": -2.353520154953003,
      "logps/chosen": -282.01727294921875,
      "logps/rejected": -273.61907958984375,
      "loss": 1428.1299,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1141771674156189,
      "rewards/margins": 0.17788676917552948,
      "rewards/rejected": -0.2920639216899872,
      "step": 11390
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.208266359456003e-07,
      "logits/chosen": -2.4023215770721436,
      "logits/rejected": -2.1396372318267822,
      "logps/chosen": -213.10205078125,
      "logps/rejected": -237.35195922851562,
      "loss": 1789.823,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07296271622180939,
      "rewards/margins": 0.1467919498682022,
      "rewards/rejected": -0.2197546660900116,
      "step": 11400
    },
    {
      "epoch": 0.75,
      "eval_logits/chosen": -2.3544914722442627,
      "eval_logits/rejected": -2.165926456451416,
      "eval_logps/chosen": -243.65545654296875,
      "eval_logps/rejected": -239.7627410888672,
      "eval_loss": 1758.207275390625,
      "eval_rewards/accuracies": 0.6700000166893005,
      "eval_rewards/chosen": -0.11650507152080536,
      "eval_rewards/margins": 0.16500335931777954,
      "eval_rewards/rejected": -0.2815084457397461,
      "eval_runtime": 723.912,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.381,
      "step": 11400
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.164042196523229e-07,
      "logits/chosen": -2.519958257675171,
      "logits/rejected": -2.2136592864990234,
      "logps/chosen": -202.1793212890625,
      "logps/rejected": -207.7535858154297,
      "loss": 1546.1566,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12778282165527344,
      "rewards/margins": 0.1909192055463791,
      "rewards/rejected": -0.3187020719051361,
      "step": 11410
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.119900641703696e-07,
      "logits/chosen": -2.4855868816375732,
      "logits/rejected": -2.2158429622650146,
      "logps/chosen": -231.71017456054688,
      "logps/rejected": -198.54025268554688,
      "loss": 1916.1539,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.18263797461986542,
      "rewards/margins": 0.13165007531642914,
      "rewards/rejected": -0.31428807973861694,
      "step": 11420
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.075841925261364e-07,
      "logits/chosen": -2.577622890472412,
      "logits/rejected": -2.3363146781921387,
      "logps/chosen": -245.819091796875,
      "logps/rejected": -251.2655029296875,
      "loss": 1526.2436,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07899948209524155,
      "rewards/margins": 0.19875195622444153,
      "rewards/rejected": -0.2777514159679413,
      "step": 11430
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.031866277028093e-07,
      "logits/chosen": -2.2783894538879395,
      "logits/rejected": -2.310218095779419,
      "logps/chosen": -201.66424560546875,
      "logps/rejected": -245.3211669921875,
      "loss": 1731.3203,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10589464008808136,
      "rewards/margins": 0.1684727668762207,
      "rewards/rejected": -0.27436739206314087,
      "step": 11440
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.987973926402391e-07,
      "logits/chosen": -2.24288010597229,
      "logits/rejected": -2.2750463485717773,
      "logps/chosen": -219.508544921875,
      "logps/rejected": -234.1844482421875,
      "loss": 1930.043,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07168947160243988,
      "rewards/margins": 0.15263313055038452,
      "rewards/rejected": -0.2243226021528244,
      "step": 11450
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.944165102348273e-07,
      "logits/chosen": -2.4590885639190674,
      "logits/rejected": -2.307673931121826,
      "logps/chosen": -164.0479278564453,
      "logps/rejected": -186.90988159179688,
      "loss": 1636.5105,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0710080936551094,
      "rewards/margins": 0.1560458242893219,
      "rewards/rejected": -0.2270539104938507,
      "step": 11460
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.900440033394018e-07,
      "logits/chosen": -2.278036594390869,
      "logits/rejected": -2.3274552822113037,
      "logps/chosen": -192.73841857910156,
      "logps/rejected": -194.4464874267578,
      "loss": 1998.9707,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.10222195088863373,
      "rewards/margins": 0.10993766784667969,
      "rewards/rejected": -0.2121596336364746,
      "step": 11470
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.856798947631009e-07,
      "logits/chosen": -2.2997279167175293,
      "logits/rejected": -2.3228790760040283,
      "logps/chosen": -197.69215393066406,
      "logps/rejected": -238.3724822998047,
      "loss": 1416.2062,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06816434860229492,
      "rewards/margins": 0.23728346824645996,
      "rewards/rejected": -0.3054478168487549,
      "step": 11480
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.813242072712519e-07,
      "logits/chosen": -2.0630974769592285,
      "logits/rejected": -1.944562554359436,
      "logps/chosen": -177.80255126953125,
      "logps/rejected": -187.6953887939453,
      "loss": 2238.0873,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.15940259397029877,
      "rewards/margins": 0.08756790310144424,
      "rewards/rejected": -0.24697045981884003,
      "step": 11490
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.769769635852557e-07,
      "logits/chosen": -2.2842228412628174,
      "logits/rejected": -2.3433406352996826,
      "logps/chosen": -224.0996856689453,
      "logps/rejected": -202.60337829589844,
      "loss": 1808.4945,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08791881799697876,
      "rewards/margins": 0.15260782837867737,
      "rewards/rejected": -0.24052663147449493,
      "step": 11500
    },
    {
      "epoch": 0.75,
      "eval_logits/chosen": -2.357572555541992,
      "eval_logits/rejected": -2.168738842010498,
      "eval_logps/chosen": -241.220458984375,
      "eval_logps/rejected": -236.46685791015625,
      "eval_loss": 1761.228271484375,
      "eval_rewards/accuracies": 0.6664999723434448,
      "eval_rewards/chosen": -0.09215506911277771,
      "eval_rewards/margins": 0.1563943475484848,
      "eval_rewards/rejected": -0.24854940176010132,
      "eval_runtime": 725.395,
      "eval_samples_per_second": 2.757,
      "eval_steps_per_second": 1.379,
      "step": 11500
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.726381863824635e-07,
      "logits/chosen": -2.509023427963257,
      "logits/rejected": -2.177807331085205,
      "logps/chosen": -294.0090026855469,
      "logps/rejected": -243.0245819091797,
      "loss": 1735.5961,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08931988477706909,
      "rewards/margins": 0.16421779990196228,
      "rewards/rejected": -0.25353771448135376,
      "step": 11510
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.683078982960638e-07,
      "logits/chosen": -2.2652676105499268,
      "logits/rejected": -1.9574466943740845,
      "logps/chosen": -235.6822967529297,
      "logps/rejected": -206.75399780273438,
      "loss": 1402.3072,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10997311770915985,
      "rewards/margins": 0.21162648499011993,
      "rewards/rejected": -0.3215996026992798,
      "step": 11520
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.639861219149584e-07,
      "logits/chosen": -2.1088919639587402,
      "logits/rejected": -2.1795811653137207,
      "logps/chosen": -275.09857177734375,
      "logps/rejected": -247.9165802001953,
      "loss": 1866.335,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11116518825292587,
      "rewards/margins": 0.15742537379264832,
      "rewards/rejected": -0.2685905694961548,
      "step": 11530
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.596728797836532e-07,
      "logits/chosen": -2.247396945953369,
      "logits/rejected": -2.0930850505828857,
      "logps/chosen": -227.16415405273438,
      "logps/rejected": -273.5677795410156,
      "loss": 1706.7742,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11855418980121613,
      "rewards/margins": 0.1681063324213028,
      "rewards/rejected": -0.2866605222225189,
      "step": 11540
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.553681944021294e-07,
      "logits/chosen": -2.285822629928589,
      "logits/rejected": -2.3091416358947754,
      "logps/chosen": -248.54736328125,
      "logps/rejected": -248.06689453125,
      "loss": 1681.1939,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09463384002447128,
      "rewards/margins": 0.18866769969463348,
      "rewards/rejected": -0.28330153226852417,
      "step": 11550
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.510720882257365e-07,
      "logits/chosen": -2.0751852989196777,
      "logits/rejected": -2.222693681716919,
      "logps/chosen": -174.25979614257812,
      "logps/rejected": -230.66012573242188,
      "loss": 1568.4097,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12959204614162445,
      "rewards/margins": 0.1516839563846588,
      "rewards/rejected": -0.28127604722976685,
      "step": 11560
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.467845836650667e-07,
      "logits/chosen": -1.962531328201294,
      "logits/rejected": -2.0121817588806152,
      "logps/chosen": -220.4276123046875,
      "logps/rejected": -242.0430908203125,
      "loss": 1793.5357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1216101422905922,
      "rewards/margins": 0.15060429275035858,
      "rewards/rejected": -0.27221447229385376,
      "step": 11570
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.425057030858461e-07,
      "logits/chosen": -2.1566555500030518,
      "logits/rejected": -1.988985300064087,
      "logps/chosen": -167.70249938964844,
      "logps/rejected": -210.2772979736328,
      "loss": 1842.1486,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.11646603047847748,
      "rewards/margins": 0.13551898300647736,
      "rewards/rejected": -0.2519850432872772,
      "step": 11580
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.382354688088098e-07,
      "logits/chosen": -2.3145182132720947,
      "logits/rejected": -2.170231580734253,
      "logps/chosen": -167.34896850585938,
      "logps/rejected": -197.62222290039062,
      "loss": 1690.5807,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0919358879327774,
      "rewards/margins": 0.14850333333015442,
      "rewards/rejected": -0.24043922126293182,
      "step": 11590
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.33973903109594e-07,
      "logits/chosen": -2.4262478351593018,
      "logits/rejected": -2.16702938079834,
      "logps/chosen": -229.05691528320312,
      "logps/rejected": -215.0450439453125,
      "loss": 1721.7291,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1245979517698288,
      "rewards/margins": 0.1754973977804184,
      "rewards/rejected": -0.3000953793525696,
      "step": 11600
    },
    {
      "epoch": 0.76,
      "eval_logits/chosen": -2.3435888290405273,
      "eval_logits/rejected": -2.1559693813323975,
      "eval_logps/chosen": -244.8027801513672,
      "eval_logps/rejected": -239.546630859375,
      "eval_loss": 1762.330322265625,
      "eval_rewards/accuracies": 0.6614999771118164,
      "eval_rewards/chosen": -0.12797829508781433,
      "eval_rewards/margins": 0.15136903524398804,
      "eval_rewards/rejected": -0.27934733033180237,
      "eval_runtime": 725.919,
      "eval_samples_per_second": 2.755,
      "eval_steps_per_second": 1.378,
      "step": 11600
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.297210282186102e-07,
      "logits/chosen": -2.189514398574829,
      "logits/rejected": -2.1581497192382812,
      "logps/chosen": -252.0922088623047,
      "logps/rejected": -284.6585388183594,
      "loss": 2117.5762,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2207474261522293,
      "rewards/margins": 0.11199697107076645,
      "rewards/rejected": -0.33274438977241516,
      "step": 11610
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.254768663209397e-07,
      "logits/chosen": -2.284914255142212,
      "logits/rejected": -2.078519821166992,
      "logps/chosen": -289.47747802734375,
      "logps/rejected": -241.3529815673828,
      "loss": 1649.8689,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10897673666477203,
      "rewards/margins": 0.1434708684682846,
      "rewards/rejected": -0.25244760513305664,
      "step": 11620
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.212414395562079e-07,
      "logits/chosen": -2.1276698112487793,
      "logits/rejected": -2.2639830112457275,
      "logps/chosen": -247.23141479492188,
      "logps/rejected": -283.796875,
      "loss": 1854.4729,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17017242312431335,
      "rewards/margins": 0.1264459639787674,
      "rewards/rejected": -0.29661840200424194,
      "step": 11630
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.170147700184775e-07,
      "logits/chosen": -2.3277761936187744,
      "logits/rejected": -2.2101144790649414,
      "logps/chosen": -268.25653076171875,
      "logps/rejected": -278.6501770019531,
      "loss": 1479.7791,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.11805230379104614,
      "rewards/margins": 0.19326400756835938,
      "rewards/rejected": -0.3113163113594055,
      "step": 11640
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.127968797561242e-07,
      "logits/chosen": -2.3800370693206787,
      "logits/rejected": -2.11928653717041,
      "logps/chosen": -241.4968719482422,
      "logps/rejected": -239.001953125,
      "loss": 1479.4,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17526161670684814,
      "rewards/margins": 0.19928061962127686,
      "rewards/rejected": -0.374542236328125,
      "step": 11650
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.085877907717338e-07,
      "logits/chosen": -2.2742931842803955,
      "logits/rejected": -2.2385847568511963,
      "logps/chosen": -231.2503204345703,
      "logps/rejected": -236.61669921875,
      "loss": 1669.0566,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10338792949914932,
      "rewards/margins": 0.17045649886131287,
      "rewards/rejected": -0.2738444209098816,
      "step": 11660
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.043875250219732e-07,
      "logits/chosen": -2.228330612182617,
      "logits/rejected": -2.1692111492156982,
      "logps/chosen": -246.3470001220703,
      "logps/rejected": -234.47000122070312,
      "loss": 1991.2852,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.16942265629768372,
      "rewards/margins": 0.0919186919927597,
      "rewards/rejected": -0.2613413631916046,
      "step": 11670
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.001961044174881e-07,
      "logits/chosen": -2.3931221961975098,
      "logits/rejected": -2.193916082382202,
      "logps/chosen": -243.92312622070312,
      "logps/rejected": -198.2577667236328,
      "loss": 2045.4062,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.15458671748638153,
      "rewards/margins": 0.11522801220417023,
      "rewards/rejected": -0.26981475949287415,
      "step": 11680
    },
    {
      "epoch": 0.76,
      "learning_rate": 7.960135508227795e-07,
      "logits/chosen": -2.405076503753662,
      "logits/rejected": -2.0538601875305176,
      "logps/chosen": -308.1584167480469,
      "logps/rejected": -259.13482666015625,
      "loss": 1822.1654,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.14289391040802002,
      "rewards/margins": 0.15292389690876007,
      "rewards/rejected": -0.2958178222179413,
      "step": 11690
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.91839886056098e-07,
      "logits/chosen": -2.420992851257324,
      "logits/rejected": -2.168729782104492,
      "logps/chosen": -296.9039001464844,
      "logps/rejected": -284.8421630859375,
      "loss": 1471.2858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1314167082309723,
      "rewards/margins": 0.18508975207805634,
      "rewards/rejected": -0.3165064752101898,
      "step": 11700
    },
    {
      "epoch": 0.77,
      "eval_logits/chosen": -2.3449456691741943,
      "eval_logits/rejected": -2.157163143157959,
      "eval_logps/chosen": -245.83273315429688,
      "eval_logps/rejected": -240.5537872314453,
      "eval_loss": 1764.1798095703125,
      "eval_rewards/accuracies": 0.6610000133514404,
      "eval_rewards/chosen": -0.1382778286933899,
      "eval_rewards/margins": 0.1511407494544983,
      "eval_rewards/rejected": -0.2894185781478882,
      "eval_runtime": 723.7931,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.382,
      "step": 11700
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.876751318893217e-07,
      "logits/chosen": -2.246511936187744,
      "logits/rejected": -1.9968385696411133,
      "logps/chosen": -250.1975555419922,
      "logps/rejected": -242.7787628173828,
      "loss": 1715.1941,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12331908941268921,
      "rewards/margins": 0.14169330894947052,
      "rewards/rejected": -0.2650124132633209,
      "step": 11710
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.8351931004785e-07,
      "logits/chosen": -2.2341372966766357,
      "logits/rejected": -1.870633840560913,
      "logps/chosen": -224.34896850585938,
      "logps/rejected": -214.8955078125,
      "loss": 1687.4385,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15720701217651367,
      "rewards/margins": 0.15429864823818207,
      "rewards/rejected": -0.31150567531585693,
      "step": 11720
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.793724422104834e-07,
      "logits/chosen": -2.0860543251037598,
      "logits/rejected": -2.245222568511963,
      "logps/chosen": -222.11038208007812,
      "logps/rejected": -308.7958984375,
      "loss": 1587.7459,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1328606903553009,
      "rewards/margins": 0.17208316922187805,
      "rewards/rejected": -0.30494385957717896,
      "step": 11730
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.752345500093184e-07,
      "logits/chosen": -2.3890018463134766,
      "logits/rejected": -2.363795757293701,
      "logps/chosen": -229.0177764892578,
      "logps/rejected": -207.2950439453125,
      "loss": 2137.9793,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18862739205360413,
      "rewards/margins": 0.080417200922966,
      "rewards/rejected": -0.26904457807540894,
      "step": 11740
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.711056550296253e-07,
      "logits/chosen": -2.446387767791748,
      "logits/rejected": -2.2817203998565674,
      "logps/chosen": -254.32766723632812,
      "logps/rejected": -237.6660614013672,
      "loss": 1416.5906,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07533854246139526,
      "rewards/margins": 0.20363454520702362,
      "rewards/rejected": -0.2789730727672577,
      "step": 11750
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.669857788097445e-07,
      "logits/chosen": -2.1872541904449463,
      "logits/rejected": -1.9422680139541626,
      "logps/chosen": -177.9638671875,
      "logps/rejected": -220.3321075439453,
      "loss": 1753.0037,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.16397897899150848,
      "rewards/margins": 0.15743349492549896,
      "rewards/rejected": -0.32141250371932983,
      "step": 11760
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.628749428409676e-07,
      "logits/chosen": -2.4421238899230957,
      "logits/rejected": -2.0218122005462646,
      "logps/chosen": -252.5674285888672,
      "logps/rejected": -207.2790069580078,
      "loss": 2211.9254,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15679652988910675,
      "rewards/margins": 0.13463090360164642,
      "rewards/rejected": -0.2914274036884308,
      "step": 11770
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.587731685674288e-07,
      "logits/chosen": -2.3191511631011963,
      "logits/rejected": -2.3603625297546387,
      "logps/chosen": -286.06329345703125,
      "logps/rejected": -306.73150634765625,
      "loss": 1985.7795,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12475011497735977,
      "rewards/margins": 0.12831243872642517,
      "rewards/rejected": -0.25306254625320435,
      "step": 11780
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.546804773859931e-07,
      "logits/chosen": -2.428969144821167,
      "logits/rejected": -2.1997106075286865,
      "logps/chosen": -239.6063232421875,
      "logps/rejected": -237.9988555908203,
      "loss": 1582.8656,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12566278874874115,
      "rewards/margins": 0.18328523635864258,
      "rewards/rejected": -0.30894801020622253,
      "step": 11790
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.505968906461409e-07,
      "logits/chosen": -2.335308074951172,
      "logits/rejected": -2.1804909706115723,
      "logps/chosen": -255.17919921875,
      "logps/rejected": -248.5977325439453,
      "loss": 1792.252,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.14239093661308289,
      "rewards/margins": 0.17564840614795685,
      "rewards/rejected": -0.3180393576622009,
      "step": 11800
    },
    {
      "epoch": 0.77,
      "eval_logits/chosen": -2.356790781021118,
      "eval_logits/rejected": -2.1682629585266113,
      "eval_logps/chosen": -244.12054443359375,
      "eval_logps/rejected": -239.302001953125,
      "eval_loss": 1759.88671875,
      "eval_rewards/accuracies": 0.6669999957084656,
      "eval_rewards/chosen": -0.1211560070514679,
      "eval_rewards/margins": 0.15574483573436737,
      "eval_rewards/rejected": -0.27690085768699646,
      "eval_runtime": 724.6407,
      "eval_samples_per_second": 2.76,
      "eval_steps_per_second": 1.38,
      "step": 11800
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.465224296498627e-07,
      "logits/chosen": -2.4071779251098633,
      "logits/rejected": -2.0205078125,
      "logps/chosen": -243.91873168945312,
      "logps/rejected": -220.3428497314453,
      "loss": 1434.0618,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10959659516811371,
      "rewards/margins": 0.17553094029426575,
      "rewards/rejected": -0.28512755036354065,
      "step": 11810
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.424571156515412e-07,
      "logits/chosen": -2.2748305797576904,
      "logits/rejected": -2.2719333171844482,
      "logps/chosen": -189.68267822265625,
      "logps/rejected": -227.21694946289062,
      "loss": 1762.3602,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11977501958608627,
      "rewards/margins": 0.14857666194438934,
      "rewards/rejected": -0.268351674079895,
      "step": 11820
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.38400969857847e-07,
      "logits/chosen": -2.2314724922180176,
      "logits/rejected": -2.01545786857605,
      "logps/chosen": -202.58847045898438,
      "logps/rejected": -223.5933837890625,
      "loss": 1383.0405,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.19446060061454773,
      "rewards/margins": 0.17852742969989777,
      "rewards/rejected": -0.3729880452156067,
      "step": 11830
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.343540134276225e-07,
      "logits/chosen": -2.3522350788116455,
      "logits/rejected": -2.277493953704834,
      "logps/chosen": -180.2784881591797,
      "logps/rejected": -196.76797485351562,
      "loss": 1910.5359,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09566868841648102,
      "rewards/margins": 0.1459321677684784,
      "rewards/rejected": -0.24160084128379822,
      "step": 11840
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.303162674717762e-07,
      "logits/chosen": -2.3356125354766846,
      "logits/rejected": -1.941916823387146,
      "logps/chosen": -226.21249389648438,
      "logps/rejected": -181.681640625,
      "loss": 2040.407,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15984733402729034,
      "rewards/margins": 0.11304545402526855,
      "rewards/rejected": -0.2728927731513977,
      "step": 11850
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.26287753053167e-07,
      "logits/chosen": -2.286027431488037,
      "logits/rejected": -2.2168824672698975,
      "logps/chosen": -282.25958251953125,
      "logps/rejected": -296.0490417480469,
      "loss": 1808.9691,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1728345900774002,
      "rewards/margins": 0.15057656168937683,
      "rewards/rejected": -0.32341113686561584,
      "step": 11860
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.222684911865013e-07,
      "logits/chosen": -2.3952112197875977,
      "logits/rejected": -2.4059479236602783,
      "logps/chosen": -214.5403289794922,
      "logps/rejected": -256.82745361328125,
      "loss": 1377.1188,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07371652126312256,
      "rewards/margins": 0.20601694285869598,
      "rewards/rejected": -0.27973347902297974,
      "step": 11870
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.182585028382166e-07,
      "logits/chosen": -2.44596266746521,
      "logits/rejected": -2.1434316635131836,
      "logps/chosen": -281.9929504394531,
      "logps/rejected": -267.156982421875,
      "loss": 1530.4185,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.05841357633471489,
      "rewards/margins": 0.17613962292671204,
      "rewards/rejected": -0.23455317318439484,
      "step": 11880
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.142578089263769e-07,
      "logits/chosen": -2.4750990867614746,
      "logits/rejected": -2.1455719470977783,
      "logps/chosen": -327.4580078125,
      "logps/rejected": -288.38934326171875,
      "loss": 1571.5391,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09029410779476166,
      "rewards/margins": 0.19731321930885315,
      "rewards/rejected": -0.287607342004776,
      "step": 11890
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.102664303205611e-07,
      "logits/chosen": -2.355468511581421,
      "logits/rejected": -2.102905750274658,
      "logps/chosen": -234.4264373779297,
      "logps/rejected": -229.1504669189453,
      "loss": 2080.723,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1386391818523407,
      "rewards/margins": 0.1260056495666504,
      "rewards/rejected": -0.2646448314189911,
      "step": 11900
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.3557183742523193,
      "eval_logits/rejected": -2.1673061847686768,
      "eval_logps/chosen": -243.10972595214844,
      "eval_logps/rejected": -238.54348754882812,
      "eval_loss": 1758.6500244140625,
      "eval_rewards/accuracies": 0.6664999723434448,
      "eval_rewards/chosen": -0.11104759573936462,
      "eval_rewards/margins": 0.15826837718486786,
      "eval_rewards/rejected": -0.2693159878253937,
      "eval_runtime": 725.2021,
      "eval_samples_per_second": 2.758,
      "eval_steps_per_second": 1.379,
      "step": 11900
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.062843878417566e-07,
      "logits/chosen": -2.463575839996338,
      "logits/rejected": -2.331946849822998,
      "logps/chosen": -232.64248657226562,
      "logps/rejected": -213.59414672851562,
      "loss": 1912.4789,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09209892153739929,
      "rewards/margins": 0.13995857536792755,
      "rewards/rejected": -0.23205752670764923,
      "step": 11910
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.023117022622458e-07,
      "logits/chosen": -2.394233226776123,
      "logits/rejected": -2.0388636589050293,
      "logps/chosen": -254.5091094970703,
      "logps/rejected": -243.8262481689453,
      "loss": 1850.1949,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15993770956993103,
      "rewards/margins": 0.12585516273975372,
      "rewards/rejected": -0.28579288721084595,
      "step": 11920
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.983483943055042e-07,
      "logits/chosen": -2.2799057960510254,
      "logits/rejected": -2.101398229598999,
      "logps/chosen": -290.40399169921875,
      "logps/rejected": -246.39370727539062,
      "loss": 1913.1725,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09937360137701035,
      "rewards/margins": 0.12762802839279175,
      "rewards/rejected": -0.2270016223192215,
      "step": 11930
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.943944846460859e-07,
      "logits/chosen": -2.3366754055023193,
      "logits/rejected": -2.2716286182403564,
      "logps/chosen": -228.8292694091797,
      "logps/rejected": -195.3153839111328,
      "loss": 2120.9512,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.10074935853481293,
      "rewards/margins": 0.13228239119052887,
      "rewards/rejected": -0.2330317497253418,
      "step": 11940
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.904499939095225e-07,
      "logits/chosen": -2.298487663269043,
      "logits/rejected": -2.2476203441619873,
      "logps/chosen": -228.70315551757812,
      "logps/rejected": -232.03164672851562,
      "loss": 1719.4844,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06355978548526764,
      "rewards/margins": 0.18656757473945618,
      "rewards/rejected": -0.250127375125885,
      "step": 11950
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.865149426722079e-07,
      "logits/chosen": -2.273329257965088,
      "logits/rejected": -2.219085454940796,
      "logps/chosen": -285.75531005859375,
      "logps/rejected": -265.9485168457031,
      "loss": 2047.2496,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.14570143818855286,
      "rewards/margins": 0.10887260735034943,
      "rewards/rejected": -0.2545740604400635,
      "step": 11960
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.825893514612985e-07,
      "logits/chosen": -2.1099815368652344,
      "logits/rejected": -2.2191219329833984,
      "logps/chosen": -242.7589111328125,
      "logps/rejected": -258.30767822265625,
      "loss": 1666.5125,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.061920829117298126,
      "rewards/margins": 0.17732374370098114,
      "rewards/rejected": -0.23924458026885986,
      "step": 11970
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.786732407546001e-07,
      "logits/chosen": -2.126340866088867,
      "logits/rejected": -2.080836772918701,
      "logps/chosen": -208.6787109375,
      "logps/rejected": -183.4300994873047,
      "loss": 1800.6842,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07797534018754959,
      "rewards/margins": 0.14466950297355652,
      "rewards/rejected": -0.2226448357105255,
      "step": 11980
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.747666309804654e-07,
      "logits/chosen": -2.5057218074798584,
      "logits/rejected": -2.184866189956665,
      "logps/chosen": -296.4242858886719,
      "logps/rejected": -236.760498046875,
      "loss": 1271.4783,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08747060596942902,
      "rewards/margins": 0.1987169086933136,
      "rewards/rejected": -0.2861874997615814,
      "step": 11990
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.708695425176831e-07,
      "logits/chosen": -2.1408398151397705,
      "logits/rejected": -2.1481544971466064,
      "logps/chosen": -179.84515380859375,
      "logps/rejected": -217.2546844482422,
      "loss": 1576.5392,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10806410014629364,
      "rewards/margins": 0.16654148697853088,
      "rewards/rejected": -0.2746056020259857,
      "step": 12000
    },
    {
      "epoch": 0.79,
      "eval_logits/chosen": -2.3568789958953857,
      "eval_logits/rejected": -2.1684751510620117,
      "eval_logps/chosen": -240.02801513671875,
      "eval_logps/rejected": -235.32835388183594,
      "eval_loss": 1760.51953125,
      "eval_rewards/accuracies": 0.6669999957084656,
      "eval_rewards/chosen": -0.08023066818714142,
      "eval_rewards/margins": 0.15693405270576477,
      "eval_rewards/rejected": -0.237164705991745,
      "eval_runtime": 723.4723,
      "eval_samples_per_second": 2.764,
      "eval_steps_per_second": 1.382,
      "step": 12000
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.669819956953768e-07,
      "logits/chosen": -2.1999356746673584,
      "logits/rejected": -2.1322684288024902,
      "logps/chosen": -176.7624969482422,
      "logps/rejected": -200.78819274902344,
      "loss": 1586.7117,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06431283056735992,
      "rewards/margins": 0.15485699474811554,
      "rewards/rejected": -0.21916981041431427,
      "step": 12010
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.631040107928957e-07,
      "logits/chosen": -2.527143955230713,
      "logits/rejected": -2.213205337524414,
      "logps/chosen": -275.36004638671875,
      "logps/rejected": -196.37973022460938,
      "loss": 1398.0136,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07175728678703308,
      "rewards/margins": 0.17318230867385864,
      "rewards/rejected": -0.24493959546089172,
      "step": 12020
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.592356080397072e-07,
      "logits/chosen": -2.4240286350250244,
      "logits/rejected": -1.8601045608520508,
      "logps/chosen": -234.77880859375,
      "logps/rejected": -196.73764038085938,
      "loss": 1555.797,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06489615887403488,
      "rewards/margins": 0.18422222137451172,
      "rewards/rejected": -0.2491184026002884,
      "step": 12030
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.553768076152963e-07,
      "logits/chosen": -2.3208096027374268,
      "logits/rejected": -2.4190030097961426,
      "logps/chosen": -164.54904174804688,
      "logps/rejected": -212.4931182861328,
      "loss": 1571.5309,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0645146369934082,
      "rewards/margins": 0.20840032398700714,
      "rewards/rejected": -0.27291494607925415,
      "step": 12040
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.51527629649055e-07,
      "logits/chosen": -2.4525504112243652,
      "logits/rejected": -2.3045573234558105,
      "logps/chosen": -265.19659423828125,
      "logps/rejected": -245.7611846923828,
      "loss": 2120.6713,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11613516509532928,
      "rewards/margins": 0.09711629152297974,
      "rewards/rejected": -0.21325147151947021,
      "step": 12050
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.476880942201824e-07,
      "logits/chosen": -2.583015203475952,
      "logits/rejected": -2.1839451789855957,
      "logps/chosen": -243.401123046875,
      "logps/rejected": -204.27493286132812,
      "loss": 1704.0209,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.020634567365050316,
      "rewards/margins": 0.19498001039028168,
      "rewards/rejected": -0.21561458706855774,
      "step": 12060
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.438582213575748e-07,
      "logits/chosen": -2.275463819503784,
      "logits/rejected": -2.2460455894470215,
      "logps/chosen": -239.5929718017578,
      "logps/rejected": -266.6346740722656,
      "loss": 1807.3643,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09676406532526016,
      "rewards/margins": 0.15896211564540863,
      "rewards/rejected": -0.2557261884212494,
      "step": 12070
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.400380310397267e-07,
      "logits/chosen": -2.2111427783966064,
      "logits/rejected": -2.2207419872283936,
      "logps/chosen": -243.8424835205078,
      "logps/rejected": -285.3353271484375,
      "loss": 2440.1971,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11732114851474762,
      "rewards/margins": 0.10081054270267487,
      "rewards/rejected": -0.21813169121742249,
      "step": 12080
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.362275431946202e-07,
      "logits/chosen": -2.1953682899475098,
      "logits/rejected": -2.2317399978637695,
      "logps/chosen": -245.20437622070312,
      "logps/rejected": -265.2196960449219,
      "loss": 1881.7752,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08850773423910141,
      "rewards/margins": 0.13967140018939972,
      "rewards/rejected": -0.22817914187908173,
      "step": 12090
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.324267776996285e-07,
      "logits/chosen": -2.4136674404144287,
      "logits/rejected": -2.035423755645752,
      "logps/chosen": -382.706298828125,
      "logps/rejected": -284.49853515625,
      "loss": 1299.8462,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.08127235621213913,
      "rewards/margins": 0.23772704601287842,
      "rewards/rejected": -0.31899940967559814,
      "step": 12100
    },
    {
      "epoch": 0.79,
      "eval_logits/chosen": -2.3554248809814453,
      "eval_logits/rejected": -2.1673924922943115,
      "eval_logps/chosen": -239.875,
      "eval_logps/rejected": -234.68032836914062,
      "eval_loss": 1767.3917236328125,
      "eval_rewards/accuracies": 0.6579999923706055,
      "eval_rewards/chosen": -0.07870057225227356,
      "eval_rewards/margins": 0.15198378264904022,
      "eval_rewards/rejected": -0.23068435490131378,
      "eval_runtime": 725.0964,
      "eval_samples_per_second": 2.758,
      "eval_steps_per_second": 1.379,
      "step": 12100
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.286357543814045e-07,
      "logits/chosen": -2.2756481170654297,
      "logits/rejected": -2.1830382347106934,
      "logps/chosen": -200.79925537109375,
      "logps/rejected": -285.17669677734375,
      "loss": 1763.618,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07269416749477386,
      "rewards/margins": 0.1636621505022049,
      "rewards/rejected": -0.23635630309581757,
      "step": 12110
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.248544930157838e-07,
      "logits/chosen": -2.4011549949645996,
      "logits/rejected": -2.153775691986084,
      "logps/chosen": -191.3563232421875,
      "logps/rejected": -194.35769653320312,
      "loss": 1513.909,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07270742952823639,
      "rewards/margins": 0.2078186273574829,
      "rewards/rejected": -0.2805260419845581,
      "step": 12120
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.21083013327678e-07,
      "logits/chosen": -2.332559108734131,
      "logits/rejected": -2.230076789855957,
      "logps/chosen": -309.12188720703125,
      "logps/rejected": -265.01318359375,
      "loss": 1938.6658,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04881126806139946,
      "rewards/margins": 0.11817065626382828,
      "rewards/rejected": -0.16698193550109863,
      "step": 12130
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.17321334990973e-07,
      "logits/chosen": -2.297025203704834,
      "logits/rejected": -2.2214016914367676,
      "logps/chosen": -207.6863250732422,
      "logps/rejected": -186.2632598876953,
      "loss": 1931.7465,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06881825625896454,
      "rewards/margins": 0.13814252614974976,
      "rewards/rejected": -0.2069607973098755,
      "step": 12140
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.135694776284243e-07,
      "logits/chosen": -2.4541053771972656,
      "logits/rejected": -2.192342519760132,
      "logps/chosen": -277.95233154296875,
      "logps/rejected": -232.65432739257812,
      "loss": 1577.0223,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07161318510770798,
      "rewards/margins": 0.19474197924137115,
      "rewards/rejected": -0.2663551867008209,
      "step": 12150
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.098274608115595e-07,
      "logits/chosen": -2.2446258068084717,
      "logits/rejected": -2.1158833503723145,
      "logps/chosen": -211.1138153076172,
      "logps/rejected": -193.41287231445312,
      "loss": 1944.4771,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.05463189631700516,
      "rewards/margins": 0.10931450128555298,
      "rewards/rejected": -0.16394639015197754,
      "step": 12160
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.060953040605697e-07,
      "logits/chosen": -2.464473247528076,
      "logits/rejected": -1.9369499683380127,
      "logps/chosen": -342.2126770019531,
      "logps/rejected": -296.26025390625,
      "loss": 1398.6549,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.030829787254333496,
      "rewards/margins": 0.2394247055053711,
      "rewards/rejected": -0.2702544629573822,
      "step": 12170
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.023730268442144e-07,
      "logits/chosen": -2.2708048820495605,
      "logits/rejected": -2.09238338470459,
      "logps/chosen": -212.4055938720703,
      "logps/rejected": -203.7031707763672,
      "loss": 1372.8158,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07584762573242188,
      "rewards/margins": 0.19742408394813538,
      "rewards/rejected": -0.27327170968055725,
      "step": 12180
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.986606485797131e-07,
      "logits/chosen": -2.2674381732940674,
      "logits/rejected": -2.050218105316162,
      "logps/chosen": -212.3667449951172,
      "logps/rejected": -229.31253051757812,
      "loss": 1873.1357,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12476187944412231,
      "rewards/margins": 0.11879698187112808,
      "rewards/rejected": -0.2435588836669922,
      "step": 12190
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.949581886326511e-07,
      "logits/chosen": -2.3818600177764893,
      "logits/rejected": -2.3597679138183594,
      "logps/chosen": -304.85516357421875,
      "logps/rejected": -290.83856201171875,
      "loss": 1648.0504,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07063116133213043,
      "rewards/margins": 0.1489768922328949,
      "rewards/rejected": -0.21960802376270294,
      "step": 12200
    },
    {
      "epoch": 0.8,
      "eval_logits/chosen": -2.353001356124878,
      "eval_logits/rejected": -2.1653261184692383,
      "eval_logps/chosen": -240.8780517578125,
      "eval_logps/rejected": -235.7217254638672,
      "eval_loss": 1765.9278564453125,
      "eval_rewards/accuracies": 0.6600000262260437,
      "eval_rewards/chosen": -0.08873096108436584,
      "eval_rewards/margins": 0.15236733853816986,
      "eval_rewards/rejected": -0.2410982847213745,
      "eval_runtime": 722.6815,
      "eval_samples_per_second": 2.767,
      "eval_steps_per_second": 1.384,
      "step": 12200
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.912656663168717e-07,
      "logits/chosen": -2.436596632003784,
      "logits/rejected": -2.380277633666992,
      "logps/chosen": -232.26504516601562,
      "logps/rejected": -241.3466033935547,
      "loss": 1602.81,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0825573205947876,
      "rewards/margins": 0.15453985333442688,
      "rewards/rejected": -0.23709718883037567,
      "step": 12210
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.875831008943817e-07,
      "logits/chosen": -2.193098545074463,
      "logits/rejected": -2.1777737140655518,
      "logps/chosen": -185.7943878173828,
      "logps/rejected": -174.64498901367188,
      "loss": 2011.7496,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08150316029787064,
      "rewards/margins": 0.1145082339644432,
      "rewards/rejected": -0.19601139426231384,
      "step": 12220
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.839105115752442e-07,
      "logits/chosen": -2.3216700553894043,
      "logits/rejected": -2.134594440460205,
      "logps/chosen": -237.3351593017578,
      "logps/rejected": -210.3347625732422,
      "loss": 2003.9154,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.14357665181159973,
      "rewards/margins": 0.1300736516714096,
      "rewards/rejected": -0.27365028858184814,
      "step": 12230
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.802479175174855e-07,
      "logits/chosen": -2.300666332244873,
      "logits/rejected": -2.1491894721984863,
      "logps/chosen": -175.96163940429688,
      "logps/rejected": -196.45407104492188,
      "loss": 1881.6727,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09967365860939026,
      "rewards/margins": 0.14565333724021912,
      "rewards/rejected": -0.245326966047287,
      "step": 12240
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.765953378269901e-07,
      "logits/chosen": -2.2157773971557617,
      "logits/rejected": -2.150618553161621,
      "logps/chosen": -216.1068572998047,
      "logps/rejected": -263.0898742675781,
      "loss": 1716.0438,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09113435447216034,
      "rewards/margins": 0.15939262509346008,
      "rewards/rejected": -0.2505269944667816,
      "step": 12250
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.729527915574037e-07,
      "logits/chosen": -2.3772082328796387,
      "logits/rejected": -2.235239267349243,
      "logps/chosen": -229.6256561279297,
      "logps/rejected": -246.18142700195312,
      "loss": 1565.1001,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09207095205783844,
      "rewards/margins": 0.16216661036014557,
      "rewards/rejected": -0.254237562417984,
      "step": 12260
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.693202977100304e-07,
      "logits/chosen": -2.3882689476013184,
      "logits/rejected": -2.0996174812316895,
      "logps/chosen": -181.91680908203125,
      "logps/rejected": -189.93724060058594,
      "loss": 2011.5711,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11771687120199203,
      "rewards/margins": 0.11356747150421143,
      "rewards/rejected": -0.23128433525562286,
      "step": 12270
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.656978752337389e-07,
      "logits/chosen": -2.4091591835021973,
      "logits/rejected": -2.2006168365478516,
      "logps/chosen": -211.9903564453125,
      "logps/rejected": -229.5045166015625,
      "loss": 1806.8143,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12335234880447388,
      "rewards/margins": 0.16114404797554016,
      "rewards/rejected": -0.2844964265823364,
      "step": 12280
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.620855430248581e-07,
      "logits/chosen": -2.3077521324157715,
      "logits/rejected": -2.163949489593506,
      "logps/chosen": -165.94668579101562,
      "logps/rejected": -183.89791870117188,
      "loss": 1440.0475,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.039516277611255646,
      "rewards/margins": 0.20129624009132385,
      "rewards/rejected": -0.2408125102519989,
      "step": 12290
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.584833199270837e-07,
      "logits/chosen": -2.365527868270874,
      "logits/rejected": -2.222726345062256,
      "logps/chosen": -239.72311401367188,
      "logps/rejected": -245.8061065673828,
      "loss": 1886.5662,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11090850830078125,
      "rewards/margins": 0.13776203989982605,
      "rewards/rejected": -0.2486705780029297,
      "step": 12300
    },
    {
      "epoch": 0.8,
      "eval_logits/chosen": -2.353067398071289,
      "eval_logits/rejected": -2.165477752685547,
      "eval_logps/chosen": -241.03871154785156,
      "eval_logps/rejected": -235.90528869628906,
      "eval_loss": 1764.2982177734375,
      "eval_rewards/accuracies": 0.6614999771118164,
      "eval_rewards/chosen": -0.0903375893831253,
      "eval_rewards/margins": 0.15259619057178497,
      "eval_rewards/rejected": -0.24293380975723267,
      "eval_runtime": 724.9294,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.379,
      "step": 12300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.548912247313742e-07,
      "logits/chosen": -2.5742082595825195,
      "logits/rejected": -2.145359516143799,
      "logps/chosen": -295.21185302734375,
      "logps/rejected": -260.882568359375,
      "loss": 1808.2764,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1141192689538002,
      "rewards/margins": 0.13246312737464905,
      "rewards/rejected": -0.24658238887786865,
      "step": 12310
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.513092761758596e-07,
      "logits/chosen": -2.3879361152648926,
      "logits/rejected": -2.1963579654693604,
      "logps/chosen": -284.59625244140625,
      "logps/rejected": -227.46878051757812,
      "loss": 2101.8111,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13386356830596924,
      "rewards/margins": 0.09853416681289673,
      "rewards/rejected": -0.23239775002002716,
      "step": 12320
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.477374929457363e-07,
      "logits/chosen": -2.3187971115112305,
      "logits/rejected": -2.330688238143921,
      "logps/chosen": -212.56417846679688,
      "logps/rejected": -205.9453125,
      "loss": 1889.5531,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10503029823303223,
      "rewards/margins": 0.12553009390830994,
      "rewards/rejected": -0.23056037724018097,
      "step": 12330
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.441758936731772e-07,
      "logits/chosen": -2.337404489517212,
      "logits/rejected": -2.228415012359619,
      "logps/chosen": -247.33132934570312,
      "logps/rejected": -244.2796630859375,
      "loss": 1912.2846,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11986031383275986,
      "rewards/margins": 0.1473861038684845,
      "rewards/rejected": -0.26724642515182495,
      "step": 12340
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.406244969372273e-07,
      "logits/chosen": -2.252225399017334,
      "logits/rejected": -2.09260892868042,
      "logps/chosen": -209.585693359375,
      "logps/rejected": -238.00643920898438,
      "loss": 1337.8583,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09862855821847916,
      "rewards/margins": 0.21918770670890808,
      "rewards/rejected": -0.31781625747680664,
      "step": 12350
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.370833212637122e-07,
      "logits/chosen": -2.320801258087158,
      "logits/rejected": -2.0274157524108887,
      "logps/chosen": -230.2029266357422,
      "logps/rejected": -232.7014923095703,
      "loss": 1679.7021,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11691944301128387,
      "rewards/margins": 0.16283735632896423,
      "rewards/rejected": -0.2797567844390869,
      "step": 12360
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.335523851251392e-07,
      "logits/chosen": -2.2678158283233643,
      "logits/rejected": -2.1951239109039307,
      "logps/chosen": -214.69039916992188,
      "logps/rejected": -208.9894561767578,
      "loss": 2124.7422,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08556528389453888,
      "rewards/margins": 0.15097244083881378,
      "rewards/rejected": -0.23653773963451385,
      "step": 12370
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.300317069406003e-07,
      "logits/chosen": -2.244819164276123,
      "logits/rejected": -2.2248756885528564,
      "logps/chosen": -170.40798950195312,
      "logps/rejected": -190.18472290039062,
      "loss": 1655.8439,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.09662125259637833,
      "rewards/margins": 0.1600993573665619,
      "rewards/rejected": -0.2567206025123596,
      "step": 12380
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.265213050756782e-07,
      "logits/chosen": -2.4712936878204346,
      "logits/rejected": -2.2991206645965576,
      "logps/chosen": -242.697021484375,
      "logps/rejected": -253.659423828125,
      "loss": 2287.0518,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.10645761340856552,
      "rewards/margins": 0.13093502819538116,
      "rewards/rejected": -0.2373926341533661,
      "step": 12390
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.230211978423477e-07,
      "logits/chosen": -2.3769500255584717,
      "logits/rejected": -2.260720729827881,
      "logps/chosen": -230.4022979736328,
      "logps/rejected": -230.37966918945312,
      "loss": 1838.3824,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11602053791284561,
      "rewards/margins": 0.15146169066429138,
      "rewards/rejected": -0.2674822211265564,
      "step": 12400
    },
    {
      "epoch": 0.81,
      "eval_logits/chosen": -2.3548293113708496,
      "eval_logits/rejected": -2.167078971862793,
      "eval_logps/chosen": -241.44473266601562,
      "eval_logps/rejected": -235.86338806152344,
      "eval_loss": 1764.401611328125,
      "eval_rewards/accuracies": 0.6610000133514404,
      "eval_rewards/chosen": -0.09439782798290253,
      "eval_rewards/margins": 0.14811697602272034,
      "eval_rewards/rejected": -0.24251480400562286,
      "eval_runtime": 724.5089,
      "eval_samples_per_second": 2.76,
      "eval_steps_per_second": 1.38,
      "step": 12400
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.195314034988835e-07,
      "logits/chosen": -2.501450300216675,
      "logits/rejected": -2.2392566204071045,
      "logps/chosen": -226.6898193359375,
      "logps/rejected": -175.55081176757812,
      "loss": 1677.6783,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07402725517749786,
      "rewards/margins": 0.15900227427482605,
      "rewards/rejected": -0.23302951455116272,
      "step": 12410
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.160519402497616e-07,
      "logits/chosen": -2.389047622680664,
      "logits/rejected": -2.2399632930755615,
      "logps/chosen": -243.2234344482422,
      "logps/rejected": -253.6158905029297,
      "loss": 1699.8174,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13169410824775696,
      "rewards/margins": 0.14452390372753143,
      "rewards/rejected": -0.2762180268764496,
      "step": 12420
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.125828262455679e-07,
      "logits/chosen": -2.281661033630371,
      "logits/rejected": -2.0835673809051514,
      "logps/chosen": -263.7641906738281,
      "logps/rejected": -250.3699188232422,
      "loss": 1599.9329,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08051087707281113,
      "rewards/margins": 0.16945312917232513,
      "rewards/rejected": -0.24996399879455566,
      "step": 12430
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.091240795828992e-07,
      "logits/chosen": -2.0497326850891113,
      "logits/rejected": -2.242506980895996,
      "logps/chosen": -211.437744140625,
      "logps/rejected": -249.03170776367188,
      "loss": 1446.9986,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08947314321994781,
      "rewards/margins": 0.17763306200504303,
      "rewards/rejected": -0.26710623502731323,
      "step": 12440
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.056757183042732e-07,
      "logits/chosen": -2.2784876823425293,
      "logits/rejected": -2.1990206241607666,
      "logps/chosen": -245.272705078125,
      "logps/rejected": -241.74923706054688,
      "loss": 1716.6078,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11716543138027191,
      "rewards/margins": 0.1635403037071228,
      "rewards/rejected": -0.2807057499885559,
      "step": 12450
    },
    {
      "epoch": 0.82,
      "learning_rate": 5.022377603980308e-07,
      "logits/chosen": -2.4438557624816895,
      "logits/rejected": -2.1182327270507812,
      "logps/chosen": -258.93572998046875,
      "logps/rejected": -209.24758911132812,
      "loss": 1536.6931,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.096510149538517,
      "rewards/margins": 0.16783210635185242,
      "rewards/rejected": -0.2643422484397888,
      "step": 12460
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.988102237982454e-07,
      "logits/chosen": -2.390838146209717,
      "logits/rejected": -2.3238790035247803,
      "logps/chosen": -236.59536743164062,
      "logps/rejected": -207.47622680664062,
      "loss": 1904.8795,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1441490352153778,
      "rewards/margins": 0.1145612969994545,
      "rewards/rejected": -0.2587103247642517,
      "step": 12470
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.953931263846251e-07,
      "logits/chosen": -2.3809030055999756,
      "logits/rejected": -2.1044983863830566,
      "logps/chosen": -272.463134765625,
      "logps/rejected": -244.44815063476562,
      "loss": 1726.9566,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10754238069057465,
      "rewards/margins": 0.1594572365283966,
      "rewards/rejected": -0.26699960231781006,
      "step": 12480
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.919864859824266e-07,
      "logits/chosen": -2.3234763145446777,
      "logits/rejected": -2.1786303520202637,
      "logps/chosen": -252.82870483398438,
      "logps/rejected": -220.4989013671875,
      "loss": 1900.4014,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15394526720046997,
      "rewards/margins": 0.12705360352993011,
      "rewards/rejected": -0.2809988856315613,
      "step": 12490
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.885903203623532e-07,
      "logits/chosen": -2.498518466949463,
      "logits/rejected": -2.107308864593506,
      "logps/chosen": -290.13751220703125,
      "logps/rejected": -249.6146697998047,
      "loss": 1238.4372,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.051236413419246674,
      "rewards/margins": 0.23250535130500793,
      "rewards/rejected": -0.2837417721748352,
      "step": 12500
    },
    {
      "epoch": 0.82,
      "eval_logits/chosen": -2.3528501987457275,
      "eval_logits/rejected": -2.1651055812835693,
      "eval_logps/chosen": -241.78085327148438,
      "eval_logps/rejected": -236.79336547851562,
      "eval_loss": 1760.1107177734375,
      "eval_rewards/accuracies": 0.6610000133514404,
      "eval_rewards/chosen": -0.09775892645120621,
      "eval_rewards/margins": 0.15405558049678802,
      "eval_rewards/rejected": -0.2518145442008972,
      "eval_runtime": 725.376,
      "eval_samples_per_second": 2.757,
      "eval_steps_per_second": 1.379,
      "step": 12500
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.852046472404695e-07,
      "logits/chosen": -2.499540328979492,
      "logits/rejected": -1.7627403736114502,
      "logps/chosen": -305.05047607421875,
      "logps/rejected": -195.26817321777344,
      "loss": 1706.6885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07161720097064972,
      "rewards/margins": 0.1589113175868988,
      "rewards/rejected": -0.23052850365638733,
      "step": 12510
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.818294842781035e-07,
      "logits/chosen": -2.41566801071167,
      "logits/rejected": -2.240298271179199,
      "logps/chosen": -234.0768280029297,
      "logps/rejected": -200.38133239746094,
      "loss": 1719.5809,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07868404686450958,
      "rewards/margins": 0.18662366271018982,
      "rewards/rejected": -0.2653077244758606,
      "step": 12520
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.784648490817601e-07,
      "logits/chosen": -2.413585662841797,
      "logits/rejected": -2.120288848876953,
      "logps/chosen": -231.2106170654297,
      "logps/rejected": -201.527587890625,
      "loss": 1995.8398,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11103849112987518,
      "rewards/margins": 0.13575485348701477,
      "rewards/rejected": -0.24679334461688995,
      "step": 12530
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.751107592030235e-07,
      "logits/chosen": -2.427438259124756,
      "logits/rejected": -2.1439523696899414,
      "logps/chosen": -177.9643096923828,
      "logps/rejected": -182.3109588623047,
      "loss": 1420.8544,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0811251550912857,
      "rewards/margins": 0.2144661396741867,
      "rewards/rejected": -0.2955912947654724,
      "step": 12540
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.717672321384703e-07,
      "logits/chosen": -2.3183751106262207,
      "logits/rejected": -2.0838351249694824,
      "logps/chosen": -229.7345428466797,
      "logps/rejected": -203.35838317871094,
      "loss": 1716.082,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06488841772079468,
      "rewards/margins": 0.1581958383321762,
      "rewards/rejected": -0.22308428585529327,
      "step": 12550
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.684342853295748e-07,
      "logits/chosen": -2.2812492847442627,
      "logits/rejected": -2.1551685333251953,
      "logps/chosen": -193.63571166992188,
      "logps/rejected": -208.3316192626953,
      "loss": 1547.6708,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0593893900513649,
      "rewards/margins": 0.1861993670463562,
      "rewards/rejected": -0.2455887496471405,
      "step": 12560
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.651119361626213e-07,
      "logits/chosen": -2.557042360305786,
      "logits/rejected": -2.228407144546509,
      "logps/chosen": -248.6354522705078,
      "logps/rejected": -216.62582397460938,
      "loss": 1680.7773,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07516884058713913,
      "rewards/margins": 0.14219233393669128,
      "rewards/rejected": -0.21736116707324982,
      "step": 12570
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.618002019686091e-07,
      "logits/chosen": -2.3262791633605957,
      "logits/rejected": -2.1434736251831055,
      "logps/chosen": -289.65655517578125,
      "logps/rejected": -247.00265502929688,
      "loss": 2110.5848,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10444200038909912,
      "rewards/margins": 0.13982179760932922,
      "rewards/rejected": -0.24426379799842834,
      "step": 12580
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.5849910002316757e-07,
      "logits/chosen": -2.408716917037964,
      "logits/rejected": -2.06657075881958,
      "logps/chosen": -193.36878967285156,
      "logps/rejected": -177.7703399658203,
      "loss": 1534.195,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12856633961200714,
      "rewards/margins": 0.17665782570838928,
      "rewards/rejected": -0.30522421002388,
      "step": 12590
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.5520864754645984e-07,
      "logits/chosen": -2.434593915939331,
      "logits/rejected": -2.3042731285095215,
      "logps/chosen": -289.6260070800781,
      "logps/rejected": -257.13055419921875,
      "loss": 1967.9301,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0827619805932045,
      "rewards/margins": 0.11329551786184311,
      "rewards/rejected": -0.1960574835538864,
      "step": 12600
    },
    {
      "epoch": 0.82,
      "eval_logits/chosen": -2.3529741764068604,
      "eval_logits/rejected": -2.165254592895508,
      "eval_logps/chosen": -241.44981384277344,
      "eval_logps/rejected": -236.67112731933594,
      "eval_loss": 1757.7244873046875,
      "eval_rewards/accuracies": 0.6614999771118164,
      "eval_rewards/chosen": -0.09444864839315414,
      "eval_rewards/margins": 0.15614351630210876,
      "eval_rewards/rejected": -0.2505921423435211,
      "eval_runtime": 721.4429,
      "eval_samples_per_second": 2.772,
      "eval_steps_per_second": 1.386,
      "step": 12600
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.5192886170309896e-07,
      "logits/chosen": -2.2827320098876953,
      "logits/rejected": -2.179940700531006,
      "logps/chosen": -201.1009521484375,
      "logps/rejected": -213.9563446044922,
      "loss": 2239.9746,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.10332496464252472,
      "rewards/margins": 0.12671324610710144,
      "rewards/rejected": -0.23003819584846497,
      "step": 12610
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.486597596020548e-07,
      "logits/chosen": -2.3789055347442627,
      "logits/rejected": -2.1024844646453857,
      "logps/chosen": -231.01260375976562,
      "logps/rejected": -211.09994506835938,
      "loss": 1544.9164,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.11431293189525604,
      "rewards/margins": 0.18409089744091034,
      "rewards/rejected": -0.29840385913848877,
      "step": 12620
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.454013582965644e-07,
      "logits/chosen": -2.304321527481079,
      "logits/rejected": -1.8886350393295288,
      "logps/chosen": -274.82427978515625,
      "logps/rejected": -236.3057861328125,
      "loss": 2003.2834,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11367567628622055,
      "rewards/margins": 0.11924811452627182,
      "rewards/rejected": -0.23292379081249237,
      "step": 12630
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.4215367478404605e-07,
      "logits/chosen": -2.148632287979126,
      "logits/rejected": -2.125662326812744,
      "logps/chosen": -283.1051940917969,
      "logps/rejected": -322.2588195800781,
      "loss": 1788.9277,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1234527975320816,
      "rewards/margins": 0.14781855046749115,
      "rewards/rejected": -0.27127137780189514,
      "step": 12640
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.389167260060068e-07,
      "logits/chosen": -2.4365358352661133,
      "logits/rejected": -2.1948304176330566,
      "logps/chosen": -213.0283660888672,
      "logps/rejected": -195.78033447265625,
      "loss": 1292.1158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04338112473487854,
      "rewards/margins": 0.20926494896411896,
      "rewards/rejected": -0.2526460886001587,
      "step": 12650
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.356905288479579e-07,
      "logits/chosen": -2.2978675365448,
      "logits/rejected": -2.0709261894226074,
      "logps/chosen": -234.98574829101562,
      "logps/rejected": -227.6422119140625,
      "loss": 1213.8726,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.08464913815259933,
      "rewards/margins": 0.2142043113708496,
      "rewards/rejected": -0.29885345697402954,
      "step": 12660
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.3247510013932377e-07,
      "logits/chosen": -2.248154640197754,
      "logits/rejected": -2.078132152557373,
      "logps/chosen": -261.9805908203125,
      "logps/rejected": -279.871826171875,
      "loss": 1742.7143,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09792111814022064,
      "rewards/margins": 0.18071343004703522,
      "rewards/rejected": -0.27863457798957825,
      "step": 12670
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.2927045665335594e-07,
      "logits/chosen": -1.9597619771957397,
      "logits/rejected": -1.8733116388320923,
      "logps/chosen": -181.56686401367188,
      "logps/rejected": -189.19589233398438,
      "loss": 1980.1352,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1668912172317505,
      "rewards/margins": 0.11184272915124893,
      "rewards/rejected": -0.27873390913009644,
      "step": 12680
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.260766151070439e-07,
      "logits/chosen": -2.1650466918945312,
      "logits/rejected": -2.2159152030944824,
      "logps/chosen": -236.6437530517578,
      "logps/rejected": -241.3082733154297,
      "loss": 1697.1186,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.12210245430469513,
      "rewards/margins": 0.15925151109695435,
      "rewards/rejected": -0.2813539505004883,
      "step": 12690
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.228935921610308e-07,
      "logits/chosen": -2.3920722007751465,
      "logits/rejected": -2.031923532485962,
      "logps/chosen": -273.67059326171875,
      "logps/rejected": -223.01467895507812,
      "loss": 1912.1277,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09951961785554886,
      "rewards/margins": 0.13395579159259796,
      "rewards/rejected": -0.23347540199756622,
      "step": 12700
    },
    {
      "epoch": 0.83,
      "eval_logits/chosen": -2.3546695709228516,
      "eval_logits/rejected": -2.1666719913482666,
      "eval_logps/chosen": -241.46438598632812,
      "eval_logps/rejected": -236.64071655273438,
      "eval_loss": 1757.38720703125,
      "eval_rewards/accuracies": 0.6620000004768372,
      "eval_rewards/chosen": -0.09459412097930908,
      "eval_rewards/margins": 0.15569402277469635,
      "eval_rewards/rejected": -0.25028812885284424,
      "eval_runtime": 723.8008,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.382,
      "step": 12700
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1972140441952246e-07,
      "logits/chosen": -2.2060556411743164,
      "logits/rejected": -2.172612428665161,
      "logps/chosen": -247.1871795654297,
      "logps/rejected": -265.71710205078125,
      "loss": 1742.0557,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07603900879621506,
      "rewards/margins": 0.1652071326971054,
      "rewards/rejected": -0.24124614894390106,
      "step": 12710
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.165600684302046e-07,
      "logits/chosen": -2.3115181922912598,
      "logits/rejected": -2.372889518737793,
      "logps/chosen": -182.8769989013672,
      "logps/rejected": -209.275634765625,
      "loss": 1456.5319,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0709301307797432,
      "rewards/margins": 0.1644066870212555,
      "rewards/rejected": -0.23533682525157928,
      "step": 12720
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.13409600684154e-07,
      "logits/chosen": -2.4179084300994873,
      "logits/rejected": -2.1326534748077393,
      "logps/chosen": -226.34835815429688,
      "logps/rejected": -209.9674835205078,
      "loss": 1859.2658,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.11328748613595963,
      "rewards/margins": 0.12802428007125854,
      "rewards/rejected": -0.24131174385547638,
      "step": 12730
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.102700176157548e-07,
      "logits/chosen": -2.4787747859954834,
      "logits/rejected": -2.1119885444641113,
      "logps/chosen": -335.3193664550781,
      "logps/rejected": -254.7163543701172,
      "loss": 1789.3066,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10564114153385162,
      "rewards/margins": 0.1718355268239975,
      "rewards/rejected": -0.2774766683578491,
      "step": 12740
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.0714133560260884e-07,
      "logits/chosen": -2.366929292678833,
      "logits/rejected": -2.200700283050537,
      "logps/chosen": -272.04559326171875,
      "logps/rejected": -226.88577270507812,
      "loss": 1714.9975,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12512561678886414,
      "rewards/margins": 0.13071008026599884,
      "rewards/rejected": -0.25583571195602417,
      "step": 12750
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.0402357096545527e-07,
      "logits/chosen": -2.249117374420166,
      "logits/rejected": -2.2036967277526855,
      "logps/chosen": -259.86865234375,
      "logps/rejected": -264.2768249511719,
      "loss": 1702.4238,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08107040822505951,
      "rewards/margins": 0.1458026021718979,
      "rewards/rejected": -0.2268730103969574,
      "step": 12760
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.0091673996808025e-07,
      "logits/chosen": -2.456432342529297,
      "logits/rejected": -2.2576537132263184,
      "logps/chosen": -212.8669891357422,
      "logps/rejected": -201.2241973876953,
      "loss": 1807.799,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15806898474693298,
      "rewards/margins": 0.12984392046928406,
      "rewards/rejected": -0.28791293501853943,
      "step": 12770
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9782085881723776e-07,
      "logits/chosen": -2.279867649078369,
      "logits/rejected": -2.130587339401245,
      "logps/chosen": -177.62420654296875,
      "logps/rejected": -204.5876007080078,
      "loss": 1886.7762,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12026622146368027,
      "rewards/margins": 0.14996710419654846,
      "rewards/rejected": -0.2702333331108093,
      "step": 12780
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.947359436625592e-07,
      "logits/chosen": -2.2889564037323,
      "logits/rejected": -2.1877830028533936,
      "logps/chosen": -246.07382202148438,
      "logps/rejected": -225.522705078125,
      "loss": 1585.2244,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11410894244909286,
      "rewards/margins": 0.17013028264045715,
      "rewards/rejected": -0.284239262342453,
      "step": 12790
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9166201059647386e-07,
      "logits/chosen": -2.4249634742736816,
      "logits/rejected": -2.2888290882110596,
      "logps/chosen": -273.11492919921875,
      "logps/rejected": -243.9048614501953,
      "loss": 2416.5143,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11264701187610626,
      "rewards/margins": 0.07720918953418732,
      "rewards/rejected": -0.18985618650913239,
      "step": 12800
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": -2.351189136505127,
      "eval_logits/rejected": -2.163454055786133,
      "eval_logps/chosen": -242.81185913085938,
      "eval_logps/rejected": -237.82569885253906,
      "eval_loss": 1756.477294921875,
      "eval_rewards/accuracies": 0.6610000133514404,
      "eval_rewards/chosen": -0.10806912183761597,
      "eval_rewards/margins": 0.15406881272792816,
      "eval_rewards/rejected": -0.2621379494667053,
      "eval_runtime": 724.1571,
      "eval_samples_per_second": 2.762,
      "eval_steps_per_second": 1.381,
      "step": 12800
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.8859907565412194e-07,
      "logits/chosen": -2.2357773780822754,
      "logits/rejected": -2.3141744136810303,
      "logps/chosen": -193.81643676757812,
      "logps/rejected": -209.11569213867188,
      "loss": 1652.4705,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.14167191088199615,
      "rewards/margins": 0.1587766706943512,
      "rewards/rejected": -0.30044859647750854,
      "step": 12810
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.8554715481327303e-07,
      "logits/chosen": -2.3908543586730957,
      "logits/rejected": -1.998772382736206,
      "logps/chosen": -247.39334106445312,
      "logps/rejected": -225.96493530273438,
      "loss": 2023.8736,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.14134298264980316,
      "rewards/margins": 0.1396934986114502,
      "rewards/rejected": -0.28103652596473694,
      "step": 12820
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.8250626399424007e-07,
      "logits/chosen": -2.4040729999542236,
      "logits/rejected": -2.1485342979431152,
      "logps/chosen": -264.1070251464844,
      "logps/rejected": -259.52252197265625,
      "loss": 1815.5469,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12628862261772156,
      "rewards/margins": 0.14265701174736023,
      "rewards/rejected": -0.2689456343650818,
      "step": 12830
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.7947641905980104e-07,
      "logits/chosen": -2.229609251022339,
      "logits/rejected": -2.2407336235046387,
      "logps/chosen": -215.80990600585938,
      "logps/rejected": -196.70230102539062,
      "loss": 1642.165,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08491475880146027,
      "rewards/margins": 0.15573066473007202,
      "rewards/rejected": -0.24064543843269348,
      "step": 12840
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.764576358151098e-07,
      "logits/chosen": -2.263810634613037,
      "logits/rejected": -2.2646942138671875,
      "logps/chosen": -186.5194091796875,
      "logps/rejected": -185.54440307617188,
      "loss": 1978.7371,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11702384799718857,
      "rewards/margins": 0.10869280993938446,
      "rewards/rejected": -0.22571666538715363,
      "step": 12850
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.7344993000761944e-07,
      "logits/chosen": -2.376746654510498,
      "logits/rejected": -2.2694809436798096,
      "logps/chosen": -191.15414428710938,
      "logps/rejected": -249.1569061279297,
      "loss": 1602.7209,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13839024305343628,
      "rewards/margins": 0.15030980110168457,
      "rewards/rejected": -0.28870004415512085,
      "step": 12860
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.7045331732699585e-07,
      "logits/chosen": -2.380181312561035,
      "logits/rejected": -2.1943130493164062,
      "logps/chosen": -216.8921356201172,
      "logps/rejected": -199.8214874267578,
      "loss": 1621.9516,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11694061756134033,
      "rewards/margins": 0.18628790974617004,
      "rewards/rejected": -0.30322855710983276,
      "step": 12870
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.6746781340503993e-07,
      "logits/chosen": -2.2263026237487793,
      "logits/rejected": -2.1297569274902344,
      "logps/chosen": -243.273681640625,
      "logps/rejected": -248.5195770263672,
      "loss": 1795.5994,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08857928216457367,
      "rewards/margins": 0.13272887468338013,
      "rewards/rejected": -0.2213081419467926,
      "step": 12880
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.6449343381560116e-07,
      "logits/chosen": -2.3246045112609863,
      "logits/rejected": -2.105888843536377,
      "logps/chosen": -247.8300323486328,
      "logps/rejected": -250.9560089111328,
      "loss": 1695.3215,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16404370963573456,
      "rewards/margins": 0.1651550531387329,
      "rewards/rejected": -0.32919877767562866,
      "step": 12890
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.615301940745017e-07,
      "logits/chosen": -2.5765836238861084,
      "logits/rejected": -1.9856640100479126,
      "logps/chosen": -325.85577392578125,
      "logps/rejected": -236.7607421875,
      "loss": 1621.5686,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08660654723644257,
      "rewards/margins": 0.16081905364990234,
      "rewards/rejected": -0.2474256008863449,
      "step": 12900
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": -2.3496968746185303,
      "eval_logits/rejected": -2.1621196269989014,
      "eval_logps/chosen": -243.3934783935547,
      "eval_logps/rejected": -238.40719604492188,
      "eval_loss": 1755.8291015625,
      "eval_rewards/accuracies": 0.6650000214576721,
      "eval_rewards/chosen": -0.1138853132724762,
      "eval_rewards/margins": 0.1540677845478058,
      "eval_rewards/rejected": -0.267953097820282,
      "eval_runtime": 723.5217,
      "eval_samples_per_second": 2.764,
      "eval_steps_per_second": 1.382,
      "step": 12900
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.5857810963945084e-07,
      "logits/chosen": -2.218391180038452,
      "logits/rejected": -1.9942808151245117,
      "logps/chosen": -227.6157989501953,
      "logps/rejected": -229.28500366210938,
      "loss": 1633.0026,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1310669481754303,
      "rewards/margins": 0.16492125391960144,
      "rewards/rejected": -0.29598820209503174,
      "step": 12910
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.556371959099678e-07,
      "logits/chosen": -2.4070942401885986,
      "logits/rejected": -2.1696436405181885,
      "logps/chosen": -308.32635498046875,
      "logps/rejected": -283.7373962402344,
      "loss": 1785.7984,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11152037233114243,
      "rewards/margins": 0.15381523966789246,
      "rewards/rejected": -0.2653356194496155,
      "step": 12920
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.5270746822729797e-07,
      "logits/chosen": -2.294844388961792,
      "logits/rejected": -2.2179951667785645,
      "logps/chosen": -263.7327880859375,
      "logps/rejected": -288.92022705078125,
      "loss": 2077.7789,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1625073105096817,
      "rewards/margins": 0.12612196803092957,
      "rewards/rejected": -0.2886292636394501,
      "step": 12930
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4978894187433746e-07,
      "logits/chosen": -2.4065170288085938,
      "logits/rejected": -2.266335964202881,
      "logps/chosen": -165.07269287109375,
      "logps/rejected": -163.32843017578125,
      "loss": 2126.6654,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13997486233711243,
      "rewards/margins": 0.09164164960384369,
      "rewards/rejected": -0.2316165268421173,
      "step": 12940
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.468816320755486e-07,
      "logits/chosen": -2.1857643127441406,
      "logits/rejected": -2.0050368309020996,
      "logps/chosen": -233.8216552734375,
      "logps/rejected": -206.8753662109375,
      "loss": 1907.748,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09893886744976044,
      "rewards/margins": 0.12663452327251434,
      "rewards/rejected": -0.2255733758211136,
      "step": 12950
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4398555399688336e-07,
      "logits/chosen": -2.4522571563720703,
      "logits/rejected": -2.0734975337982178,
      "logps/chosen": -228.2733612060547,
      "logps/rejected": -213.73123168945312,
      "loss": 2365.5295,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1692286729812622,
      "rewards/margins": 0.06122065708041191,
      "rewards/rejected": -0.2304493486881256,
      "step": 12960
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.411007227457047e-07,
      "logits/chosen": -2.3396944999694824,
      "logits/rejected": -2.292318820953369,
      "logps/chosen": -257.23907470703125,
      "logps/rejected": -240.35848999023438,
      "loss": 1584.4968,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09477768838405609,
      "rewards/margins": 0.1539420336484909,
      "rewards/rejected": -0.248719722032547,
      "step": 12970
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.382271533707043e-07,
      "logits/chosen": -2.2677955627441406,
      "logits/rejected": -2.254593849182129,
      "logps/chosen": -201.43359375,
      "logps/rejected": -191.1988525390625,
      "loss": 1955.8822,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0982837826013565,
      "rewards/margins": 0.12648960947990417,
      "rewards/rejected": -0.2247733771800995,
      "step": 12980
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.353648608618287e-07,
      "logits/chosen": -2.341491937637329,
      "logits/rejected": -2.0881080627441406,
      "logps/chosen": -180.15113830566406,
      "logps/rejected": -183.06163024902344,
      "loss": 1867.4459,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12279865890741348,
      "rewards/margins": 0.12881214916706085,
      "rewards/rejected": -0.25161081552505493,
      "step": 12990
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.3251386015019676e-07,
      "logits/chosen": -2.368988513946533,
      "logits/rejected": -2.157670736312866,
      "logps/chosen": -214.96731567382812,
      "logps/rejected": -194.64791870117188,
      "loss": 1645.2689,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1220787763595581,
      "rewards/margins": 0.13425137102603912,
      "rewards/rejected": -0.25633013248443604,
      "step": 13000
    },
    {
      "epoch": 0.85,
      "eval_logits/chosen": -2.3476548194885254,
      "eval_logits/rejected": -2.1602706909179688,
      "eval_logps/chosen": -243.2641143798828,
      "eval_logps/rejected": -238.19064331054688,
      "eval_loss": 1755.0059814453125,
      "eval_rewards/accuracies": 0.6639999747276306,
      "eval_rewards/chosen": -0.11259139329195023,
      "eval_rewards/margins": 0.15319600701332092,
      "eval_rewards/rejected": -0.26578739285469055,
      "eval_runtime": 724.8273,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.38,
      "step": 13000
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.296741661080255e-07,
      "logits/chosen": -2.2990310192108154,
      "logits/rejected": -2.212502956390381,
      "logps/chosen": -249.18881225585938,
      "logps/rejected": -255.80764770507812,
      "loss": 2351.9094,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1605045646429062,
      "rewards/margins": 0.13160006701946259,
      "rewards/rejected": -0.2921046316623688,
      "step": 13010
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.2684579354854974e-07,
      "logits/chosen": -2.391723394393921,
      "logits/rejected": -2.286519765853882,
      "logps/chosen": -306.3222351074219,
      "logps/rejected": -334.36419677734375,
      "loss": 1797.0959,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13863664865493774,
      "rewards/margins": 0.17783468961715698,
      "rewards/rejected": -0.3164713382720947,
      "step": 13020
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.2402875722594653e-07,
      "logits/chosen": -2.4300198554992676,
      "logits/rejected": -2.1459197998046875,
      "logps/chosen": -179.18894958496094,
      "logps/rejected": -200.12417602539062,
      "loss": 1694.7115,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10143520683050156,
      "rewards/margins": 0.14783641695976257,
      "rewards/rejected": -0.24927166104316711,
      "step": 13030
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.212230718352566e-07,
      "logits/chosen": -2.2786428928375244,
      "logits/rejected": -2.298488140106201,
      "logps/chosen": -238.57382202148438,
      "logps/rejected": -179.4725341796875,
      "loss": 2324.5529,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13523949682712555,
      "rewards/margins": 0.04595714807510376,
      "rewards/rejected": -0.1811966449022293,
      "step": 13040
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.1842875201231025e-07,
      "logits/chosen": -2.3447372913360596,
      "logits/rejected": -2.0560357570648193,
      "logps/chosen": -234.4106903076172,
      "logps/rejected": -222.1627655029297,
      "loss": 1798.5451,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12670625746250153,
      "rewards/margins": 0.1352727711200714,
      "rewards/rejected": -0.26197901368141174,
      "step": 13050
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.156458123336478e-07,
      "logits/chosen": -2.189135789871216,
      "logits/rejected": -2.0165061950683594,
      "logps/chosen": -169.20169067382812,
      "logps/rejected": -177.94007873535156,
      "loss": 1615.8548,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.10689838975667953,
      "rewards/margins": 0.18887147307395935,
      "rewards/rejected": -0.2957698702812195,
      "step": 13060
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.128742673164459e-07,
      "logits/chosen": -2.415038585662842,
      "logits/rejected": -2.054231643676758,
      "logps/chosen": -292.4279479980469,
      "logps/rejected": -268.6998596191406,
      "loss": 1283.446,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.09500737488269806,
      "rewards/margins": 0.20787818729877472,
      "rewards/rejected": -0.3028855621814728,
      "step": 13070
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.101141314184414e-07,
      "logits/chosen": -2.525285005569458,
      "logits/rejected": -2.282944679260254,
      "logps/chosen": -213.61471557617188,
      "logps/rejected": -220.43521118164062,
      "loss": 1461.5715,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0823017954826355,
      "rewards/margins": 0.1603173315525055,
      "rewards/rejected": -0.242619127035141,
      "step": 13080
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.0736541903785526e-07,
      "logits/chosen": -2.1892294883728027,
      "logits/rejected": -2.151439666748047,
      "logps/chosen": -223.6445770263672,
      "logps/rejected": -286.0027160644531,
      "loss": 1937.7354,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1462966501712799,
      "rewards/margins": 0.1289856731891632,
      "rewards/rejected": -0.2752823233604431,
      "step": 13090
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.0462814451331704e-07,
      "logits/chosen": -2.23403000831604,
      "logits/rejected": -2.103334903717041,
      "logps/chosen": -246.27597045898438,
      "logps/rejected": -254.24423217773438,
      "loss": 2114.2795,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15599580109119415,
      "rewards/margins": 0.08063185960054398,
      "rewards/rejected": -0.23662766814231873,
      "step": 13100
    },
    {
      "epoch": 0.86,
      "eval_logits/chosen": -2.345999240875244,
      "eval_logits/rejected": -2.158658266067505,
      "eval_logps/chosen": -243.78396606445312,
      "eval_logps/rejected": -238.8558807373047,
      "eval_loss": 1753.9947509765625,
      "eval_rewards/accuracies": 0.6650000214576721,
      "eval_rewards/chosen": -0.11779005825519562,
      "eval_rewards/margins": 0.1546495258808136,
      "eval_rewards/rejected": -0.272439569234848,
      "eval_runtime": 724.8177,
      "eval_samples_per_second": 2.759,
      "eval_steps_per_second": 1.38,
      "step": 13100
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.019023221237927e-07,
      "logits/chosen": -2.2977519035339355,
      "logits/rejected": -2.1277058124542236,
      "logps/chosen": -248.1866912841797,
      "logps/rejected": -207.8753204345703,
      "loss": 1654.5158,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11454341560602188,
      "rewards/margins": 0.17256048321723938,
      "rewards/rejected": -0.28710389137268066,
      "step": 13110
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.991879660885058e-07,
      "logits/chosen": -2.46199893951416,
      "logits/rejected": -2.2021241188049316,
      "logps/chosen": -276.08203125,
      "logps/rejected": -275.280029296875,
      "loss": 1771.293,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1253511607646942,
      "rewards/margins": 0.15558239817619324,
      "rewards/rejected": -0.28093355894088745,
      "step": 13120
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.9648509056686786e-07,
      "logits/chosen": -2.3713479042053223,
      "logits/rejected": -2.2038354873657227,
      "logps/chosen": -187.05670166015625,
      "logps/rejected": -175.7555389404297,
      "loss": 1879.9361,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10114918649196625,
      "rewards/margins": 0.14274568855762482,
      "rewards/rejected": -0.24389486014842987,
      "step": 13130
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.937937096584012e-07,
      "logits/chosen": -2.3491787910461426,
      "logits/rejected": -2.131972551345825,
      "logps/chosen": -302.1286926269531,
      "logps/rejected": -256.2218017578125,
      "loss": 1869.2926,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11727768182754517,
      "rewards/margins": 0.1457705795764923,
      "rewards/rejected": -0.2630482614040375,
      "step": 13140
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.9111383740266756e-07,
      "logits/chosen": -2.1542248725891113,
      "logits/rejected": -2.0185928344726562,
      "logps/chosen": -249.68789672851562,
      "logps/rejected": -256.65728759765625,
      "loss": 1765.4994,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13903729617595673,
      "rewards/margins": 0.12757453322410583,
      "rewards/rejected": -0.26661184430122375,
      "step": 13150
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.8844548777919255e-07,
      "logits/chosen": -2.395974636077881,
      "logits/rejected": -2.113468647003174,
      "logps/chosen": -213.87539672851562,
      "logps/rejected": -203.4140167236328,
      "loss": 2126.2518,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09991560131311417,
      "rewards/margins": 0.11818840354681015,
      "rewards/rejected": -0.2181040346622467,
      "step": 13160
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.8578867470739594e-07,
      "logits/chosen": -2.2456743717193604,
      "logits/rejected": -2.1050240993499756,
      "logps/chosen": -196.46435546875,
      "logps/rejected": -183.75253295898438,
      "loss": 1607.4187,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11976015567779541,
      "rewards/margins": 0.18710866570472717,
      "rewards/rejected": -0.3068688213825226,
      "step": 13170
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.8314341204651484e-07,
      "logits/chosen": -2.450092315673828,
      "logits/rejected": -2.181081771850586,
      "logps/chosen": -281.95928955078125,
      "logps/rejected": -228.1092529296875,
      "loss": 1398.7136,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05920163542032242,
      "rewards/margins": 0.18932537734508514,
      "rewards/rejected": -0.24852702021598816,
      "step": 13180
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.805097135955362e-07,
      "logits/chosen": -2.3811404705047607,
      "logits/rejected": -2.1690518856048584,
      "logps/chosen": -223.0667724609375,
      "logps/rejected": -202.48733520507812,
      "loss": 1702.7156,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10542796552181244,
      "rewards/margins": 0.1434640884399414,
      "rewards/rejected": -0.24889206886291504,
      "step": 13190
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.778875930931213e-07,
      "logits/chosen": -2.3781535625457764,
      "logits/rejected": -2.0523698329925537,
      "logps/chosen": -246.26998901367188,
      "logps/rejected": -243.38485717773438,
      "loss": 1719.5012,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11151113361120224,
      "rewards/margins": 0.14477068185806274,
      "rewards/rejected": -0.2562818229198456,
      "step": 13200
    },
    {
      "epoch": 0.86,
      "eval_logits/chosen": -2.348130226135254,
      "eval_logits/rejected": -2.160688638687134,
      "eval_logps/chosen": -242.72756958007812,
      "eval_logps/rejected": -238.03858947753906,
      "eval_loss": 1755.7774658203125,
      "eval_rewards/accuracies": 0.6629999876022339,
      "eval_rewards/chosen": -0.10722615569829941,
      "eval_rewards/margins": 0.15704070031642914,
      "eval_rewards/rejected": -0.26426684856414795,
      "eval_runtime": 724.6812,
      "eval_samples_per_second": 2.76,
      "eval_steps_per_second": 1.38,
      "step": 13200
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.7527706421753426e-07,
      "logits/chosen": -2.3434062004089355,
      "logits/rejected": -2.2829527854919434,
      "logps/chosen": -209.6172637939453,
      "logps/rejected": -225.86276245117188,
      "loss": 1966.9658,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11216064542531967,
      "rewards/margins": 0.11756797134876251,
      "rewards/rejected": -0.2297286093235016,
      "step": 13210
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.726781405865736e-07,
      "logits/chosen": -2.432314395904541,
      "logits/rejected": -1.8859539031982422,
      "logps/chosen": -313.43511962890625,
      "logps/rejected": -212.71896362304688,
      "loss": 1765.0092,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10769345611333847,
      "rewards/margins": 0.150523841381073,
      "rewards/rejected": -0.2582172751426697,
      "step": 13220
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.7009083575749687e-07,
      "logits/chosen": -2.328192710876465,
      "logits/rejected": -2.2318806648254395,
      "logps/chosen": -256.3060607910156,
      "logps/rejected": -267.0457458496094,
      "loss": 1947.6164,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12445763498544693,
      "rewards/margins": 0.12375207990407944,
      "rewards/rejected": -0.24820971488952637,
      "step": 13230
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.6751516322695457e-07,
      "logits/chosen": -2.405379056930542,
      "logits/rejected": -2.33868408203125,
      "logps/chosen": -202.52243041992188,
      "logps/rejected": -205.9892578125,
      "loss": 1987.4588,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12278878688812256,
      "rewards/margins": 0.09506727755069733,
      "rewards/rejected": -0.2178560495376587,
      "step": 13240
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.649511364309154e-07,
      "logits/chosen": -2.3319852352142334,
      "logits/rejected": -2.311432361602783,
      "logps/chosen": -212.4677734375,
      "logps/rejected": -207.6529083251953,
      "loss": 1547.3972,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1017812117934227,
      "rewards/margins": 0.16066250205039978,
      "rewards/rejected": -0.2624437212944031,
      "step": 13250
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.6239876874460003e-07,
      "logits/chosen": -2.439059019088745,
      "logits/rejected": -2.315342426300049,
      "logps/chosen": -294.7291564941406,
      "logps/rejected": -285.35638427734375,
      "loss": 1560.8948,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10313956439495087,
      "rewards/margins": 0.2118341475725174,
      "rewards/rejected": -0.31497371196746826,
      "step": 13260
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5985807348240744e-07,
      "logits/chosen": -2.495514154434204,
      "logits/rejected": -2.031292200088501,
      "logps/chosen": -238.3489227294922,
      "logps/rejected": -214.0382080078125,
      "loss": 1343.8792,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06005353853106499,
      "rewards/margins": 0.23636405169963837,
      "rewards/rejected": -0.29641759395599365,
      "step": 13270
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5732906389785014e-07,
      "logits/chosen": -2.366839647293091,
      "logits/rejected": -2.2016913890838623,
      "logps/chosen": -284.45574951171875,
      "logps/rejected": -264.6962890625,
      "loss": 1612.3196,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06749661266803741,
      "rewards/margins": 0.18386101722717285,
      "rewards/rejected": -0.2513576149940491,
      "step": 13280
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5481175318347956e-07,
      "logits/chosen": -2.239269495010376,
      "logits/rejected": -2.2813994884490967,
      "logps/chosen": -235.7333221435547,
      "logps/rejected": -270.91497802734375,
      "loss": 1568.9496,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08235698938369751,
      "rewards/margins": 0.1728564202785492,
      "rewards/rejected": -0.2552134096622467,
      "step": 13290
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5230615447082246e-07,
      "logits/chosen": -2.3373067378997803,
      "logits/rejected": -1.9933191537857056,
      "logps/chosen": -263.7798767089844,
      "logps/rejected": -264.7540283203125,
      "loss": 2001.4379,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12463216483592987,
      "rewards/margins": 0.15759319067001343,
      "rewards/rejected": -0.2822253704071045,
      "step": 13300
    },
    {
      "epoch": 0.87,
      "eval_logits/chosen": -2.3474156856536865,
      "eval_logits/rejected": -2.1600124835968018,
      "eval_logps/chosen": -242.93479919433594,
      "eval_logps/rejected": -238.28860473632812,
      "eval_loss": 1754.6165771484375,
      "eval_rewards/accuracies": 0.6635000109672546,
      "eval_rewards/chosen": -0.10929837822914124,
      "eval_rewards/margins": 0.15746866166591644,
      "eval_rewards/rejected": -0.26676705479621887,
      "eval_runtime": 724.4901,
      "eval_samples_per_second": 2.761,
      "eval_steps_per_second": 1.38,
      "step": 13300
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.49812280830308e-07,
      "logits/chosen": -2.377274513244629,
      "logits/rejected": -1.9086488485336304,
      "logps/chosen": -236.3880157470703,
      "logps/rejected": -226.5410614013672,
      "loss": 1348.1293,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.11350829899311066,
      "rewards/margins": 0.25009047985076904,
      "rewards/rejected": -0.3635987639427185,
      "step": 13310
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.4733014527120457e-07,
      "logits/chosen": -2.2500624656677246,
      "logits/rejected": -2.0748238563537598,
      "logps/chosen": -222.2921905517578,
      "logps/rejected": -216.4639434814453,
      "loss": 1680.1018,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19877322018146515,
      "rewards/margins": 0.17064984142780304,
      "rewards/rejected": -0.3694230318069458,
      "step": 13320
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.4485976074154565e-07,
      "logits/chosen": -2.304439067840576,
      "logits/rejected": -2.361459970474243,
      "logps/chosen": -223.77297973632812,
      "logps/rejected": -257.6127014160156,
      "loss": 2405.968,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.1477496325969696,
      "rewards/margins": 0.03796560689806938,
      "rewards/rejected": -0.1857152134180069,
      "step": 13330
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.4240114012806763e-07,
      "logits/chosen": -2.3313541412353516,
      "logits/rejected": -2.3096864223480225,
      "logps/chosen": -217.3317413330078,
      "logps/rejected": -213.3277587890625,
      "loss": 1960.4521,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09533555805683136,
      "rewards/margins": 0.13065870106220245,
      "rewards/rejected": -0.2259942591190338,
      "step": 13340
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.399542962561399e-07,
      "logits/chosen": -2.2424967288970947,
      "logits/rejected": -2.080265522003174,
      "logps/chosen": -234.34274291992188,
      "logps/rejected": -202.61734008789062,
      "loss": 1645.757,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08886785805225372,
      "rewards/margins": 0.1568392813205719,
      "rewards/rejected": -0.24570715427398682,
      "step": 13350
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3751924188969876e-07,
      "logits/chosen": -2.2778706550598145,
      "logits/rejected": -2.150668144226074,
      "logps/chosen": -260.43463134765625,
      "logps/rejected": -258.49066162109375,
      "loss": 1610.7408,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0869608074426651,
      "rewards/margins": 0.1789068877696991,
      "rewards/rejected": -0.265867680311203,
      "step": 13360
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3509598973118024e-07,
      "logits/chosen": -2.466491222381592,
      "logits/rejected": -2.2724196910858154,
      "logps/chosen": -233.4585418701172,
      "logps/rejected": -181.59771728515625,
      "loss": 2002.6088,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12453192472457886,
      "rewards/margins": 0.10926959663629532,
      "rewards/rejected": -0.23380151391029358,
      "step": 13370
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.326845524214555e-07,
      "logits/chosen": -2.1327478885650635,
      "logits/rejected": -2.2172887325286865,
      "logps/chosen": -249.34310913085938,
      "logps/rejected": -225.1226043701172,
      "loss": 2794.0449,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1428876519203186,
      "rewards/margins": 0.019042372703552246,
      "rewards/rejected": -0.16193000972270966,
      "step": 13380
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.3028494253976158e-07,
      "logits/chosen": -2.40440034866333,
      "logits/rejected": -2.198272705078125,
      "logps/chosen": -355.88104248046875,
      "logps/rejected": -301.50421142578125,
      "loss": 2115.4121,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.14615550637245178,
      "rewards/margins": 0.10204567015171051,
      "rewards/rejected": -0.2482011765241623,
      "step": 13390
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2789717260364026e-07,
      "logits/chosen": -2.3983378410339355,
      "logits/rejected": -2.2215027809143066,
      "logps/chosen": -175.76011657714844,
      "logps/rejected": -167.30177307128906,
      "loss": 1796.4686,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.10654046386480331,
      "rewards/margins": 0.13269367814064026,
      "rewards/rejected": -0.23923416435718536,
      "step": 13400
    },
    {
      "epoch": 0.88,
      "eval_logits/chosen": -2.3467912673950195,
      "eval_logits/rejected": -2.1594457626342773,
      "eval_logps/chosen": -243.21743774414062,
      "eval_logps/rejected": -238.4784393310547,
      "eval_loss": 1754.6705322265625,
      "eval_rewards/accuracies": 0.6660000085830688,
      "eval_rewards/chosen": -0.11212486773729324,
      "eval_rewards/margins": 0.15654033422470093,
      "eval_rewards/rejected": -0.2686651647090912,
      "eval_runtime": 726.9658,
      "eval_samples_per_second": 2.751,
      "eval_steps_per_second": 1.376,
      "step": 13400
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.255212550688682e-07,
      "logits/chosen": -2.296682357788086,
      "logits/rejected": -2.3946003913879395,
      "logps/chosen": -231.0108184814453,
      "logps/rejected": -314.93560791015625,
      "loss": 1660.6289,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.13055798411369324,
      "rewards/margins": 0.19738134741783142,
      "rewards/rejected": -0.32793933153152466,
      "step": 13410
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2315720232939598e-07,
      "logits/chosen": -2.6522374153137207,
      "logits/rejected": -2.199693441390991,
      "logps/chosen": -269.6587829589844,
      "logps/rejected": -201.1451873779297,
      "loss": 1347.6682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07545066624879837,
      "rewards/margins": 0.21088755130767822,
      "rewards/rejected": -0.286338210105896,
      "step": 13420
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2080502671727956e-07,
      "logits/chosen": -2.4427454471588135,
      "logits/rejected": -2.0991287231445312,
      "logps/chosen": -229.20632934570312,
      "logps/rejected": -221.53076171875,
      "loss": 1802.2229,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09124470502138138,
      "rewards/margins": 0.14152109622955322,
      "rewards/rejected": -0.2327658236026764,
      "step": 13430
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1846474050262078e-07,
      "logits/chosen": -2.417556047439575,
      "logits/rejected": -2.2897114753723145,
      "logps/chosen": -256.66497802734375,
      "logps/rejected": -205.99191284179688,
      "loss": 1681.0145,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10104218870401382,
      "rewards/margins": 0.15124091506004333,
      "rewards/rejected": -0.25228309631347656,
      "step": 13440
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1613635589349756e-07,
      "logits/chosen": -1.9890124797821045,
      "logits/rejected": -2.082338809967041,
      "logps/chosen": -205.3116455078125,
      "logps/rejected": -252.1767120361328,
      "loss": 1561.8028,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08148379623889923,
      "rewards/margins": 0.19344353675842285,
      "rewards/rejected": -0.2749273180961609,
      "step": 13450
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1381988503590578e-07,
      "logits/chosen": -2.0807101726531982,
      "logits/rejected": -2.1482956409454346,
      "logps/chosen": -226.9981689453125,
      "logps/rejected": -239.90011596679688,
      "loss": 1403.4047,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.11126313358545303,
      "rewards/margins": 0.19051769375801086,
      "rewards/rejected": -0.3017807900905609,
      "step": 13460
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.11515340013691e-07,
      "logits/chosen": -2.4283270835876465,
      "logits/rejected": -2.40568208694458,
      "logps/chosen": -241.0494384765625,
      "logps/rejected": -240.2838134765625,
      "loss": 1386.8065,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.09876888245344162,
      "rewards/margins": 0.2081459015607834,
      "rewards/rejected": -0.3069148063659668,
      "step": 13470
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.092227328484897e-07,
      "logits/chosen": -2.212730646133423,
      "logits/rejected": -2.1704113483428955,
      "logps/chosen": -215.0039520263672,
      "logps/rejected": -262.45782470703125,
      "loss": 1706.5404,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10995247215032578,
      "rewards/margins": 0.1679636687040329,
      "rewards/rejected": -0.2779161334037781,
      "step": 13480
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0694207549966345e-07,
      "logits/chosen": -2.2417845726013184,
      "logits/rejected": -2.1046087741851807,
      "logps/chosen": -223.4938507080078,
      "logps/rejected": -215.11630249023438,
      "loss": 2201.9396,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14585386216640472,
      "rewards/margins": 0.08491906523704529,
      "rewards/rejected": -0.23077292740345,
      "step": 13490
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0467337986423864e-07,
      "logits/chosen": -2.474929094314575,
      "logits/rejected": -2.1780519485473633,
      "logps/chosen": -313.8457946777344,
      "logps/rejected": -297.32183837890625,
      "loss": 1621.7527,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08122257888317108,
      "rewards/margins": 0.166888028383255,
      "rewards/rejected": -0.2481106072664261,
      "step": 13500
    },
    {
      "epoch": 0.88,
      "eval_logits/chosen": -2.3466689586639404,
      "eval_logits/rejected": -2.159330368041992,
      "eval_logps/chosen": -243.5742950439453,
      "eval_logps/rejected": -238.83111572265625,
      "eval_loss": 1753.5302734375,
      "eval_rewards/accuracies": 0.6639999747276306,
      "eval_rewards/chosen": -0.11569356173276901,
      "eval_rewards/margins": 0.15649853646755219,
      "eval_rewards/rejected": -0.2721921503543854,
      "eval_runtime": 724.1227,
      "eval_samples_per_second": 2.762,
      "eval_steps_per_second": 1.381,
      "step": 13500
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0241665777684272e-07,
      "logits/chosen": -2.389523983001709,
      "logits/rejected": -2.3051490783691406,
      "logps/chosen": -275.768798828125,
      "logps/rejected": -255.1609649658203,
      "loss": 1251.2314,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07475773990154266,
      "rewards/margins": 0.23007413744926453,
      "rewards/rejected": -0.3048318922519684,
      "step": 13510
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0017192100964366e-07,
      "logits/chosen": -2.056959629058838,
      "logits/rejected": -2.1443307399749756,
      "logps/chosen": -214.5357208251953,
      "logps/rejected": -231.7271270751953,
      "loss": 1680.6148,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.14579695463180542,
      "rewards/margins": 0.15074291825294495,
      "rewards/rejected": -0.296539843082428,
      "step": 13520
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.9793918127228777e-07,
      "logits/chosen": -2.4231154918670654,
      "logits/rejected": -2.0626559257507324,
      "logps/chosen": -327.86822509765625,
      "logps/rejected": -292.716552734375,
      "loss": 1599.2062,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11568768322467804,
      "rewards/margins": 0.18930375576019287,
      "rewards/rejected": -0.3049914240837097,
      "step": 13530
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.9571845021184005e-07,
      "logits/chosen": -2.2009639739990234,
      "logits/rejected": -2.106663703918457,
      "logps/chosen": -245.4452362060547,
      "logps/rejected": -262.6272888183594,
      "loss": 1927.0613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14459358155727386,
      "rewards/margins": 0.13064053654670715,
      "rewards/rejected": -0.2752341032028198,
      "step": 13540
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.9350973941272027e-07,
      "logits/chosen": -2.3240561485290527,
      "logits/rejected": -2.3074772357940674,
      "logps/chosen": -221.6490478515625,
      "logps/rejected": -213.5911407470703,
      "loss": 1959.1766,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1787400245666504,
      "rewards/margins": 0.12442886829376221,
      "rewards/rejected": -0.303168922662735,
      "step": 13550
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.9131306039664676e-07,
      "logits/chosen": -2.201411008834839,
      "logits/rejected": -2.1508049964904785,
      "logps/chosen": -209.1273651123047,
      "logps/rejected": -252.2547149658203,
      "loss": 1552.3877,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09223302453756332,
      "rewards/margins": 0.18770921230316162,
      "rewards/rejected": -0.27994224429130554,
      "step": 13560
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8912842462257358e-07,
      "logits/chosen": -2.23241925239563,
      "logits/rejected": -2.1687450408935547,
      "logps/chosen": -229.80410766601562,
      "logps/rejected": -234.39785766601562,
      "loss": 1461.8519,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09634847939014435,
      "rewards/margins": 0.21580667793750763,
      "rewards/rejected": -0.312155157327652,
      "step": 13570
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.869558434866303e-07,
      "logits/chosen": -2.3052706718444824,
      "logits/rejected": -2.36867356300354,
      "logps/chosen": -195.76123046875,
      "logps/rejected": -230.22836303710938,
      "loss": 1875.9637,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.14671394228935242,
      "rewards/margins": 0.16437289118766785,
      "rewards/rejected": -0.31108683347702026,
      "step": 13580
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.847953283220652e-07,
      "logits/chosen": -2.4721105098724365,
      "logits/rejected": -2.1537983417510986,
      "logps/chosen": -265.67724609375,
      "logps/rejected": -210.75735473632812,
      "loss": 1502.0496,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0956752747297287,
      "rewards/margins": 0.2371629923582077,
      "rewards/rejected": -0.332838237285614,
      "step": 13590
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8264689039918265e-07,
      "logits/chosen": -2.440962553024292,
      "logits/rejected": -2.101656436920166,
      "logps/chosen": -273.24176025390625,
      "logps/rejected": -256.9324951171875,
      "loss": 2175.7262,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13411381840705872,
      "rewards/margins": 0.11116938292980194,
      "rewards/rejected": -0.24528317153453827,
      "step": 13600
    },
    {
      "epoch": 0.89,
      "eval_logits/chosen": -2.345820665359497,
      "eval_logits/rejected": -2.1585347652435303,
      "eval_logps/chosen": -243.89022827148438,
      "eval_logps/rejected": -239.07879638671875,
      "eval_loss": 1754.93798828125,
      "eval_rewards/accuracies": 0.6650000214576721,
      "eval_rewards/chosen": -0.11885278671979904,
      "eval_rewards/margins": 0.15581615269184113,
      "eval_rewards/rejected": -0.2746689021587372,
      "eval_runtime": 722.8485,
      "eval_samples_per_second": 2.767,
      "eval_steps_per_second": 1.383,
      "step": 13600
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8051054092528857e-07,
      "logits/chosen": -2.369603395462036,
      "logits/rejected": -2.187791109085083,
      "logps/chosen": -272.3529968261719,
      "logps/rejected": -278.944580078125,
      "loss": 1839.5477,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11251578480005264,
      "rewards/margins": 0.1635177880525589,
      "rewards/rejected": -0.27603358030319214,
      "step": 13610
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.783862910446271e-07,
      "logits/chosen": -2.014069080352783,
      "logits/rejected": -2.154714822769165,
      "logps/chosen": -184.99124145507812,
      "logps/rejected": -192.1890869140625,
      "loss": 1712.8154,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12137937545776367,
      "rewards/margins": 0.16512703895568848,
      "rewards/rejected": -0.28650641441345215,
      "step": 13620
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.762741518383271e-07,
      "logits/chosen": -2.3821609020233154,
      "logits/rejected": -2.225235939025879,
      "logps/chosen": -235.3060302734375,
      "logps/rejected": -222.5266876220703,
      "loss": 1856.7225,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13473662734031677,
      "rewards/margins": 0.1488475352525711,
      "rewards/rejected": -0.2835841178894043,
      "step": 13630
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7417413432434082e-07,
      "logits/chosen": -2.453998565673828,
      "logits/rejected": -2.059866428375244,
      "logps/chosen": -266.7665100097656,
      "logps/rejected": -228.9359130859375,
      "loss": 1901.8699,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1410680115222931,
      "rewards/margins": 0.14936773478984833,
      "rewards/rejected": -0.2904357612133026,
      "step": 13640
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7208624945738855e-07,
      "logits/chosen": -2.4492390155792236,
      "logits/rejected": -2.297363758087158,
      "logps/chosen": -230.3117218017578,
      "logps/rejected": -250.00094604492188,
      "loss": 1887.1781,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12227406352758408,
      "rewards/margins": 0.12251828610897064,
      "rewards/rejected": -0.24479234218597412,
      "step": 13650
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7001050812889995e-07,
      "logits/chosen": -2.42549467086792,
      "logits/rejected": -2.1087913513183594,
      "logps/chosen": -273.09515380859375,
      "logps/rejected": -245.15701293945312,
      "loss": 1939.2883,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17562313377857208,
      "rewards/margins": 0.12312822043895721,
      "rewards/rejected": -0.2987513840198517,
      "step": 13660
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.679469211669596e-07,
      "logits/chosen": -2.3549065589904785,
      "logits/rejected": -2.198781728744507,
      "logps/chosen": -238.4629669189453,
      "logps/rejected": -202.82814025878906,
      "loss": 1538.0891,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.13517813384532928,
      "rewards/margins": 0.159695565700531,
      "rewards/rejected": -0.2948737144470215,
      "step": 13670
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6589549933624715e-07,
      "logits/chosen": -2.331890106201172,
      "logits/rejected": -2.1530439853668213,
      "logps/chosen": -252.07125854492188,
      "logps/rejected": -222.9897918701172,
      "loss": 1462.2755,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1186424270272255,
      "rewards/margins": 0.22403955459594727,
      "rewards/rejected": -0.34268200397491455,
      "step": 13680
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.638562533379845e-07,
      "logits/chosen": -2.3359556198120117,
      "logits/rejected": -2.1710317134857178,
      "logps/chosen": -265.05255126953125,
      "logps/rejected": -217.8121795654297,
      "loss": 1876.0527,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0934990718960762,
      "rewards/margins": 0.15027651190757751,
      "rewards/rejected": -0.24377556145191193,
      "step": 13690
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6182919380987676e-07,
      "logits/chosen": -2.3831872940063477,
      "logits/rejected": -2.301400661468506,
      "logps/chosen": -238.47140502929688,
      "logps/rejected": -232.7339630126953,
      "loss": 2119.6848,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12753424048423767,
      "rewards/margins": 0.09639700502157211,
      "rewards/rejected": -0.2239312380552292,
      "step": 13700
    },
    {
      "epoch": 0.9,
      "eval_logits/chosen": -2.344073534011841,
      "eval_logits/rejected": -2.15694260597229,
      "eval_logps/chosen": -244.05441284179688,
      "eval_logps/rejected": -239.03968811035156,
      "eval_loss": 1754.270263671875,
      "eval_rewards/accuracies": 0.6654999852180481,
      "eval_rewards/chosen": -0.12049450725317001,
      "eval_rewards/margins": 0.1537831723690033,
      "eval_rewards/rejected": -0.2742776870727539,
      "eval_runtime": 722.7989,
      "eval_samples_per_second": 2.767,
      "eval_steps_per_second": 1.384,
      "step": 13700
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.598143313260603e-07,
      "logits/chosen": -2.3030202388763428,
      "logits/rejected": -2.205902338027954,
      "logps/chosen": -202.69361877441406,
      "logps/rejected": -203.13424682617188,
      "loss": 1727.1043,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1463984251022339,
      "rewards/margins": 0.12455177307128906,
      "rewards/rejected": -0.27095022797584534,
      "step": 13710
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.5781167639704415e-07,
      "logits/chosen": -2.5292487144470215,
      "logits/rejected": -1.9871896505355835,
      "logps/chosen": -342.32806396484375,
      "logps/rejected": -234.9232940673828,
      "loss": 1738.8637,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10161248594522476,
      "rewards/margins": 0.16838693618774414,
      "rewards/rejected": -0.2699994444847107,
      "step": 13720
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.5582123946965787e-07,
      "logits/chosen": -2.20857310295105,
      "logits/rejected": -2.0954978466033936,
      "logps/chosen": -247.48794555664062,
      "logps/rejected": -272.25823974609375,
      "loss": 1700.593,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1309293508529663,
      "rewards/margins": 0.14622342586517334,
      "rewards/rejected": -0.27715277671813965,
      "step": 13730
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.5384303092699504e-07,
      "logits/chosen": -2.407092571258545,
      "logits/rejected": -2.183859348297119,
      "logps/chosen": -292.9111328125,
      "logps/rejected": -317.5841064453125,
      "loss": 1062.2133,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05926471948623657,
      "rewards/margins": 0.263541042804718,
      "rewards/rejected": -0.322805792093277,
      "step": 13740
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.518770610883613e-07,
      "logits/chosen": -2.2973599433898926,
      "logits/rejected": -2.0243818759918213,
      "logps/chosen": -228.43264770507812,
      "logps/rejected": -219.4680633544922,
      "loss": 1424.9246,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.13893964886665344,
      "rewards/margins": 0.20449288189411163,
      "rewards/rejected": -0.3434325158596039,
      "step": 13750
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4992334020921735e-07,
      "logits/chosen": -2.292459487915039,
      "logits/rejected": -2.1910786628723145,
      "logps/chosen": -180.3132781982422,
      "logps/rejected": -172.5894775390625,
      "loss": 1297.7433,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07340843975543976,
      "rewards/margins": 0.21890044212341309,
      "rewards/rejected": -0.29230886697769165,
      "step": 13760
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4798187848112905e-07,
      "logits/chosen": -2.2184767723083496,
      "logits/rejected": -2.228837490081787,
      "logps/chosen": -242.86160278320312,
      "logps/rejected": -216.32040405273438,
      "loss": 1840.3234,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19890835881233215,
      "rewards/margins": 0.14316312968730927,
      "rewards/rejected": -0.3420714735984802,
      "step": 13770
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.460526860317113e-07,
      "logits/chosen": -2.4182865619659424,
      "logits/rejected": -2.3499302864074707,
      "logps/chosen": -186.46209716796875,
      "logps/rejected": -236.97122192382812,
      "loss": 1507.47,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1317743957042694,
      "rewards/margins": 0.1876230686903,
      "rewards/rejected": -0.3193974494934082,
      "step": 13780
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.441357729245771e-07,
      "logits/chosen": -2.5390188694000244,
      "logits/rejected": -1.9643266201019287,
      "logps/chosen": -268.406005859375,
      "logps/rejected": -223.5176544189453,
      "loss": 1951.6914,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2041414976119995,
      "rewards/margins": 0.11098869889974594,
      "rewards/rejected": -0.3151302635669708,
      "step": 13790
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4223114915928482e-07,
      "logits/chosen": -2.1964828968048096,
      "logits/rejected": -1.9413315057754517,
      "logps/chosen": -238.6462860107422,
      "logps/rejected": -262.0941467285156,
      "loss": 1448.3924,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.12723703682422638,
      "rewards/margins": 0.1668069064617157,
      "rewards/rejected": -0.29404395818710327,
      "step": 13800
    },
    {
      "epoch": 0.9,
      "eval_logits/chosen": -2.345473289489746,
      "eval_logits/rejected": -2.158222198486328,
      "eval_logps/chosen": -243.75439453125,
      "eval_logps/rejected": -238.92115783691406,
      "eval_loss": 1754.062255859375,
      "eval_rewards/accuracies": 0.6650000214576721,
      "eval_rewards/chosen": -0.11749444156885147,
      "eval_rewards/margins": 0.15559816360473633,
      "eval_rewards/rejected": -0.2730926275253296,
      "eval_runtime": 723.8834,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.381,
      "step": 13800
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.403388246712842e-07,
      "logits/chosen": -2.2534501552581787,
      "logits/rejected": -2.0139777660369873,
      "logps/chosen": -177.90037536621094,
      "logps/rejected": -181.38209533691406,
      "loss": 1909.3896,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13649320602416992,
      "rewards/margins": 0.10633499920368195,
      "rewards/rejected": -0.24282817542552948,
      "step": 13810
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.3845880933186757e-07,
      "logits/chosen": -2.5113167762756348,
      "logits/rejected": -2.246593952178955,
      "logps/chosen": -249.75888061523438,
      "logps/rejected": -226.8916778564453,
      "loss": 1985.9889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11020083725452423,
      "rewards/margins": 0.11368707567453384,
      "rewards/rejected": -0.22388792037963867,
      "step": 13820
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.3659111294811457e-07,
      "logits/chosen": -2.345294237136841,
      "logits/rejected": -2.2051594257354736,
      "logps/chosen": -205.89266967773438,
      "logps/rejected": -203.26119995117188,
      "loss": 1693.6232,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1404949128627777,
      "rewards/margins": 0.15264761447906494,
      "rewards/rejected": -0.29314249753952026,
      "step": 13830
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.347357452628459e-07,
      "logits/chosen": -2.4888830184936523,
      "logits/rejected": -2.35544490814209,
      "logps/chosen": -254.90744018554688,
      "logps/rejected": -262.67059326171875,
      "loss": 1875.2223,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09609898179769516,
      "rewards/margins": 0.1526482105255127,
      "rewards/rejected": -0.24874719977378845,
      "step": 13840
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.3289271595456732e-07,
      "logits/chosen": -2.2717719078063965,
      "logits/rejected": -2.0894250869750977,
      "logps/chosen": -220.32534790039062,
      "logps/rejected": -210.92330932617188,
      "loss": 1640.549,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.15996167063713074,
      "rewards/margins": 0.16391651332378387,
      "rewards/rejected": -0.3238781988620758,
      "step": 13850
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.310620346374228e-07,
      "logits/chosen": -2.1835789680480957,
      "logits/rejected": -2.0586397647857666,
      "logps/chosen": -243.33395385742188,
      "logps/rejected": -223.63174438476562,
      "loss": 1724.6162,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11112016439437866,
      "rewards/margins": 0.1710091084241867,
      "rewards/rejected": -0.2821292579174042,
      "step": 13860
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2924371086114274e-07,
      "logits/chosen": -2.2265589237213135,
      "logits/rejected": -1.989508032798767,
      "logps/chosen": -248.4755401611328,
      "logps/rejected": -258.7160339355469,
      "loss": 1779.283,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12414033710956573,
      "rewards/margins": 0.1626734584569931,
      "rewards/rejected": -0.28681379556655884,
      "step": 13870
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.274377541109953e-07,
      "logits/chosen": -2.1988863945007324,
      "logits/rejected": -2.268820285797119,
      "logps/chosen": -178.72723388671875,
      "logps/rejected": -268.6763610839844,
      "loss": 1886.5996,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.14376714825630188,
      "rewards/margins": 0.12025441229343414,
      "rewards/rejected": -0.2640215754508972,
      "step": 13880
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2564417380773435e-07,
      "logits/chosen": -2.1269640922546387,
      "logits/rejected": -1.9930492639541626,
      "logps/chosen": -192.80255126953125,
      "logps/rejected": -233.09622192382812,
      "loss": 1588.2561,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.14305493235588074,
      "rewards/margins": 0.178117036819458,
      "rewards/rejected": -0.32117193937301636,
      "step": 13890
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2386297930755436e-07,
      "logits/chosen": -2.337871789932251,
      "logits/rejected": -2.3502695560455322,
      "logps/chosen": -267.2091064453125,
      "logps/rejected": -277.0139465332031,
      "loss": 1953.3191,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1932331919670105,
      "rewards/margins": 0.1463126540184021,
      "rewards/rejected": -0.3395458161830902,
      "step": 13900
    },
    {
      "epoch": 0.91,
      "eval_logits/chosen": -2.3461110591888428,
      "eval_logits/rejected": -2.1588072776794434,
      "eval_logps/chosen": -243.5794677734375,
      "eval_logps/rejected": -238.8184051513672,
      "eval_loss": 1754.0230712890625,
      "eval_rewards/accuracies": 0.6650000214576721,
      "eval_rewards/chosen": -0.1157451868057251,
      "eval_rewards/margins": 0.15631982684135437,
      "eval_rewards/rejected": -0.27206501364707947,
      "eval_runtime": 725.5684,
      "eval_samples_per_second": 2.756,
      "eval_steps_per_second": 1.378,
      "step": 13900
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.220941799020378e-07,
      "logits/chosen": -2.123575448989868,
      "logits/rejected": -2.0426182746887207,
      "logps/chosen": -230.9011688232422,
      "logps/rejected": -225.5489959716797,
      "loss": 1631.8912,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11820058524608612,
      "rewards/margins": 0.17000697553157806,
      "rewards/rejected": -0.2882075905799866,
      "step": 13910
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2033778481810975e-07,
      "logits/chosen": -2.401193380355835,
      "logits/rejected": -2.134850263595581,
      "logps/chosen": -230.31613159179688,
      "logps/rejected": -210.4309844970703,
      "loss": 1748.4959,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10322584211826324,
      "rewards/margins": 0.17461058497428894,
      "rewards/rejected": -0.277836412191391,
      "step": 13920
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1859380321798591e-07,
      "logits/chosen": -2.343634605407715,
      "logits/rejected": -2.407421588897705,
      "logps/chosen": -216.51547241210938,
      "logps/rejected": -242.3054656982422,
      "loss": 1944.4545,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1395423412322998,
      "rewards/margins": 0.11045052856206894,
      "rewards/rejected": -0.24999287724494934,
      "step": 13930
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1686224419912989e-07,
      "logits/chosen": -2.2427189350128174,
      "logits/rejected": -2.036081314086914,
      "logps/chosen": -264.5870666503906,
      "logps/rejected": -256.62005615234375,
      "loss": 1467.0031,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1171443909406662,
      "rewards/margins": 0.21323764324188232,
      "rewards/rejected": -0.33038201928138733,
      "step": 13940
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1514311679420104e-07,
      "logits/chosen": -2.0678160190582275,
      "logits/rejected": -2.1402063369750977,
      "logps/chosen": -172.3576202392578,
      "logps/rejected": -238.7569122314453,
      "loss": 2250.3074,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17209061980247498,
      "rewards/margins": 0.10579583793878555,
      "rewards/rejected": -0.27788645029067993,
      "step": 13950
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1343642997101029e-07,
      "logits/chosen": -2.3408255577087402,
      "logits/rejected": -2.248295307159424,
      "logps/chosen": -213.69570922851562,
      "logps/rejected": -214.92318725585938,
      "loss": 1828.8832,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11342382431030273,
      "rewards/margins": 0.1444929540157318,
      "rewards/rejected": -0.25791677832603455,
      "step": 13960
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1174219263247188e-07,
      "logits/chosen": -2.0945498943328857,
      "logits/rejected": -1.986140251159668,
      "logps/chosen": -211.49118041992188,
      "logps/rejected": -206.6245880126953,
      "loss": 1908.091,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12673640251159668,
      "rewards/margins": 0.12664535641670227,
      "rewards/rejected": -0.25338175892829895,
      "step": 13970
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1006041361655839e-07,
      "logits/chosen": -2.5157954692840576,
      "logits/rejected": -2.055115222930908,
      "logps/chosen": -220.3848419189453,
      "logps/rejected": -195.51856994628906,
      "loss": 1765.3135,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11947546899318695,
      "rewards/margins": 0.15071254968643188,
      "rewards/rejected": -0.27018803358078003,
      "step": 13980
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0839110169625189e-07,
      "logits/chosen": -2.0732357501983643,
      "logits/rejected": -2.3762660026550293,
      "logps/chosen": -222.16049194335938,
      "logps/rejected": -235.0498046875,
      "loss": 1506.5421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15318220853805542,
      "rewards/margins": 0.2024463713169098,
      "rewards/rejected": -0.3556285500526428,
      "step": 13990
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.06734265579502e-07,
      "logits/chosen": -2.355957508087158,
      "logits/rejected": -2.0589470863342285,
      "logps/chosen": -274.600830078125,
      "logps/rejected": -224.1596221923828,
      "loss": 1684.251,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13084113597869873,
      "rewards/margins": 0.1606401652097702,
      "rewards/rejected": -0.2914813160896301,
      "step": 14000
    },
    {
      "epoch": 0.92,
      "eval_logits/chosen": -2.3463921546936035,
      "eval_logits/rejected": -2.1591100692749023,
      "eval_logps/chosen": -243.4635772705078,
      "eval_logps/rejected": -238.63746643066406,
      "eval_loss": 1754.74755859375,
      "eval_rewards/accuracies": 0.6635000109672546,
      "eval_rewards/chosen": -0.11458618193864822,
      "eval_rewards/margins": 0.15566939115524292,
      "eval_rewards/rejected": -0.27025559544563293,
      "eval_runtime": 722.042,
      "eval_samples_per_second": 2.77,
      "eval_steps_per_second": 1.385,
      "step": 14000
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.050899139091771e-07,
      "logits/chosen": -2.4245903491973877,
      "logits/rejected": -2.08056378364563,
      "logps/chosen": -289.9082336425781,
      "logps/rejected": -264.00091552734375,
      "loss": 1234.4372,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11280794441699982,
      "rewards/margins": 0.2061082124710083,
      "rewards/rejected": -0.3189161419868469,
      "step": 14010
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0345805526302072e-07,
      "logits/chosen": -2.261943817138672,
      "logits/rejected": -2.374112129211426,
      "logps/chosen": -212.29641723632812,
      "logps/rejected": -223.0238037109375,
      "loss": 1472.3037,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.11134089529514313,
      "rewards/margins": 0.17965057492256165,
      "rewards/rejected": -0.29099148511886597,
      "step": 14020
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0183869815360764e-07,
      "logits/chosen": -2.2483181953430176,
      "logits/rejected": -2.3696656227111816,
      "logps/chosen": -203.56878662109375,
      "logps/rejected": -250.1732940673828,
      "loss": 1821.0598,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11655312776565552,
      "rewards/margins": 0.13964414596557617,
      "rewards/rejected": -0.2561972737312317,
      "step": 14030
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0023185102829763e-07,
      "logits/chosen": -2.061488389968872,
      "logits/rejected": -2.2805399894714355,
      "logps/chosen": -243.485595703125,
      "logps/rejected": -264.13775634765625,
      "loss": 1707.124,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.126633882522583,
      "rewards/margins": 0.16467411816120148,
      "rewards/rejected": -0.2913079857826233,
      "step": 14040
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.863752226919182e-08,
      "logits/chosen": -2.27036452293396,
      "logits/rejected": -1.7793105840682983,
      "logps/chosen": -247.064453125,
      "logps/rejected": -203.147705078125,
      "loss": 1475.9483,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09332261979579926,
      "rewards/margins": 0.2166186273097992,
      "rewards/rejected": -0.30994123220443726,
      "step": 14050
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.705572019309107e-08,
      "logits/chosen": -2.187352180480957,
      "logits/rejected": -2.2290992736816406,
      "logps/chosen": -281.07415771484375,
      "logps/rejected": -265.98870849609375,
      "loss": 1752.7209,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13046763837337494,
      "rewards/margins": 0.1639849692583084,
      "rewards/rejected": -0.29445260763168335,
      "step": 14060
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.548645305144849e-08,
      "logits/chosen": -2.4042441844940186,
      "logits/rejected": -2.273912191390991,
      "logps/chosen": -183.91830444335938,
      "logps/rejected": -204.17153930664062,
      "loss": 1756.1881,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.10423145443201065,
      "rewards/margins": 0.16279512643814087,
      "rewards/rejected": -0.2670265734195709,
      "step": 14070
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.392972903033149e-08,
      "logits/chosen": -2.330324649810791,
      "logits/rejected": -2.176776170730591,
      "logps/chosen": -232.614013671875,
      "logps/rejected": -234.63394165039062,
      "loss": 2016.0381,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.09482771158218384,
      "rewards/margins": 0.09563405811786652,
      "rewards/rejected": -0.19046176970005035,
      "step": 14080
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.238555625037449e-08,
      "logits/chosen": -2.350982189178467,
      "logits/rejected": -2.135450839996338,
      "logps/chosen": -203.74049377441406,
      "logps/rejected": -186.6983184814453,
      "loss": 1871.374,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1303228735923767,
      "rewards/margins": 0.12639638781547546,
      "rewards/rejected": -0.25671929121017456,
      "step": 14090
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.085394276673903e-08,
      "logits/chosen": -2.3481802940368652,
      "logits/rejected": -2.045968770980835,
      "logps/chosen": -280.8945007324219,
      "logps/rejected": -282.7396240234375,
      "loss": 1545.3156,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.12728367745876312,
      "rewards/margins": 0.16294625401496887,
      "rewards/rejected": -0.2902299463748932,
      "step": 14100
    },
    {
      "epoch": 0.92,
      "eval_logits/chosen": -2.3445770740509033,
      "eval_logits/rejected": -2.157391309738159,
      "eval_logps/chosen": -243.55126953125,
      "eval_logps/rejected": -238.72972106933594,
      "eval_loss": 1755.1064453125,
      "eval_rewards/accuracies": 0.6650000214576721,
      "eval_rewards/chosen": -0.11546298116445541,
      "eval_rewards/margins": 0.15571531653404236,
      "eval_rewards/rejected": -0.27117830514907837,
      "eval_runtime": 725.3595,
      "eval_samples_per_second": 2.757,
      "eval_steps_per_second": 1.379,
      "step": 14100
    },
    {
      "epoch": 0.92,
      "learning_rate": 8.933489656907157e-08,
      "logits/chosen": -2.3288893699645996,
      "logits/rejected": -2.2169697284698486,
      "logps/chosen": -231.6782684326172,
      "logps/rejected": -265.7174072265625,
      "loss": 2114.174,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.14342829585075378,
      "rewards/margins": 0.09098762273788452,
      "rewards/rejected": -0.2344159185886383,
      "step": 14110
    },
    {
      "epoch": 0.92,
      "learning_rate": 8.782842558146127e-08,
      "logits/chosen": -2.391118288040161,
      "logits/rejected": -2.3201541900634766,
      "logps/chosen": -170.07455444335938,
      "logps/rejected": -187.50924682617188,
      "loss": 1380.3342,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08190619945526123,
      "rewards/margins": 0.1914120465517044,
      "rewards/rejected": -0.2733182907104492,
      "step": 14120
    },
    {
      "epoch": 0.92,
      "learning_rate": 8.633453766239836e-08,
      "logits/chosen": -2.4329991340637207,
      "logits/rejected": -2.2017290592193604,
      "logps/chosen": -243.4434356689453,
      "logps/rejected": -222.77706909179688,
      "loss": 1938.3652,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08322300016880035,
      "rewards/margins": 0.11188875138759613,
      "rewards/rejected": -0.19511176645755768,
      "step": 14130
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.485324060473448e-08,
      "logits/chosen": -2.2668330669403076,
      "logits/rejected": -2.1573243141174316,
      "logps/chosen": -252.8908233642578,
      "logps/rejected": -254.57229614257812,
      "loss": 1929.8533,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12608733773231506,
      "rewards/margins": 0.14072270691394806,
      "rewards/rejected": -0.26681002974510193,
      "step": 14140
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.338454213564052e-08,
      "logits/chosen": -2.3068039417266846,
      "logits/rejected": -2.0356509685516357,
      "logps/chosen": -242.44216918945312,
      "logps/rejected": -246.11767578125,
      "loss": 1416.0274,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.11545213311910629,
      "rewards/margins": 0.2086973637342453,
      "rewards/rejected": -0.3241495192050934,
      "step": 14150
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.192844991656679e-08,
      "logits/chosen": -2.3066954612731934,
      "logits/rejected": -2.0747666358947754,
      "logps/chosen": -250.7014617919922,
      "logps/rejected": -228.00930786132812,
      "loss": 1824.4467,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11984783411026001,
      "rewards/margins": 0.13715848326683044,
      "rewards/rejected": -0.25700634717941284,
      "step": 14160
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.048497154320434e-08,
      "logits/chosen": -2.3507096767425537,
      "logits/rejected": -2.395566463470459,
      "logps/chosen": -146.2378692626953,
      "logps/rejected": -162.6152801513672,
      "loss": 2089.841,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1668834388256073,
      "rewards/margins": 0.07279939204454422,
      "rewards/rejected": -0.23968283832073212,
      "step": 14170
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.905411454544265e-08,
      "logits/chosen": -2.3429198265075684,
      "logits/rejected": -2.190000295639038,
      "logps/chosen": -250.3459014892578,
      "logps/rejected": -266.3189392089844,
      "loss": 2110.5916,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1384165734052658,
      "rewards/margins": 0.11230772733688354,
      "rewards/rejected": -0.25072425603866577,
      "step": 14180
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.763588638733332e-08,
      "logits/chosen": -2.357968807220459,
      "logits/rejected": -2.3119089603424072,
      "logps/chosen": -275.1587829589844,
      "logps/rejected": -268.12847900390625,
      "loss": 1896.2953,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12738670408725739,
      "rewards/margins": 0.12793752551078796,
      "rewards/rejected": -0.25532421469688416,
      "step": 14190
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.623029446704899e-08,
      "logits/chosen": -2.21120023727417,
      "logits/rejected": -2.3920819759368896,
      "logps/chosen": -319.13970947265625,
      "logps/rejected": -307.9252014160156,
      "loss": 1391.3224,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07698237895965576,
      "rewards/margins": 0.2568013072013855,
      "rewards/rejected": -0.33378368616104126,
      "step": 14200
    },
    {
      "epoch": 0.93,
      "eval_logits/chosen": -2.343921184539795,
      "eval_logits/rejected": -2.1568198204040527,
      "eval_logps/chosen": -243.383056640625,
      "eval_logps/rejected": -238.62802124023438,
      "eval_loss": 1756.5655517578125,
      "eval_rewards/accuracies": 0.6614999771118164,
      "eval_rewards/chosen": -0.11378104984760284,
      "eval_rewards/margins": 0.15638026595115662,
      "eval_rewards/rejected": -0.2701612710952759,
      "eval_runtime": 724.3082,
      "eval_samples_per_second": 2.761,
      "eval_steps_per_second": 1.381,
      "step": 14200
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.483734611684557e-08,
      "logits/chosen": -2.137566089630127,
      "logits/rejected": -1.9804027080535889,
      "logps/chosen": -263.27410888671875,
      "logps/rejected": -228.4842987060547,
      "loss": 1619.2999,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.063882976770401,
      "rewards/margins": 0.19371116161346436,
      "rewards/rejected": -0.25759413838386536,
      "step": 14210
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.345704860302366e-08,
      "logits/chosen": -2.4090185165405273,
      "logits/rejected": -2.4050068855285645,
      "logps/chosen": -257.5478515625,
      "logps/rejected": -277.10076904296875,
      "loss": 1403.6926,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09295176714658737,
      "rewards/margins": 0.20231318473815918,
      "rewards/rejected": -0.29526492953300476,
      "step": 14220
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.208940912589224e-08,
      "logits/chosen": -2.3491759300231934,
      "logits/rejected": -2.019956350326538,
      "logps/chosen": -223.9944610595703,
      "logps/rejected": -205.76278686523438,
      "loss": 1605.3204,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1440814882516861,
      "rewards/margins": 0.19146788120269775,
      "rewards/rejected": -0.33554935455322266,
      "step": 14230
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.073443481972753e-08,
      "logits/chosen": -2.1687004566192627,
      "logits/rejected": -2.1498425006866455,
      "logps/chosen": -200.83572387695312,
      "logps/rejected": -232.8833770751953,
      "loss": 2132.1936,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.16679982841014862,
      "rewards/margins": 0.12545950710773468,
      "rewards/rejected": -0.2922593355178833,
      "step": 14240
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.939213275274027e-08,
      "logits/chosen": -2.2963318824768066,
      "logits/rejected": -2.25905179977417,
      "logps/chosen": -251.19442749023438,
      "logps/rejected": -244.81991577148438,
      "loss": 1795.9357,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11940628290176392,
      "rewards/margins": 0.13228197395801544,
      "rewards/rejected": -0.25168827176094055,
      "step": 14250
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.806250992703461e-08,
      "logits/chosen": -2.3213610649108887,
      "logits/rejected": -2.1692914962768555,
      "logps/chosen": -225.44091796875,
      "logps/rejected": -215.24996948242188,
      "loss": 1622.4871,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08746316283941269,
      "rewards/margins": 0.16019216179847717,
      "rewards/rejected": -0.24765534698963165,
      "step": 14260
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.674557327857572e-08,
      "logits/chosen": -2.313218593597412,
      "logits/rejected": -2.326613664627075,
      "logps/chosen": -266.9938659667969,
      "logps/rejected": -275.0752258300781,
      "loss": 1828.6219,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12723203003406525,
      "rewards/margins": 0.17169377207756042,
      "rewards/rejected": -0.29892581701278687,
      "step": 14270
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.544132967714917e-08,
      "logits/chosen": -2.0654211044311523,
      "logits/rejected": -2.0778846740722656,
      "logps/chosen": -262.28765869140625,
      "logps/rejected": -264.19000244140625,
      "loss": 2270.5215,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15427663922309875,
      "rewards/margins": 0.15614831447601318,
      "rewards/rejected": -0.31042495369911194,
      "step": 14280
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.414978592632932e-08,
      "logits/chosen": -2.394740104675293,
      "logits/rejected": -1.9486885070800781,
      "logps/chosen": -274.8138732910156,
      "logps/rejected": -241.82949829101562,
      "loss": 1756.9453,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1371452510356903,
      "rewards/margins": 0.13471072912216187,
      "rewards/rejected": -0.2718559801578522,
      "step": 14290
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.287094876344046e-08,
      "logits/chosen": -2.3430347442626953,
      "logits/rejected": -2.3814728260040283,
      "logps/chosen": -179.88137817382812,
      "logps/rejected": -201.48204040527344,
      "loss": 1588.1222,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07461106032133102,
      "rewards/margins": 0.14961650967597961,
      "rewards/rejected": -0.22422757744789124,
      "step": 14300
    },
    {
      "epoch": 0.94,
      "eval_logits/chosen": -2.345165967941284,
      "eval_logits/rejected": -2.157975196838379,
      "eval_logps/chosen": -243.50830078125,
      "eval_logps/rejected": -238.7296142578125,
      "eval_loss": 1755.2308349609375,
      "eval_rewards/accuracies": 0.6650000214576721,
      "eval_rewards/chosen": -0.1150333434343338,
      "eval_rewards/margins": 0.15614359080791473,
      "eval_rewards/rejected": -0.27117693424224854,
      "eval_runtime": 723.0581,
      "eval_samples_per_second": 2.766,
      "eval_steps_per_second": 1.383,
      "step": 14300
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.160482485952413e-08,
      "logits/chosen": -2.4855408668518066,
      "logits/rejected": -2.2154054641723633,
      "logps/chosen": -248.1533660888672,
      "logps/rejected": -232.2559814453125,
      "loss": 1629.3428,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11703590303659439,
      "rewards/margins": 0.15898486971855164,
      "rewards/rejected": -0.27602076530456543,
      "step": 14310
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.035142081930234e-08,
      "logits/chosen": -2.3523106575012207,
      "logits/rejected": -1.9807660579681396,
      "logps/chosen": -278.71728515625,
      "logps/rejected": -214.4222412109375,
      "loss": 1711.0033,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15920953452587128,
      "rewards/margins": 0.12721839547157288,
      "rewards/rejected": -0.28642791509628296,
      "step": 14320
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.911074318114496e-08,
      "logits/chosen": -2.1858577728271484,
      "logits/rejected": -2.3020291328430176,
      "logps/chosen": -218.28091430664062,
      "logps/rejected": -275.77850341796875,
      "loss": 1649.2277,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14692769944667816,
      "rewards/margins": 0.15372733771800995,
      "rewards/rejected": -0.3006550669670105,
      "step": 14330
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.788279841703381e-08,
      "logits/chosen": -2.386441707611084,
      "logits/rejected": -2.135468006134033,
      "logps/chosen": -195.0791015625,
      "logps/rejected": -205.4237823486328,
      "loss": 1574.9247,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11169525235891342,
      "rewards/margins": 0.1801174432039261,
      "rewards/rejected": -0.2918127179145813,
      "step": 14340
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.66675929325311e-08,
      "logits/chosen": -2.3844265937805176,
      "logits/rejected": -2.1434741020202637,
      "logps/chosen": -230.9381103515625,
      "logps/rejected": -237.17324829101562,
      "loss": 1654.5648,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10273660719394684,
      "rewards/margins": 0.17583905160427094,
      "rewards/rejected": -0.278575599193573,
      "step": 14350
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.546513306674301e-08,
      "logits/chosen": -2.3126797676086426,
      "logits/rejected": -1.9176400899887085,
      "logps/chosen": -291.2501525878906,
      "logps/rejected": -231.6427459716797,
      "loss": 1717.066,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.127334862947464,
      "rewards/margins": 0.15475480258464813,
      "rewards/rejected": -0.2820896506309509,
      "step": 14360
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.4275425092290004e-08,
      "logits/chosen": -2.4252126216888428,
      "logits/rejected": -2.3836467266082764,
      "logps/chosen": -263.10675048828125,
      "logps/rejected": -262.96209716796875,
      "loss": 1506.7836,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08310611546039581,
      "rewards/margins": 0.1829802691936493,
      "rewards/rejected": -0.2660863399505615,
      "step": 14370
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.309847521527078e-08,
      "logits/chosen": -2.289580821990967,
      "logits/rejected": -1.9175958633422852,
      "logps/chosen": -294.80340576171875,
      "logps/rejected": -267.6952819824219,
      "loss": 1548.7974,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10230030119419098,
      "rewards/margins": 0.15820232033729553,
      "rewards/rejected": -0.2605026364326477,
      "step": 14380
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.1934289575233385e-08,
      "logits/chosen": -2.164163112640381,
      "logits/rejected": -1.8297512531280518,
      "logps/chosen": -254.22085571289062,
      "logps/rejected": -234.9806671142578,
      "loss": 1854.2094,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14691409468650818,
      "rewards/margins": 0.15127232670783997,
      "rewards/rejected": -0.29818639159202576,
      "step": 14390
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.078287424513994e-08,
      "logits/chosen": -2.4062767028808594,
      "logits/rejected": -2.2998061180114746,
      "logps/chosen": -285.44677734375,
      "logps/rejected": -225.17855834960938,
      "loss": 1734.7881,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15399052202701569,
      "rewards/margins": 0.14562062919139862,
      "rewards/rejected": -0.2996111512184143,
      "step": 14400
    },
    {
      "epoch": 0.94,
      "eval_logits/chosen": -2.3442108631134033,
      "eval_logits/rejected": -2.1570913791656494,
      "eval_logps/chosen": -243.45433044433594,
      "eval_logps/rejected": -238.75839233398438,
      "eval_loss": 1755.41015625,
      "eval_rewards/accuracies": 0.6660000085830688,
      "eval_rewards/chosen": -0.11449379473924637,
      "eval_rewards/margins": 0.15697112679481506,
      "eval_rewards/rejected": -0.27146491408348083,
      "eval_runtime": 726.2683,
      "eval_samples_per_second": 2.754,
      "eval_steps_per_second": 1.377,
      "step": 14400
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.964423523133671e-08,
      "logits/chosen": -2.4076712131500244,
      "logits/rejected": -2.1759860515594482,
      "logps/chosen": -225.8399200439453,
      "logps/rejected": -205.88589477539062,
      "loss": 1570.1146,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08108416944742203,
      "rewards/margins": 0.14395181834697723,
      "rewards/rejected": -0.22503598034381866,
      "step": 14410
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.8518378473522976e-08,
      "logits/chosen": -2.2773380279541016,
      "logits/rejected": -2.1288530826568604,
      "logps/chosen": -261.8817138671875,
      "logps/rejected": -267.4660949707031,
      "loss": 2089.3795,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.1137181892991066,
      "rewards/margins": 0.12311627715826035,
      "rewards/rejected": -0.23683448135852814,
      "step": 14420
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.7405309844718584e-08,
      "logits/chosen": -2.20715594291687,
      "logits/rejected": -2.104396343231201,
      "logps/chosen": -209.18472290039062,
      "logps/rejected": -236.4004364013672,
      "loss": 1657.6336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16685688495635986,
      "rewards/margins": 0.18019473552703857,
      "rewards/rejected": -0.34705162048339844,
      "step": 14430
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.630503515123508e-08,
      "logits/chosen": -2.451308250427246,
      "logits/rejected": -2.1538302898406982,
      "logps/chosen": -213.4972686767578,
      "logps/rejected": -178.93968200683594,
      "loss": 1720.5678,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1263124644756317,
      "rewards/margins": 0.15308813750743866,
      "rewards/rejected": -0.27940061688423157,
      "step": 14440
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.5217560132644056e-08,
      "logits/chosen": -2.268876314163208,
      "logits/rejected": -2.199364185333252,
      "logps/chosen": -160.91078186035156,
      "logps/rejected": -193.87796020507812,
      "loss": 1908.1941,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10291179269552231,
      "rewards/margins": 0.14566278457641602,
      "rewards/rejected": -0.24857458472251892,
      "step": 14450
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.41428904617483e-08,
      "logits/chosen": -2.3149216175079346,
      "logits/rejected": -2.306036949157715,
      "logps/chosen": -189.7136993408203,
      "logps/rejected": -209.6298828125,
      "loss": 1795.1039,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12605008482933044,
      "rewards/margins": 0.15255668759346008,
      "rewards/rejected": -0.2786068022251129,
      "step": 14460
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.3081031744550696e-08,
      "logits/chosen": -2.39857816696167,
      "logits/rejected": -2.325693130493164,
      "logps/chosen": -260.35296630859375,
      "logps/rejected": -256.3681945800781,
      "loss": 1418.1006,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07207634299993515,
      "rewards/margins": 0.18342944979667664,
      "rewards/rejected": -0.25550583004951477,
      "step": 14470
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.2031989520227025e-08,
      "logits/chosen": -2.3827738761901855,
      "logits/rejected": -2.1851248741149902,
      "logps/chosen": -236.3775177001953,
      "logps/rejected": -232.0126953125,
      "loss": 1543.5579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08063056319952011,
      "rewards/margins": 0.17522695660591125,
      "rewards/rejected": -0.25585752725601196,
      "step": 14480
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.099576926109461e-08,
      "logits/chosen": -2.4595134258270264,
      "logits/rejected": -1.9548670053482056,
      "logps/chosen": -250.75570678710938,
      "logps/rejected": -188.27928161621094,
      "loss": 1706.1455,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10973851382732391,
      "rewards/margins": 0.16678059101104736,
      "rewards/rejected": -0.27651911973953247,
      "step": 14490
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.997237637258705e-08,
      "logits/chosen": -2.2871203422546387,
      "logits/rejected": -2.351755142211914,
      "logps/chosen": -324.89862060546875,
      "logps/rejected": -300.61224365234375,
      "loss": 1655.0535,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07645292580127716,
      "rewards/margins": 0.19056789577007294,
      "rewards/rejected": -0.2670208513736725,
      "step": 14500
    },
    {
      "epoch": 0.95,
      "eval_logits/chosen": -2.3447322845458984,
      "eval_logits/rejected": -2.157543659210205,
      "eval_logps/chosen": -243.40638732910156,
      "eval_logps/rejected": -238.72279357910156,
      "eval_loss": 1754.942626953125,
      "eval_rewards/accuracies": 0.6629999876022339,
      "eval_rewards/chosen": -0.11401434987783432,
      "eval_rewards/margins": 0.1570945829153061,
      "eval_rewards/rejected": -0.2711089551448822,
      "eval_runtime": 723.1645,
      "eval_samples_per_second": 2.766,
      "eval_steps_per_second": 1.383,
      "step": 14500
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.8961816193222035e-08,
      "logits/chosen": -2.434767246246338,
      "logits/rejected": -2.2198853492736816,
      "logps/chosen": -252.49307250976562,
      "logps/rejected": -203.490234375,
      "loss": 2298.1338,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.16642776131629944,
      "rewards/margins": 0.0833040177822113,
      "rewards/rejected": -0.24973177909851074,
      "step": 14510
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.79640939945769e-08,
      "logits/chosen": -2.3867263793945312,
      "logits/rejected": -2.262131452560425,
      "logps/chosen": -291.5511169433594,
      "logps/rejected": -218.9336700439453,
      "loss": 1704.7814,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05330218747258186,
      "rewards/margins": 0.13646642863750458,
      "rewards/rejected": -0.18976862728595734,
      "step": 14520
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.697921498125895e-08,
      "logits/chosen": -2.1315157413482666,
      "logits/rejected": -2.2267227172851562,
      "logps/chosen": -223.8726043701172,
      "logps/rejected": -243.6025390625,
      "loss": 2002.6312,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13919594883918762,
      "rewards/margins": 0.13850137591362,
      "rewards/rejected": -0.27769729495048523,
      "step": 14530
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.6007184290880456e-08,
      "logits/chosen": -2.3476033210754395,
      "logits/rejected": -2.250824451446533,
      "logps/chosen": -227.66970825195312,
      "logps/rejected": -225.08633422851562,
      "loss": 2114.7318,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.15696708858013153,
      "rewards/margins": 0.14080265164375305,
      "rewards/rejected": -0.2977697253227234,
      "step": 14540
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.504800699402872e-08,
      "logits/chosen": -2.5682365894317627,
      "logits/rejected": -2.2721850872039795,
      "logps/chosen": -356.9365539550781,
      "logps/rejected": -299.3744812011719,
      "loss": 1899.5432,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1128232479095459,
      "rewards/margins": 0.15804831683635712,
      "rewards/rejected": -0.2708715796470642,
      "step": 14550
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.4101688094242967e-08,
      "logits/chosen": -2.28175687789917,
      "logits/rejected": -2.1759145259857178,
      "logps/chosen": -283.11199951171875,
      "logps/rejected": -277.6907958984375,
      "loss": 1573.1848,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1355103850364685,
      "rewards/margins": 0.19754980504512787,
      "rewards/rejected": -0.33306020498275757,
      "step": 14560
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.3168232527985564e-08,
      "logits/chosen": -2.2301955223083496,
      "logits/rejected": -1.9302619695663452,
      "logps/chosen": -250.8253173828125,
      "logps/rejected": -215.4320068359375,
      "loss": 1733.818,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09206750243902206,
      "rewards/margins": 0.19356587529182434,
      "rewards/rejected": -0.285633385181427,
      "step": 14570
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.224764516461892e-08,
      "logits/chosen": -2.3313794136047363,
      "logits/rejected": -2.1601405143737793,
      "logps/chosen": -265.72125244140625,
      "logps/rejected": -253.18911743164062,
      "loss": 1621.6596,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10722391307353973,
      "rewards/margins": 0.18216519057750702,
      "rewards/rejected": -0.28938913345336914,
      "step": 14580
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.133993080637665e-08,
      "logits/chosen": -2.323228359222412,
      "logits/rejected": -2.1346726417541504,
      "logps/chosen": -214.2357177734375,
      "logps/rejected": -221.05899047851562,
      "loss": 1525.7791,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11354829370975494,
      "rewards/margins": 0.1923418641090393,
      "rewards/rejected": -0.30589017271995544,
      "step": 14590
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.0445094188342186e-08,
      "logits/chosen": -2.137878894805908,
      "logits/rejected": -1.8444684743881226,
      "logps/chosen": -264.8699035644531,
      "logps/rejected": -202.3433074951172,
      "loss": 1412.3269,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11393339931964874,
      "rewards/margins": 0.17737577855587006,
      "rewards/rejected": -0.2913091778755188,
      "step": 14600
    },
    {
      "epoch": 0.96,
      "eval_logits/chosen": -2.345289707183838,
      "eval_logits/rejected": -2.158090829849243,
      "eval_logps/chosen": -243.47486877441406,
      "eval_logps/rejected": -238.74017333984375,
      "eval_loss": 1756.218994140625,
      "eval_rewards/accuracies": 0.6654999852180481,
      "eval_rewards/chosen": -0.11469940841197968,
      "eval_rewards/margins": 0.15658311545848846,
      "eval_rewards/rejected": -0.27128249406814575,
      "eval_runtime": 720.8174,
      "eval_samples_per_second": 2.775,
      "eval_steps_per_second": 1.387,
      "step": 14600
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.9563139978421028e-08,
      "logits/chosen": -2.252467632293701,
      "logits/rejected": -2.276352882385254,
      "logps/chosen": -237.4102325439453,
      "logps/rejected": -239.51889038085938,
      "loss": 2279.3187,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11538355052471161,
      "rewards/margins": 0.09893187135457993,
      "rewards/rejected": -0.21431544423103333,
      "step": 14610
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.869407277731939e-08,
      "logits/chosen": -2.2228925228118896,
      "logits/rejected": -2.151127815246582,
      "logps/chosen": -194.53271484375,
      "logps/rejected": -187.36260986328125,
      "loss": 1568.2383,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09275592863559723,
      "rewards/margins": 0.15999475121498108,
      "rewards/rejected": -0.2527506649494171,
      "step": 14620
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.783789711851642e-08,
      "logits/chosen": -2.3487181663513184,
      "logits/rejected": -2.1291961669921875,
      "logps/chosen": -169.86410522460938,
      "logps/rejected": -167.00466918945312,
      "loss": 1597.5015,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08013113588094711,
      "rewards/margins": 0.1957322657108307,
      "rewards/rejected": -0.275863379240036,
      "step": 14630
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.6994617468244778e-08,
      "logits/chosen": -2.446563243865967,
      "logits/rejected": -1.968849539756775,
      "logps/chosen": -224.6818084716797,
      "logps/rejected": -178.614013671875,
      "loss": 1439.7357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11661139875650406,
      "rewards/margins": 0.1785092055797577,
      "rewards/rejected": -0.29512059688568115,
      "step": 14640
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.6164238225463155e-08,
      "logits/chosen": -2.2791390419006348,
      "logits/rejected": -1.9296939373016357,
      "logps/chosen": -291.14801025390625,
      "logps/rejected": -232.16015625,
      "loss": 1572.7487,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0872085690498352,
      "rewards/margins": 0.19520151615142822,
      "rewards/rejected": -0.2824100852012634,
      "step": 14650
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.534676372183742e-08,
      "logits/chosen": -2.3055472373962402,
      "logits/rejected": -2.1529393196105957,
      "logps/chosen": -293.67352294921875,
      "logps/rejected": -260.25726318359375,
      "loss": 1817.2176,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09719154983758926,
      "rewards/margins": 0.1891038864850998,
      "rewards/rejected": -0.28629544377326965,
      "step": 14660
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4542198221714218e-08,
      "logits/chosen": -2.205369710922241,
      "logits/rejected": -1.9309327602386475,
      "logps/chosen": -148.51953125,
      "logps/rejected": -163.5492401123047,
      "loss": 1798.3756,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1041257381439209,
      "rewards/margins": 0.14802828431129456,
      "rewards/rejected": -0.25215405225753784,
      "step": 14670
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3750545922101854e-08,
      "logits/chosen": -2.601306915283203,
      "logits/rejected": -2.2248146533966064,
      "logps/chosen": -320.0009460449219,
      "logps/rejected": -269.42388916015625,
      "loss": 1441.0441,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.10609765350818634,
      "rewards/margins": 0.1781700849533081,
      "rewards/rejected": -0.28426772356033325,
      "step": 14680
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.2971810952646112e-08,
      "logits/chosen": -2.3170502185821533,
      "logits/rejected": -2.2311322689056396,
      "logps/chosen": -270.6275939941406,
      "logps/rejected": -233.7604217529297,
      "loss": 2079.441,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12264804542064667,
      "rewards/margins": 0.10838682949542999,
      "rewards/rejected": -0.23103487491607666,
      "step": 14690
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.2205997375610576e-08,
      "logits/chosen": -2.1689231395721436,
      "logits/rejected": -2.1099231243133545,
      "logps/chosen": -190.51622009277344,
      "logps/rejected": -215.9954071044922,
      "loss": 1504.9481,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.050102997571229935,
      "rewards/margins": 0.20319123566150665,
      "rewards/rejected": -0.2532942295074463,
      "step": 14700
    },
    {
      "epoch": 0.96,
      "eval_logits/chosen": -2.3439536094665527,
      "eval_logits/rejected": -2.156829833984375,
      "eval_logps/chosen": -243.49813842773438,
      "eval_logps/rejected": -238.74496459960938,
      "eval_loss": 1756.396728515625,
      "eval_rewards/accuracies": 0.6644999980926514,
      "eval_rewards/chosen": -0.1149320900440216,
      "eval_rewards/margins": 0.15639850497245789,
      "eval_rewards/rejected": -0.2713305652141571,
      "eval_runtime": 723.7336,
      "eval_samples_per_second": 2.763,
      "eval_steps_per_second": 1.382,
      "step": 14700
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.1453109185853304e-08,
      "logits/chosen": -2.3741512298583984,
      "logits/rejected": -2.3167805671691895,
      "logps/chosen": -210.87539672851562,
      "logps/rejected": -226.72695922851562,
      "loss": 1983.609,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08796913921833038,
      "rewards/margins": 0.14104311168193817,
      "rewards/rejected": -0.22901228070259094,
      "step": 14710
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.0713150310808784e-08,
      "logits/chosen": -2.1360185146331787,
      "logits/rejected": -2.4715590476989746,
      "logps/chosen": -238.7700958251953,
      "logps/rejected": -251.58615112304688,
      "loss": 2161.6273,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13282257318496704,
      "rewards/margins": 0.07052706182003021,
      "rewards/rejected": -0.20334962010383606,
      "step": 14720
    },
    {
      "epoch": 0.96,
      "learning_rate": 1.9986124610464064e-08,
      "logits/chosen": -2.2351553440093994,
      "logits/rejected": -1.9309247732162476,
      "logps/chosen": -301.1767578125,
      "logps/rejected": -252.90139770507812,
      "loss": 1462.9733,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.1394665241241455,
      "rewards/margins": 0.2153692990541458,
      "rewards/rejected": -0.3548358082771301,
      "step": 14730
    },
    {
      "epoch": 0.96,
      "learning_rate": 1.927203587734211e-08,
      "logits/chosen": -2.2187814712524414,
      "logits/rejected": -1.8177343606948853,
      "logps/chosen": -258.6115417480469,
      "logps/rejected": -231.11123657226562,
      "loss": 1640.377,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05619867891073227,
      "rewards/margins": 0.2102610170841217,
      "rewards/rejected": -0.26645970344543457,
      "step": 14740
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.8570887836479034e-08,
      "logits/chosen": -2.334831714630127,
      "logits/rejected": -2.1394412517547607,
      "logps/chosen": -209.2335968017578,
      "logps/rejected": -275.2274475097656,
      "loss": 1893.7287,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1561044305562973,
      "rewards/margins": 0.1317848265171051,
      "rewards/rejected": -0.2878892421722412,
      "step": 14750
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.7882684145406616e-08,
      "logits/chosen": -2.3986196517944336,
      "logits/rejected": -2.3380439281463623,
      "logps/chosen": -304.5469055175781,
      "logps/rejected": -319.450927734375,
      "loss": 1727.3447,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08126304298639297,
      "rewards/margins": 0.16792842745780945,
      "rewards/rejected": -0.24919147789478302,
      "step": 14760
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.7207428394132865e-08,
      "logits/chosen": -2.553154945373535,
      "logits/rejected": -2.0709140300750732,
      "logps/chosen": -279.75396728515625,
      "logps/rejected": -248.90267944335938,
      "loss": 1438.3188,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.09973666071891785,
      "rewards/margins": 0.2383594959974289,
      "rewards/rejected": -0.3380962014198303,
      "step": 14770
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.654512410512177e-08,
      "logits/chosen": -2.277434825897217,
      "logits/rejected": -2.038137912750244,
      "logps/chosen": -265.58087158203125,
      "logps/rejected": -216.32894897460938,
      "loss": 1964.3506,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12003912776708603,
      "rewards/margins": 0.13136547803878784,
      "rewards/rejected": -0.25140461325645447,
      "step": 14780
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.5895774733277468e-08,
      "logits/chosen": -2.3411812782287598,
      "logits/rejected": -1.9985847473144531,
      "logps/chosen": -289.6302490234375,
      "logps/rejected": -257.2305603027344,
      "loss": 1619.169,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09813287854194641,
      "rewards/margins": 0.17836883664131165,
      "rewards/rejected": -0.27650171518325806,
      "step": 14790
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.5259383665924e-08,
      "logits/chosen": -2.5866286754608154,
      "logits/rejected": -2.2018492221832275,
      "logps/chosen": -348.4439392089844,
      "logps/rejected": -275.0132751464844,
      "loss": 1509.7718,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06352052092552185,
      "rewards/margins": 0.20578232407569885,
      "rewards/rejected": -0.2693028748035431,
      "step": 14800
    },
    {
      "epoch": 0.97,
      "eval_logits/chosen": -2.344806432723999,
      "eval_logits/rejected": -2.1576449871063232,
      "eval_logps/chosen": -243.4784393310547,
      "eval_logps/rejected": -238.7247772216797,
      "eval_loss": 1755.3797607421875,
      "eval_rewards/accuracies": 0.6625000238418579,
      "eval_rewards/chosen": -0.11473479121923447,
      "eval_rewards/margins": 0.15639401972293854,
      "eval_rewards/rejected": -0.2711288034915924,
      "eval_runtime": 719.5509,
      "eval_samples_per_second": 2.78,
      "eval_steps_per_second": 1.39,
      "step": 14800
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.4635954222789461e-08,
      "logits/chosen": -2.2872562408447266,
      "logits/rejected": -2.202056884765625,
      "logps/chosen": -228.4236297607422,
      "logps/rejected": -248.9585723876953,
      "loss": 1902.5502,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1131603866815567,
      "rewards/margins": 0.15427032113075256,
      "rewards/rejected": -0.26743072271347046,
      "step": 14810
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.402548965598688e-08,
      "logits/chosen": -2.222163438796997,
      "logits/rejected": -2.327205181121826,
      "logps/chosen": -216.5567626953125,
      "logps/rejected": -231.3907012939453,
      "loss": 1527.4884,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11929510533809662,
      "rewards/margins": 0.1805037558078766,
      "rewards/rejected": -0.2997989058494568,
      "step": 14820
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.3427993149998375e-08,
      "logits/chosen": -2.4768872261047363,
      "logits/rejected": -2.2277302742004395,
      "logps/chosen": -252.4502716064453,
      "logps/rejected": -213.08779907226562,
      "loss": 1706.2182,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10413049161434174,
      "rewards/margins": 0.16007684171199799,
      "rewards/rejected": -0.2642073631286621,
      "step": 14830
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.2843467821658518e-08,
      "logits/chosen": -2.433828830718994,
      "logits/rejected": -2.370725154876709,
      "logps/chosen": -230.56201171875,
      "logps/rejected": -249.99472045898438,
      "loss": 1390.466,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04339112713932991,
      "rewards/margins": 0.19801194965839386,
      "rewards/rejected": -0.24140305817127228,
      "step": 14840
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.2271916720137666e-08,
      "logits/chosen": -2.5363497734069824,
      "logits/rejected": -2.2229340076446533,
      "logps/chosen": -299.01824951171875,
      "logps/rejected": -264.07366943359375,
      "loss": 2100.2973,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13248726725578308,
      "rewards/margins": 0.11190402507781982,
      "rewards/rejected": -0.2443912923336029,
      "step": 14850
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.171334282692671e-08,
      "logits/chosen": -2.3759212493896484,
      "logits/rejected": -2.312915563583374,
      "logps/chosen": -290.940673828125,
      "logps/rejected": -282.8968505859375,
      "loss": 1562.8558,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08342249691486359,
      "rewards/margins": 0.22588083148002625,
      "rewards/rejected": -0.30930331349372864,
      "step": 14860
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.116774905582041e-08,
      "logits/chosen": -2.4384074211120605,
      "logits/rejected": -2.107351779937744,
      "logps/chosen": -196.92654418945312,
      "logps/rejected": -195.10423278808594,
      "loss": 1941.9195,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12108032405376434,
      "rewards/margins": 0.10880926996469498,
      "rewards/rejected": -0.22988960146903992,
      "step": 14870
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.0635138252902966e-08,
      "logits/chosen": -2.518921375274658,
      "logits/rejected": -2.26084303855896,
      "logps/chosen": -241.8000946044922,
      "logps/rejected": -236.13381958007812,
      "loss": 1661.2465,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11386130750179291,
      "rewards/margins": 0.16462181508541107,
      "rewards/rejected": -0.278483122587204,
      "step": 14880
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.0115513196533589e-08,
      "logits/chosen": -2.3540704250335693,
      "logits/rejected": -2.1872241497039795,
      "logps/chosen": -273.9299011230469,
      "logps/rejected": -268.36236572265625,
      "loss": 2094.5197,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11319758743047714,
      "rewards/margins": 0.11740664392709732,
      "rewards/rejected": -0.23060421645641327,
      "step": 14890
    },
    {
      "epoch": 0.97,
      "learning_rate": 9.608876597330952e-09,
      "logits/chosen": -2.3617584705352783,
      "logits/rejected": -2.0387043952941895,
      "logps/chosen": -306.14923095703125,
      "logps/rejected": -309.53643798828125,
      "loss": 1881.0627,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.19526515901088715,
      "rewards/margins": 0.1507243812084198,
      "rewards/rejected": -0.34598949551582336,
      "step": 14900
    },
    {
      "epoch": 0.97,
      "eval_logits/chosen": -2.3451874256134033,
      "eval_logits/rejected": -2.158019781112671,
      "eval_logps/chosen": -243.46603393554688,
      "eval_logps/rejected": -238.71018981933594,
      "eval_loss": 1755.147216796875,
      "eval_rewards/accuracies": 0.6650000214576721,
      "eval_rewards/chosen": -0.1146109476685524,
      "eval_rewards/margins": 0.1563720554113388,
      "eval_rewards/rejected": -0.2709830403327942,
      "eval_runtime": 721.9629,
      "eval_samples_per_second": 2.77,
      "eval_steps_per_second": 1.385,
      "step": 14900
    },
    {
      "epoch": 0.98,
      "learning_rate": 9.115231098159594e-09,
      "logits/chosen": -2.4154810905456543,
      "logits/rejected": -2.3058924674987793,
      "logps/chosen": -263.7437438964844,
      "logps/rejected": -255.2856903076172,
      "loss": 2138.4381,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10429998487234116,
      "rewards/margins": 0.11843911558389664,
      "rewards/rejected": -0.22273913025856018,
      "step": 14910
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.634579274116317e-09,
      "logits/chosen": -2.2822928428649902,
      "logits/rejected": -2.213261127471924,
      "logps/chosen": -199.53839111328125,
      "logps/rejected": -233.07620239257812,
      "loss": 2012.0393,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0947912260890007,
      "rewards/margins": 0.12928275763988495,
      "rewards/rejected": -0.22407397627830505,
      "step": 14920
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.166923632516865e-09,
      "logits/chosen": -2.4574990272521973,
      "logits/rejected": -2.1689932346343994,
      "logps/chosen": -235.9760284423828,
      "logps/rejected": -298.189208984375,
      "loss": 1338.2018,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08244818449020386,
      "rewards/margins": 0.24586090445518494,
      "rewards/rejected": -0.3283091187477112,
      "step": 14930
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.712266612881492e-09,
      "logits/chosen": -2.2132229804992676,
      "logits/rejected": -2.084080219268799,
      "logps/chosen": -192.06051635742188,
      "logps/rejected": -206.5467071533203,
      "loss": 1505.5216,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06823839992284775,
      "rewards/margins": 0.1936199814081192,
      "rewards/rejected": -0.26185840368270874,
      "step": 14940
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.270610586924687e-09,
      "logits/chosen": -2.463947057723999,
      "logits/rejected": -2.2258262634277344,
      "logps/chosen": -273.31195068359375,
      "logps/rejected": -246.1824493408203,
      "loss": 1626.5279,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08346611261367798,
      "rewards/margins": 0.16950808465480804,
      "rewards/rejected": -0.2529742121696472,
      "step": 14950
    },
    {
      "epoch": 0.98,
      "learning_rate": 6.841957858539916e-09,
      "logits/chosen": -2.284597158432007,
      "logits/rejected": -2.139622211456299,
      "logps/chosen": -185.13320922851562,
      "logps/rejected": -206.1970672607422,
      "loss": 2098.2066,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.18477652966976166,
      "rewards/margins": 0.10213065147399902,
      "rewards/rejected": -0.2869071960449219,
      "step": 14960
    },
    {
      "epoch": 0.98,
      "learning_rate": 6.426310663790181e-09,
      "logits/chosen": -2.215423822402954,
      "logits/rejected": -2.1268699169158936,
      "logps/chosen": -252.93124389648438,
      "logps/rejected": -235.5459442138672,
      "loss": 2000.2178,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1595548391342163,
      "rewards/margins": 0.14083118736743927,
      "rewards/rejected": -0.3003860414028168,
      "step": 14970
    },
    {
      "epoch": 0.98,
      "learning_rate": 6.023671170894696e-09,
      "logits/chosen": -2.530273675918579,
      "logits/rejected": -1.9427354335784912,
      "logps/chosen": -300.4677429199219,
      "logps/rejected": -238.73995971679688,
      "loss": 1062.9697,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08139168471097946,
      "rewards/margins": 0.25051823258399963,
      "rewards/rejected": -0.3319099247455597,
      "step": 14980
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.634041480218344e-09,
      "logits/chosen": -2.4931185245513916,
      "logits/rejected": -2.1452651023864746,
      "logps/chosen": -262.18927001953125,
      "logps/rejected": -275.85614013671875,
      "loss": 1671.9363,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.09115663915872574,
      "rewards/margins": 0.18107278645038605,
      "rewards/rejected": -0.27222940325737,
      "step": 14990
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.257423624260849e-09,
      "logits/chosen": -2.5348258018493652,
      "logits/rejected": -2.100780487060547,
      "logps/chosen": -273.679443359375,
      "logps/rejected": -246.7283477783203,
      "loss": 1820.8113,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10356035083532333,
      "rewards/margins": 0.16292151808738708,
      "rewards/rejected": -0.266481876373291,
      "step": 15000
    },
    {
      "epoch": 0.98,
      "eval_logits/chosen": -2.3453927040100098,
      "eval_logits/rejected": -2.1581103801727295,
      "eval_logps/chosen": -243.4661102294922,
      "eval_logps/rejected": -238.70730590820312,
      "eval_loss": 1754.8675537109375,
      "eval_rewards/accuracies": 0.6660000085830688,
      "eval_rewards/chosen": -0.11461148411035538,
      "eval_rewards/margins": 0.1563425213098526,
      "eval_rewards/rejected": -0.2709540128707886,
      "eval_runtime": 720.1314,
      "eval_samples_per_second": 2.777,
      "eval_steps_per_second": 1.389,
      "step": 15000
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.893819567644564e-09,
      "logits/chosen": -2.2239840030670166,
      "logits/rejected": -2.200934410095215,
      "logps/chosen": -198.62867736816406,
      "logps/rejected": -220.85385131835938,
      "loss": 2058.44,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10631934553384781,
      "rewards/margins": 0.1204916462302208,
      "rewards/rejected": -0.2268110066652298,
      "step": 15010
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.543231207107257e-09,
      "logits/chosen": -2.2935476303100586,
      "logits/rejected": -2.0775296688079834,
      "logps/chosen": -268.8769226074219,
      "logps/rejected": -260.10272216796875,
      "loss": 1846.2023,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11055328696966171,
      "rewards/margins": 0.14834541082382202,
      "rewards/rejected": -0.25889870524406433,
      "step": 15020
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.205660371488785e-09,
      "logits/chosen": -2.585259437561035,
      "logits/rejected": -2.224368095397949,
      "logps/chosen": -295.079345703125,
      "logps/rejected": -267.5276794433594,
      "loss": 2099.5447,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10925974696874619,
      "rewards/margins": 0.14238341152668,
      "rewards/rejected": -0.2516431510448456,
      "step": 15030
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.88110882172471e-09,
      "logits/chosen": -2.2619118690490723,
      "logits/rejected": -2.2005677223205566,
      "logps/chosen": -234.2068634033203,
      "logps/rejected": -243.8390350341797,
      "loss": 1841.5732,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1350843608379364,
      "rewards/margins": 0.1426878273487091,
      "rewards/rejected": -0.2777721583843231,
      "step": 15040
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.569578250834371e-09,
      "logits/chosen": -2.363293170928955,
      "logits/rejected": -2.089813709259033,
      "logps/chosen": -308.189208984375,
      "logps/rejected": -291.08929443359375,
      "loss": 1346.5727,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06211404129862785,
      "rewards/margins": 0.20662875473499298,
      "rewards/rejected": -0.26874274015426636,
      "step": 15050
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.2710702839139353e-09,
      "logits/chosen": -2.390526533126831,
      "logits/rejected": -2.2443912029266357,
      "logps/chosen": -214.2931671142578,
      "logps/rejected": -234.26174926757812,
      "loss": 1999.3119,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0969388335943222,
      "rewards/margins": 0.10695371776819229,
      "rewards/rejected": -0.2038925439119339,
      "step": 15060
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.9855864781272448e-09,
      "logits/chosen": -2.3530771732330322,
      "logits/rejected": -2.389768600463867,
      "logps/chosen": -219.4556884765625,
      "logps/rejected": -269.90338134765625,
      "loss": 1778.8965,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07782004773616791,
      "rewards/margins": 0.15653666853904724,
      "rewards/rejected": -0.23435673117637634,
      "step": 15070
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.7131283226977665e-09,
      "logits/chosen": -2.359271287918091,
      "logits/rejected": -2.429197311401367,
      "logps/chosen": -229.75521850585938,
      "logps/rejected": -259.5672607421875,
      "loss": 1777.0398,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10306936502456665,
      "rewards/margins": 0.15961900353431702,
      "rewards/rejected": -0.26268836855888367,
      "step": 15080
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.4536972389008205e-09,
      "logits/chosen": -2.309981346130371,
      "logits/rejected": -2.077146053314209,
      "logps/chosen": -246.08938598632812,
      "logps/rejected": -227.2568359375,
      "loss": 1641.2615,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08954377472400665,
      "rewards/margins": 0.18858453631401062,
      "rewards/rejected": -0.27812832593917847,
      "step": 15090
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.20729458005553e-09,
      "logits/chosen": -2.221825122833252,
      "logits/rejected": -2.062643527984619,
      "logps/chosen": -205.6139678955078,
      "logps/rejected": -202.2505340576172,
      "loss": 1512.1538,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11204735189676285,
      "rewards/margins": 0.18838240206241608,
      "rewards/rejected": -0.3004297614097595,
      "step": 15100
    },
    {
      "epoch": 0.99,
      "eval_logits/chosen": -2.3453314304351807,
      "eval_logits/rejected": -2.1580898761749268,
      "eval_logps/chosen": -243.4615020751953,
      "eval_logps/rejected": -238.70053100585938,
      "eval_loss": 1754.2144775390625,
      "eval_rewards/accuracies": 0.6635000109672546,
      "eval_rewards/chosen": -0.11456552892923355,
      "eval_rewards/margins": 0.15632058680057526,
      "eval_rewards/rejected": -0.2708861231803894,
      "eval_runtime": 720.0327,
      "eval_samples_per_second": 2.778,
      "eval_steps_per_second": 1.389,
      "step": 15100
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.9739216315192712e-09,
      "logits/chosen": -2.3239998817443848,
      "logits/rejected": -2.11338472366333,
      "logps/chosen": -240.9944610595703,
      "logps/rejected": -230.3395538330078,
      "loss": 1798.9738,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11685649305582047,
      "rewards/margins": 0.13042503595352173,
      "rewards/rejected": -0.2472815066576004,
      "step": 15110
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.7535796106796231e-09,
      "logits/chosen": -2.4070184230804443,
      "logits/rejected": -2.0923027992248535,
      "logps/chosen": -286.97772216796875,
      "logps/rejected": -220.5601348876953,
      "loss": 1745.35,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11441268771886826,
      "rewards/margins": 0.13913391530513763,
      "rewards/rejected": -0.25354665517807007,
      "step": 15120
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.5462696669482636e-09,
      "logits/chosen": -2.376131534576416,
      "logits/rejected": -2.258821487426758,
      "logps/chosen": -232.0293426513672,
      "logps/rejected": -252.01962280273438,
      "loss": 1955.3076,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10787205398082733,
      "rewards/margins": 0.12346227467060089,
      "rewards/rejected": -0.23133432865142822,
      "step": 15130
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.3519928817556927e-09,
      "logits/chosen": -2.232820987701416,
      "logits/rejected": -2.192532777786255,
      "logps/chosen": -183.15576171875,
      "logps/rejected": -196.3177032470703,
      "loss": 1906.4008,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09762244671583176,
      "rewards/margins": 0.12615005671977997,
      "rewards/rejected": -0.22377249598503113,
      "step": 15140
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.1707502685448512e-09,
      "logits/chosen": -2.4631078243255615,
      "logits/rejected": -2.126909017562866,
      "logps/chosen": -240.1180419921875,
      "logps/rejected": -200.20216369628906,
      "loss": 1534.8595,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1784803569316864,
      "rewards/margins": 0.18272601068019867,
      "rewards/rejected": -0.3612063527107239,
      "step": 15150
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.002542772765569e-09,
      "logits/chosen": -2.2957310676574707,
      "logits/rejected": -2.0127415657043457,
      "logps/chosen": -198.36758422851562,
      "logps/rejected": -177.6871337890625,
      "loss": 1611.424,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0976375862956047,
      "rewards/margins": 0.1762719601392746,
      "rewards/rejected": -0.2739095389842987,
      "step": 15160
    },
    {
      "epoch": 0.99,
      "learning_rate": 8.473712718709559e-10,
      "logits/chosen": -2.1500306129455566,
      "logits/rejected": -2.168015956878662,
      "logps/chosen": -209.2052764892578,
      "logps/rejected": -206.0652618408203,
      "loss": 2103.8172,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1546037644147873,
      "rewards/margins": 0.07638068497180939,
      "rewards/rejected": -0.23098444938659668,
      "step": 15170
    },
    {
      "epoch": 0.99,
      "learning_rate": 7.052365753112966e-10,
      "logits/chosen": -2.1892058849334717,
      "logits/rejected": -1.8921606540679932,
      "logps/chosen": -250.9332275390625,
      "logps/rejected": -247.93362426757812,
      "loss": 1891.3033,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15617124736309052,
      "rewards/margins": 0.15776070952415466,
      "rewards/rejected": -0.31393197178840637,
      "step": 15180
    },
    {
      "epoch": 0.99,
      "learning_rate": 5.761394245307195e-10,
      "logits/chosen": -2.1700949668884277,
      "logits/rejected": -2.1966946125030518,
      "logps/chosen": -247.8317108154297,
      "logps/rejected": -257.30291748046875,
      "loss": 1964.2982,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10749546438455582,
      "rewards/margins": 0.10682658851146698,
      "rewards/rejected": -0.2143220603466034,
      "step": 15190
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.6008049296358826e-10,
      "logits/chosen": -2.253401041030884,
      "logits/rejected": -2.1524932384490967,
      "logps/chosen": -196.65591430664062,
      "logps/rejected": -184.73748779296875,
      "loss": 1698.8312,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10633338987827301,
      "rewards/margins": 0.16325244307518005,
      "rewards/rejected": -0.26958581805229187,
      "step": 15200
    },
    {
      "epoch": 0.99,
      "eval_logits/chosen": -2.344879388809204,
      "eval_logits/rejected": -2.1577203273773193,
      "eval_logps/chosen": -243.4614715576172,
      "eval_logps/rejected": -238.69151306152344,
      "eval_loss": 1755.1842041015625,
      "eval_rewards/accuracies": 0.6629999876022339,
      "eval_rewards/chosen": -0.11456498503684998,
      "eval_rewards/margins": 0.15623119473457336,
      "eval_rewards/rejected": -0.27079617977142334,
      "eval_runtime": 718.3605,
      "eval_samples_per_second": 2.784,
      "eval_steps_per_second": 1.392,
      "step": 15200
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.5706038603006146e-10,
      "logits/chosen": -2.430497646331787,
      "logits/rejected": -2.3825058937072754,
      "logps/chosen": -290.90673828125,
      "logps/rejected": -296.6795654296875,
      "loss": 1553.6001,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.012963606044650078,
      "rewards/margins": 0.20015950500965118,
      "rewards/rejected": -0.21312308311462402,
      "step": 15210
    },
    {
      "epoch": 1.0,
      "learning_rate": 2.670796411333165e-10,
      "logits/chosen": -2.608908176422119,
      "logits/rejected": -2.3190643787384033,
      "logps/chosen": -232.03125,
      "logps/rejected": -234.5654754638672,
      "loss": 1794.9951,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12219800800085068,
      "rewards/margins": 0.14840340614318848,
      "rewards/rejected": -0.27060142159461975,
      "step": 15220
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.9013872765677455e-10,
      "logits/chosen": -2.3324027061462402,
      "logits/rejected": -2.127634048461914,
      "logps/chosen": -227.2728271484375,
      "logps/rejected": -227.78005981445312,
      "loss": 1788.8002,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07759709656238556,
      "rewards/margins": 0.13041789829730988,
      "rewards/rejected": -0.20801499485969543,
      "step": 15230
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.262380469624347e-10,
      "logits/chosen": -2.310046672821045,
      "logits/rejected": -2.1428942680358887,
      "logps/chosen": -214.257080078125,
      "logps/rejected": -206.60110473632812,
      "loss": 1765.4699,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10250154882669449,
      "rewards/margins": 0.15253737568855286,
      "rewards/rejected": -0.25503891706466675,
      "step": 15240
    },
    {
      "epoch": 1.0,
      "learning_rate": 7.53779323872661e-11,
      "logits/chosen": -2.2056996822357178,
      "logits/rejected": -2.3175556659698486,
      "logps/chosen": -202.9796142578125,
      "logps/rejected": -226.6588134765625,
      "loss": 2022.0285,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1111622080206871,
      "rewards/margins": 0.17635276913642883,
      "rewards/rejected": -0.28751498460769653,
      "step": 15250
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.7558649242652734e-11,
      "logits/chosen": -2.5009121894836426,
      "logits/rejected": -2.27042818069458,
      "logps/chosen": -406.96636962890625,
      "logps/rejected": -343.4335021972656,
      "loss": 1724.1916,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10282541811466217,
      "rewards/margins": 0.16037984192371368,
      "rewards/rejected": -0.26320526003837585,
      "step": 15260
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.2780394812450526e-11,
      "logits/chosen": -2.1512584686279297,
      "logits/rejected": -2.1113929748535156,
      "logps/chosen": -247.2060546875,
      "logps/rejected": -264.48095703125,
      "loss": 1595.7446,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15062271058559418,
      "rewards/margins": 0.17357513308525085,
      "rewards/rejected": -0.3241978883743286,
      "step": 15270
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.0432983521546646e-12,
      "logits/chosen": -2.191702365875244,
      "logits/rejected": -2.101569890975952,
      "logps/chosen": -194.95314025878906,
      "logps/rejected": -227.9114227294922,
      "loss": 1693.5664,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1148693710565567,
      "rewards/margins": 0.1411261111497879,
      "rewards/rejected": -0.2559954822063446,
      "step": 15280
    },
    {
      "epoch": 1.0,
      "step": 15284,
      "total_flos": 0.0,
      "train_loss": 1830.215161976219,
      "train_runtime": 182270.758,
      "train_samples_per_second": 0.335,
      "train_steps_per_second": 0.084
    }
  ],
  "logging_steps": 10,
  "max_steps": 15284,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}