{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.9968602825745683,
  "eval_steps": 100,
  "global_step": 954,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": 0.952304482460022,
      "logits/rejected": 0.5888463854789734,
      "logps/chosen": -223.79486083984375,
      "logps/rejected": -209.482666015625,
      "loss": 2500.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 5.208333333333334e-07,
      "logits/chosen": 0.8362942934036255,
      "logits/rejected": 0.8542055487632751,
      "logps/chosen": -236.253662109375,
      "logps/rejected": -221.88853454589844,
      "loss": 2503.2357,
      "rewards/accuracies": 0.3958333432674408,
      "rewards/chosen": 8.874866762198508e-05,
      "rewards/margins": -0.00026307348161935806,
      "rewards/rejected": 0.0003518221783451736,
      "step": 10
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.0416666666666667e-06,
      "logits/chosen": 0.8335070610046387,
      "logits/rejected": 0.9283286929130554,
      "logps/chosen": -254.7803192138672,
      "logps/rejected": -247.91357421875,
      "loss": 2498.9305,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0006878537242300808,
      "rewards/margins": 0.00017734414723236114,
      "rewards/rejected": -0.0008651980315335095,
      "step": 20
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.5625e-06,
      "logits/chosen": 0.8601231575012207,
      "logits/rejected": 0.9173057675361633,
      "logps/chosen": -260.49664306640625,
      "logps/rejected": -232.03378295898438,
      "loss": 2500.6068,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": 0.0006941998144611716,
      "rewards/margins": 0.0007489144918508828,
      "rewards/rejected": -5.471452095662244e-05,
      "step": 30
    },
    {
      "epoch": 0.08,
      "learning_rate": 2.0833333333333334e-06,
      "logits/chosen": 0.811628520488739,
      "logits/rejected": 0.9033697843551636,
      "logps/chosen": -280.28118896484375,
      "logps/rejected": -228.5680694580078,
      "loss": 2502.0561,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.000692486937623471,
      "rewards/margins": -0.0004873524303548038,
      "rewards/rejected": -0.00020513453637249768,
      "step": 40
    },
    {
      "epoch": 0.1,
      "learning_rate": 2.604166666666667e-06,
      "logits/chosen": 0.8561393022537231,
      "logits/rejected": 0.9358364343643188,
      "logps/chosen": -257.61163330078125,
      "logps/rejected": -219.1778106689453,
      "loss": 2492.4258,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0008641455206088722,
      "rewards/margins": 0.0018391588237136602,
      "rewards/rejected": -0.000975013361312449,
      "step": 50
    },
    {
      "epoch": 0.13,
      "learning_rate": 3.125e-06,
      "logits/chosen": 0.8744305372238159,
      "logits/rejected": 0.9009464383125305,
      "logps/chosen": -237.00228881835938,
      "logps/rejected": -237.4504852294922,
      "loss": 2497.3361,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -9.825383131101262e-06,
      "rewards/margins": 0.00019281035929452628,
      "rewards/rejected": -0.0002026357251452282,
      "step": 60
    },
    {
      "epoch": 0.15,
      "learning_rate": 3.6458333333333333e-06,
      "logits/chosen": 0.8502357602119446,
      "logits/rejected": 0.8783925771713257,
      "logps/chosen": -260.8014221191406,
      "logps/rejected": -227.61328125,
      "loss": 2500.9129,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -0.0007933862507343292,
      "rewards/margins": -0.00043974880827590823,
      "rewards/rejected": -0.000353637442458421,
      "step": 70
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.166666666666667e-06,
      "logits/chosen": 0.8865741491317749,
      "logits/rejected": 0.9179280996322632,
      "logps/chosen": -251.12197875976562,
      "logps/rejected": -231.300048828125,
      "loss": 2494.3678,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.00037043695920147,
      "rewards/margins": 0.0005658747395500541,
      "rewards/rejected": -0.0009363117860630155,
      "step": 80
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.6875000000000004e-06,
      "logits/chosen": 0.8638327717781067,
      "logits/rejected": 0.9173502922058105,
      "logps/chosen": -225.3396453857422,
      "logps/rejected": -241.39352416992188,
      "loss": 2497.2648,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.0007204846478998661,
      "rewards/margins": 0.0005254354909993708,
      "rewards/rejected": -0.001245920080691576,
      "step": 90
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.999731868769027e-06,
      "logits/chosen": 0.9247162938117981,
      "logits/rejected": 0.9241034388542175,
      "logps/chosen": -242.1609344482422,
      "logps/rejected": -221.4512939453125,
      "loss": 2494.2439,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.00025123285013251007,
      "rewards/margins": 0.0006884234608151019,
      "rewards/rejected": -0.0009396563400514424,
      "step": 100
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": 0.8141916990280151,
      "eval_logits/rejected": 0.9164313673973083,
      "eval_logps/chosen": -260.2576599121094,
      "eval_logps/rejected": -231.54052734375,
      "eval_loss": 2494.119384765625,
      "eval_rewards/accuracies": 0.5479999780654907,
      "eval_rewards/chosen": -0.00013494741870090365,
      "eval_rewards/margins": 0.0009091641986742616,
      "eval_rewards/rejected": -0.0010441114427521825,
      "eval_runtime": 325.9254,
      "eval_samples_per_second": 6.136,
      "eval_steps_per_second": 0.384,
      "step": 100
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.996716052911017e-06,
      "logits/chosen": 0.8349224328994751,
      "logits/rejected": 0.8758266568183899,
      "logps/chosen": -264.0060119628906,
      "logps/rejected": -219.6302490234375,
      "loss": 2491.0754,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.00016143513494171202,
      "rewards/margins": 0.0016861247131600976,
      "rewards/rejected": -0.001524689607322216,
      "step": 110
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.9903533134293035e-06,
      "logits/chosen": 0.8607719540596008,
      "logits/rejected": 0.9709636569023132,
      "logps/chosen": -254.9365234375,
      "logps/rejected": -219.2154541015625,
      "loss": 2483.0162,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -4.470603380468674e-05,
      "rewards/margins": 0.0026740250177681446,
      "rewards/rejected": -0.002718730829656124,
      "step": 120
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.9806521797692184e-06,
      "logits/chosen": 0.8791080713272095,
      "logits/rejected": 0.8794806599617004,
      "logps/chosen": -264.69219970703125,
      "logps/rejected": -247.05224609375,
      "loss": 2478.3258,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.000567199953366071,
      "rewards/margins": 0.0015755310887470841,
      "rewards/rejected": -0.002142731100320816,
      "step": 130
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.967625656594782e-06,
      "logits/chosen": 0.8544095754623413,
      "logits/rejected": 0.9317782521247864,
      "logps/chosen": -222.06851196289062,
      "logps/rejected": -232.3370819091797,
      "loss": 2474.4896,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0013934863964095712,
      "rewards/margins": 0.0016968228155747056,
      "rewards/rejected": -0.0030903094448149204,
      "step": 140
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.95129120635556e-06,
      "logits/chosen": 0.8754276037216187,
      "logits/rejected": 0.905910849571228,
      "logps/chosen": -258.3515625,
      "logps/rejected": -215.86328125,
      "loss": 2470.2957,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.0012219983618706465,
      "rewards/margins": 0.0023179189302027225,
      "rewards/rejected": -0.003539917292073369,
      "step": 150
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.93167072587771e-06,
      "logits/chosen": 0.7798808813095093,
      "logits/rejected": 0.8481999635696411,
      "logps/chosen": -257.81036376953125,
      "logps/rejected": -250.8020477294922,
      "loss": 2466.5924,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0015566629590466619,
      "rewards/margins": 0.002945856424048543,
      "rewards/rejected": -0.004502518568187952,
      "step": 160
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.908790517010637e-06,
      "logits/chosen": 0.9313274621963501,
      "logits/rejected": 0.9725171327590942,
      "logps/chosen": -239.09017944335938,
      "logps/rejected": -253.09255981445312,
      "loss": 2458.4684,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.0023754839785397053,
      "rewards/margins": 0.003910133149474859,
      "rewards/rejected": -0.006285616662353277,
      "step": 170
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.882681251368549e-06,
      "logits/chosen": 0.8546341061592102,
      "logits/rejected": 0.8491582870483398,
      "logps/chosen": -270.9264831542969,
      "logps/rejected": -257.13836669921875,
      "loss": 2442.4984,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0015862795989960432,
      "rewards/margins": 0.005088582634925842,
      "rewards/rejected": -0.006674862466752529,
      "step": 180
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.853377929214243e-06,
      "logits/chosen": 0.7889066934585571,
      "logits/rejected": 0.8611849546432495,
      "logps/chosen": -252.2519989013672,
      "logps/rejected": -245.2388458251953,
      "loss": 2436.8207,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0018671129364520311,
      "rewards/margins": 0.0070212846621870995,
      "rewards/rejected": -0.008888397365808487,
      "step": 190
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.8209198325401815e-06,
      "logits/chosen": 0.8786072731018066,
      "logits/rejected": 0.8954092264175415,
      "logps/chosen": -237.77017211914062,
      "logps/rejected": -233.86325073242188,
      "loss": 2425.7957,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0038148313760757446,
      "rewards/margins": 0.00766246672719717,
      "rewards/rejected": -0.01147729717195034,
      "step": 200
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": 0.8182709813117981,
      "eval_logits/rejected": 0.9217536449432373,
      "eval_logps/chosen": -260.7673034667969,
      "eval_logps/rejected": -232.97280883789062,
      "eval_loss": 2420.32958984375,
      "eval_rewards/accuracies": 0.656000018119812,
      "eval_rewards/chosen": -0.00523131899535656,
      "eval_rewards/margins": 0.010135524906218052,
      "eval_rewards/rejected": -0.015366843901574612,
      "eval_runtime": 326.0274,
      "eval_samples_per_second": 6.134,
      "eval_steps_per_second": 0.383,
      "step": 200
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.785350472409792e-06,
      "logits/chosen": 0.9068363904953003,
      "logits/rejected": 0.911398708820343,
      "logps/chosen": -233.5305633544922,
      "logps/rejected": -229.12158203125,
      "loss": 2425.1516,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.009300420060753822,
      "rewards/margins": 0.007756868842989206,
      "rewards/rejected": -0.01705729030072689,
      "step": 210
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.746717530629565e-06,
      "logits/chosen": 0.8564063906669617,
      "logits/rejected": 0.8920175433158875,
      "logps/chosen": -260.2020263671875,
      "logps/rejected": -240.0993194580078,
      "loss": 2418.3254,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008474646136164665,
      "rewards/margins": 0.012274968437850475,
      "rewards/rejected": -0.020749617367982864,
      "step": 220
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.7050727958301505e-06,
      "logits/chosen": 0.8939176797866821,
      "logits/rejected": 0.8774939775466919,
      "logps/chosen": -245.0527801513672,
      "logps/rejected": -230.6669921875,
      "loss": 2386.8547,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.011173086240887642,
      "rewards/margins": 0.013905773870646954,
      "rewards/rejected": -0.02507885918021202,
      "step": 230
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.660472094042121e-06,
      "logits/chosen": 0.8556815385818481,
      "logits/rejected": 0.863630473613739,
      "logps/chosen": -282.67669677734375,
      "logps/rejected": -236.6685028076172,
      "loss": 2355.6057,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.013796107843518257,
      "rewards/margins": 0.017949409782886505,
      "rewards/rejected": -0.03174551948904991,
      "step": 240
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.612975213859487e-06,
      "logits/chosen": 0.8427609205245972,
      "logits/rejected": 0.915958046913147,
      "logps/chosen": -269.03228759765625,
      "logps/rejected": -244.0561981201172,
      "loss": 2353.825,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.014950485900044441,
      "rewards/margins": 0.016987096518278122,
      "rewards/rejected": -0.031937580555677414,
      "step": 250
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.5626458262912745e-06,
      "logits/chosen": 0.8542389869689941,
      "logits/rejected": 0.8951870203018188,
      "logps/chosen": -275.2603454589844,
      "logps/rejected": -262.59820556640625,
      "loss": 2344.6988,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.010937584564089775,
      "rewards/margins": 0.02299124002456665,
      "rewards/rejected": -0.033928822726011276,
      "step": 260
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.509551399408598e-06,
      "logits/chosen": 0.9320127367973328,
      "logits/rejected": 0.9399789571762085,
      "logps/chosen": -254.00363159179688,
      "logps/rejected": -211.70822143554688,
      "loss": 2326.323,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.017307719215750694,
      "rewards/margins": 0.020856201648712158,
      "rewards/rejected": -0.038163922727108,
      "step": 270
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.453763107901676e-06,
      "logits/chosen": 0.9306808710098267,
      "logits/rejected": 0.8955329060554504,
      "logps/chosen": -248.00350952148438,
      "logps/rejected": -256.06072998046875,
      "loss": 2353.0586,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.0263187438249588,
      "rewards/margins": 0.0119154192507267,
      "rewards/rejected": -0.0382341630756855,
      "step": 280
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.3953557376679856e-06,
      "logits/chosen": 0.8539811372756958,
      "logits/rejected": 0.860480785369873,
      "logps/chosen": -262.2434387207031,
      "logps/rejected": -258.9552917480469,
      "loss": 2328.1365,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.026760926470160484,
      "rewards/margins": 0.01638859696686268,
      "rewards/rejected": -0.04314952343702316,
      "step": 290
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.33440758555951e-06,
      "logits/chosen": 0.8304749727249146,
      "logits/rejected": 0.9085506200790405,
      "logps/chosen": -251.0150604248047,
      "logps/rejected": -247.31289672851562,
      "loss": 2310.102,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.019448721781373024,
      "rewards/margins": 0.029306888580322266,
      "rewards/rejected": -0.04875560849905014,
      "step": 300
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": 0.8040502071380615,
      "eval_logits/rejected": 0.9088209271430969,
      "eval_logps/chosen": -263.2439880371094,
      "eval_logps/rejected": -237.19593811035156,
      "eval_loss": 2309.945068359375,
      "eval_rewards/accuracies": 0.6679999828338623,
      "eval_rewards/chosen": -0.02999839559197426,
      "eval_rewards/margins": 0.02759976126253605,
      "eval_rewards/rejected": -0.05759815126657486,
      "eval_runtime": 325.793,
      "eval_samples_per_second": 6.139,
      "eval_steps_per_second": 0.384,
      "step": 300
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.2710003544234255e-06,
      "logits/chosen": 0.8506165742874146,
      "logits/rejected": 0.8639786839485168,
      "logps/chosen": -238.32284545898438,
      "logps/rejected": -230.3314971923828,
      "loss": 2280.0254,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.034138236194849014,
      "rewards/margins": 0.023389272391796112,
      "rewards/rejected": -0.05752750486135483,
      "step": 310
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.205219043576955e-06,
      "logits/chosen": 0.8560878038406372,
      "logits/rejected": 0.878685474395752,
      "logps/chosen": -226.87521362304688,
      "logps/rejected": -220.47201538085938,
      "loss": 2272.9057,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03677995875477791,
      "rewards/margins": 0.026556292548775673,
      "rewards/rejected": -0.06333625316619873,
      "step": 320
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.137151834863213e-06,
      "logits/chosen": 0.8369635343551636,
      "logits/rejected": 0.8179060220718384,
      "logps/chosen": -255.9115753173828,
      "logps/rejected": -263.99334716796875,
      "loss": 2288.366,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03619007021188736,
      "rewards/margins": 0.036657560616731644,
      "rewards/rejected": -0.07284761965274811,
      "step": 330
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.066889974440757e-06,
      "logits/chosen": 0.8635396957397461,
      "logits/rejected": 0.8681753873825073,
      "logps/chosen": -268.9547119140625,
      "logps/rejected": -254.25814819335938,
      "loss": 2217.676,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.03367992490530014,
      "rewards/margins": 0.03633836284279823,
      "rewards/rejected": -0.07001828402280807,
      "step": 340
    },
    {
      "epoch": 0.73,
      "learning_rate": 3.994527650465352e-06,
      "logits/chosen": 0.8572274446487427,
      "logits/rejected": 0.8776391744613647,
      "logps/chosen": -226.3207550048828,
      "logps/rejected": -211.4627685546875,
      "loss": 2251.0934,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.046974435448646545,
      "rewards/margins": 0.03458600863814354,
      "rewards/rejected": -0.08156044781208038,
      "step": 350
    },
    {
      "epoch": 0.75,
      "learning_rate": 3.92016186682789e-06,
      "logits/chosen": 0.7831335067749023,
      "logits/rejected": 0.7933910489082336,
      "logps/chosen": -213.00491333007812,
      "logps/rejected": -242.6522979736328,
      "loss": 2245.7613,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.05574915558099747,
      "rewards/margins": 0.030930276960134506,
      "rewards/rejected": -0.08667943626642227,
      "step": 360
    },
    {
      "epoch": 0.77,
      "learning_rate": 3.843892313117724e-06,
      "logits/chosen": 0.8648300170898438,
      "logits/rejected": 0.895352840423584,
      "logps/chosen": -272.79400634765625,
      "logps/rejected": -249.0972900390625,
      "loss": 2275.2465,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.05149676650762558,
      "rewards/margins": 0.0362628772854805,
      "rewards/rejected": -0.08775965869426727,
      "step": 370
    },
    {
      "epoch": 0.8,
      "learning_rate": 3.7658212309857576e-06,
      "logits/chosen": 0.8409647941589355,
      "logits/rejected": 0.8629050254821777,
      "logps/chosen": -243.03182983398438,
      "logps/rejected": -222.12338256835938,
      "loss": 2199.2453,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.053347665816545486,
      "rewards/margins": 0.0404227040708065,
      "rewards/rejected": -0.09377036988735199,
      "step": 380
    },
    {
      "epoch": 0.82,
      "learning_rate": 3.686053277086401e-06,
      "logits/chosen": 0.7878540754318237,
      "logits/rejected": 0.875022292137146,
      "logps/chosen": -266.7389831542969,
      "logps/rejected": -244.7388916015625,
      "loss": 2180.4859,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.05637942627072334,
      "rewards/margins": 0.04192977398633957,
      "rewards/rejected": -0.09830919653177261,
      "step": 390
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.604695382782159e-06,
      "logits/chosen": 0.7989987134933472,
      "logits/rejected": 0.8174804449081421,
      "logps/chosen": -282.3369445800781,
      "logps/rejected": -260.7422790527344,
      "loss": 2159.0707,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05871356651186943,
      "rewards/margins": 0.03764244168996811,
      "rewards/rejected": -0.09635601192712784,
      "step": 400
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": 0.7577926516532898,
      "eval_logits/rejected": 0.8636941313743591,
      "eval_logps/chosen": -266.5838623046875,
      "eval_logps/rejected": -242.28573608398438,
      "eval_loss": 2236.27587890625,
      "eval_rewards/accuracies": 0.6840000152587891,
      "eval_rewards/chosen": -0.06339714676141739,
      "eval_rewards/margins": 0.04509904235601425,
      "eval_rewards/rejected": -0.10849618166685104,
      "eval_runtime": 325.6827,
      "eval_samples_per_second": 6.141,
      "eval_steps_per_second": 0.384,
      "step": 400
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.5218566107988872e-06,
      "logits/chosen": 0.8149998784065247,
      "logits/rejected": 0.8398739695549011,
      "logps/chosen": -274.44647216796875,
      "logps/rejected": -244.11441040039062,
      "loss": 2235.4445,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.060248058289289474,
      "rewards/margins": 0.0490226149559021,
      "rewards/rejected": -0.10927066951990128,
      "step": 410
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.437648009023905e-06,
      "logits/chosen": 0.7635517716407776,
      "logits/rejected": 0.8307437896728516,
      "logps/chosen": -219.5519256591797,
      "logps/rejected": -215.0430145263672,
      "loss": 2210.1293,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07285571843385696,
      "rewards/margins": 0.03309701010584831,
      "rewards/rejected": -0.10595273971557617,
      "step": 420
    },
    {
      "epoch": 0.9,
      "learning_rate": 3.352182461642929e-06,
      "logits/chosen": 0.7868200540542603,
      "logits/rejected": 0.8455543518066406,
      "logps/chosen": -240.498779296875,
      "logps/rejected": -228.08535766601562,
      "loss": 2132.6711,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06197371333837509,
      "rewards/margins": 0.048869095742702484,
      "rewards/rejected": -0.11084280163049698,
      "step": 430
    },
    {
      "epoch": 0.92,
      "learning_rate": 3.265574537815398e-06,
      "logits/chosen": 0.7955938577651978,
      "logits/rejected": 0.8357489705085754,
      "logps/chosen": -285.87493896484375,
      "logps/rejected": -252.0729522705078,
      "loss": 2188.6035,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.05718296021223068,
      "rewards/margins": 0.05849381536245346,
      "rewards/rejected": -0.11567678302526474,
      "step": 440
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.177940338091043e-06,
      "logits/chosen": 0.8076552152633667,
      "logits/rejected": 0.8619295954704285,
      "logps/chosen": -259.4091796875,
      "logps/rejected": -231.5826873779297,
      "loss": 2190.8027,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07417537271976471,
      "rewards/margins": 0.042368099093437195,
      "rewards/rejected": -0.1165434867143631,
      "step": 450
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.089397338773569e-06,
      "logits/chosen": 0.787534236907959,
      "logits/rejected": 0.8202370405197144,
      "logps/chosen": -265.3134460449219,
      "logps/rejected": -237.35116577148438,
      "loss": 2146.2295,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0778832882642746,
      "rewards/margins": 0.05344442278146744,
      "rewards/rejected": -0.13132771849632263,
      "step": 460
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.0000642344401115e-06,
      "logits/chosen": 0.7577365040779114,
      "logits/rejected": 0.803280234336853,
      "logps/chosen": -246.31301879882812,
      "logps/rejected": -235.45675659179688,
      "loss": 2136.8623,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0812118723988533,
      "rewards/margins": 0.04804684966802597,
      "rewards/rejected": -0.12925872206687927,
      "step": 470
    },
    {
      "epoch": 1.0,
      "learning_rate": 2.9100607788275547e-06,
      "logits/chosen": 0.7916151881217957,
      "logits/rejected": 0.8478500247001648,
      "logps/chosen": -251.5865020751953,
      "logps/rejected": -246.8653564453125,
      "loss": 2219.0154,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.08595123142004013,
      "rewards/margins": 0.04087045416235924,
      "rewards/rejected": -0.12682169675827026,
      "step": 480
    },
    {
      "epoch": 1.03,
      "learning_rate": 2.8195076242990124e-06,
      "logits/chosen": 0.7650834321975708,
      "logits/rejected": 0.8132265210151672,
      "logps/chosen": -246.82644653320312,
      "logps/rejected": -227.9211883544922,
      "loss": 2162.0559,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0883156806230545,
      "rewards/margins": 0.050779860466718674,
      "rewards/rejected": -0.13909552991390228,
      "step": 490
    },
    {
      "epoch": 1.05,
      "learning_rate": 2.72852616010567e-06,
      "logits/chosen": 0.7439101338386536,
      "logits/rejected": 0.7856588363647461,
      "logps/chosen": -261.5977478027344,
      "logps/rejected": -247.032470703125,
      "loss": 2176.8641,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.08485061675310135,
      "rewards/margins": 0.05461747199296951,
      "rewards/rejected": -0.13946808874607086,
      "step": 500
    },
    {
      "epoch": 1.05,
      "eval_logits/chosen": 0.7125015258789062,
      "eval_logits/rejected": 0.8179839849472046,
      "eval_logps/chosen": -269.2715759277344,
      "eval_logps/rejected": -246.0634002685547,
      "eval_loss": 2197.5419921875,
      "eval_rewards/accuracies": 0.6980000138282776,
      "eval_rewards/chosen": -0.09027400612831116,
      "eval_rewards/margins": 0.05599898844957352,
      "eval_rewards/rejected": -0.14627300202846527,
      "eval_runtime": 325.7893,
      "eval_samples_per_second": 6.139,
      "eval_steps_per_second": 0.384,
      "step": 500
    },
    {
      "epoch": 1.07,
      "learning_rate": 2.637238349660819e-06,
      "logits/chosen": 0.7544692754745483,
      "logits/rejected": 0.8448120355606079,
      "logps/chosen": -245.3660430908203,
      "logps/rejected": -210.2096405029297,
      "loss": 2188.5398,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0969894677400589,
      "rewards/margins": 0.04615020379424095,
      "rewards/rejected": -0.14313964545726776,
      "step": 510
    },
    {
      "epoch": 1.09,
      "learning_rate": 2.5457665670441937e-06,
      "logits/chosen": 0.8052291870117188,
      "logits/rejected": 0.8244439959526062,
      "logps/chosen": -257.635009765625,
      "logps/rejected": -238.87258911132812,
      "loss": 2140.0687,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.082728311419487,
      "rewards/margins": 0.061602912843227386,
      "rewards/rejected": -0.14433124661445618,
      "step": 520
    },
    {
      "epoch": 1.11,
      "learning_rate": 2.4542334329558075e-06,
      "logits/chosen": 0.7256805896759033,
      "logits/rejected": 0.7552824020385742,
      "logps/chosen": -250.42422485351562,
      "logps/rejected": -241.7630615234375,
      "loss": 2133.6596,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0894266813993454,
      "rewards/margins": 0.05599946528673172,
      "rewards/rejected": -0.14542615413665771,
      "step": 530
    },
    {
      "epoch": 1.13,
      "learning_rate": 2.3627616503391813e-06,
      "logits/chosen": 0.7206599116325378,
      "logits/rejected": 0.7505000233650208,
      "logps/chosen": -267.65875244140625,
      "logps/rejected": -228.6749725341797,
      "loss": 2182.2404,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.09456731379032135,
      "rewards/margins": 0.04886298626661301,
      "rewards/rejected": -0.14343029260635376,
      "step": 540
    },
    {
      "epoch": 1.15,
      "learning_rate": 2.271473839894331e-06,
      "logits/chosen": 0.7229181528091431,
      "logits/rejected": 0.749284029006958,
      "logps/chosen": -276.0394287109375,
      "logps/rejected": -262.62982177734375,
      "loss": 2171.0582,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.09430189430713654,
      "rewards/margins": 0.05570146441459656,
      "rewards/rejected": -0.1500033438205719,
      "step": 550
    },
    {
      "epoch": 1.17,
      "learning_rate": 2.1804923757009885e-06,
      "logits/chosen": 0.694362998008728,
      "logits/rejected": 0.7259857654571533,
      "logps/chosen": -261.21038818359375,
      "logps/rejected": -239.57852172851562,
      "loss": 2163.352,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.100721076130867,
      "rewards/margins": 0.05086208134889603,
      "rewards/rejected": -0.15158315002918243,
      "step": 560
    },
    {
      "epoch": 1.19,
      "learning_rate": 2.089939221172446e-06,
      "logits/chosen": 0.7141777276992798,
      "logits/rejected": 0.7269617915153503,
      "logps/chosen": -279.9233093261719,
      "logps/rejected": -245.4532928466797,
      "loss": 2219.2641,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.10629091411828995,
      "rewards/margins": 0.06119798496365547,
      "rewards/rejected": -0.16748890280723572,
      "step": 570
    },
    {
      "epoch": 1.21,
      "learning_rate": 1.9999357655598894e-06,
      "logits/chosen": 0.7328698635101318,
      "logits/rejected": 0.7043576240539551,
      "logps/chosen": -250.1948699951172,
      "logps/rejected": -239.837890625,
      "loss": 2180.0646,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.10338902473449707,
      "rewards/margins": 0.0403132289648056,
      "rewards/rejected": -0.14370223879814148,
      "step": 580
    },
    {
      "epoch": 1.23,
      "learning_rate": 1.9106026612264316e-06,
      "logits/chosen": 0.7637673616409302,
      "logits/rejected": 0.8286052942276001,
      "logps/chosen": -227.8063507080078,
      "logps/rejected": -227.64193725585938,
      "loss": 2125.3691,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.10240204632282257,
      "rewards/margins": 0.04882645606994629,
      "rewards/rejected": -0.15122851729393005,
      "step": 590
    },
    {
      "epoch": 1.26,
      "learning_rate": 1.8220596619089576e-06,
      "logits/chosen": 0.7344295978546143,
      "logits/rejected": 0.7085897922515869,
      "logps/chosen": -271.92706298828125,
      "logps/rejected": -245.1657257080078,
      "loss": 2066.3285,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.10316000878810883,
      "rewards/margins": 0.061250198632478714,
      "rewards/rejected": -0.16441020369529724,
      "step": 600
    },
    {
      "epoch": 1.26,
      "eval_logits/chosen": 0.6878580451011658,
      "eval_logits/rejected": 0.7927125096321106,
      "eval_logps/chosen": -270.385498046875,
      "eval_logps/rejected": -247.7128448486328,
      "eval_loss": 2177.3388671875,
      "eval_rewards/accuracies": 0.6959999799728394,
      "eval_rewards/chosen": -0.10141333192586899,
      "eval_rewards/margins": 0.0613539032638073,
      "eval_rewards/rejected": -0.16276724636554718,
      "eval_runtime": 325.4497,
      "eval_samples_per_second": 6.145,
      "eval_steps_per_second": 0.384,
      "step": 600
    },
    {
      "epoch": 1.28,
      "learning_rate": 1.7344254621846018e-06,
      "logits/chosen": 0.7047083377838135,
      "logits/rejected": 0.7504035830497742,
      "logps/chosen": -273.6443176269531,
      "logps/rejected": -265.7873840332031,
      "loss": 2070.9469,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.09533126652240753,
      "rewards/margins": 0.06477675586938858,
      "rewards/rejected": -0.16010800004005432,
      "step": 610
    },
    {
      "epoch": 1.3,
      "learning_rate": 1.647817538357072e-06,
      "logits/chosen": 0.7320979237556458,
      "logits/rejected": 0.8024471998214722,
      "logps/chosen": -257.2734680175781,
      "logps/rejected": -243.01651000976562,
      "loss": 2189.2789,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10891245305538177,
      "rewards/margins": 0.04389963299036026,
      "rewards/rejected": -0.15281209349632263,
      "step": 620
    },
    {
      "epoch": 1.32,
      "learning_rate": 1.5623519909760953e-06,
      "logits/chosen": 0.7088596820831299,
      "logits/rejected": 0.7686936259269714,
      "logps/chosen": -253.4821014404297,
      "logps/rejected": -253.3447723388672,
      "loss": 2140.6246,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.11572308838367462,
      "rewards/margins": 0.046286530792713165,
      "rewards/rejected": -0.16200962662696838,
      "step": 630
    },
    {
      "epoch": 1.34,
      "learning_rate": 1.4781433892011132e-06,
      "logits/chosen": 0.7846838235855103,
      "logits/rejected": 0.8031150698661804,
      "logps/chosen": -251.593505859375,
      "logps/rejected": -272.357177734375,
      "loss": 2168.2102,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.10245855897665024,
      "rewards/margins": 0.04714034125208855,
      "rewards/rejected": -0.1495988965034485,
      "step": 640
    },
    {
      "epoch": 1.36,
      "learning_rate": 1.3953046172178413e-06,
      "logits/chosen": 0.7221434116363525,
      "logits/rejected": 0.7379263043403625,
      "logps/chosen": -264.2921142578125,
      "logps/rejected": -261.3705749511719,
      "loss": 2137.5834,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.11888917535543442,
      "rewards/margins": 0.05505634471774101,
      "rewards/rejected": -0.17394550144672394,
      "step": 650
    },
    {
      "epoch": 1.38,
      "learning_rate": 1.3139467229135999e-06,
      "logits/chosen": 0.6965005397796631,
      "logits/rejected": 0.7608405351638794,
      "logps/chosen": -230.50830078125,
      "logps/rejected": -261.0777282714844,
      "loss": 2178.6738,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10801998525857925,
      "rewards/margins": 0.049364469945430756,
      "rewards/rejected": -0.15738445520401,
      "step": 660
    },
    {
      "epoch": 1.4,
      "learning_rate": 1.2341787690142436e-06,
      "logits/chosen": 0.6697909235954285,
      "logits/rejected": 0.7529794573783875,
      "logps/chosen": -309.4987487792969,
      "logps/rejected": -255.5410919189453,
      "loss": 2163.5801,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.09704665839672089,
      "rewards/margins": 0.06983451545238495,
      "rewards/rejected": -0.16688117384910583,
      "step": 670
    },
    {
      "epoch": 1.42,
      "learning_rate": 1.1561076868822756e-06,
      "logits/chosen": 0.6984275579452515,
      "logits/rejected": 0.7366929650306702,
      "logps/chosen": -253.47988891601562,
      "logps/rejected": -230.54940795898438,
      "loss": 2114.4482,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.10325287282466888,
      "rewards/margins": 0.05643494054675102,
      "rewards/rejected": -0.1596878170967102,
      "step": 680
    },
    {
      "epoch": 1.44,
      "learning_rate": 1.079838133172111e-06,
      "logits/chosen": 0.760990560054779,
      "logits/rejected": 0.7594455480575562,
      "logps/chosen": -267.423583984375,
      "logps/rejected": -251.7685089111328,
      "loss": 2198.5418,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0984167829155922,
      "rewards/margins": 0.05027080327272415,
      "rewards/rejected": -0.14868757128715515,
      "step": 690
    },
    {
      "epoch": 1.47,
      "learning_rate": 1.0054723495346484e-06,
      "logits/chosen": 0.6380269527435303,
      "logits/rejected": 0.7157927751541138,
      "logps/chosen": -269.24908447265625,
      "logps/rejected": -241.45919799804688,
      "loss": 2119.5369,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.10670281946659088,
      "rewards/margins": 0.05142299085855484,
      "rewards/rejected": -0.15812578797340393,
      "step": 700
    },
    {
      "epoch": 1.47,
      "eval_logits/chosen": 0.6726287007331848,
      "eval_logits/rejected": 0.7770729660987854,
      "eval_logps/chosen": -270.7824401855469,
      "eval_logps/rejected": -248.45333862304688,
      "eval_loss": 2166.385498046875,
      "eval_rewards/accuracies": 0.6959999799728394,
      "eval_rewards/chosen": -0.10538262128829956,
      "eval_rewards/margins": 0.06478944420814514,
      "eval_rewards/rejected": -0.17017203569412231,
      "eval_runtime": 325.443,
      "eval_samples_per_second": 6.145,
      "eval_steps_per_second": 0.384,
      "step": 700
    },
    {
      "epoch": 1.49,
      "learning_rate": 9.331100255592437e-07,
      "logits/chosen": 0.7291372418403625,
      "logits/rejected": 0.7911130785942078,
      "logps/chosen": -288.43621826171875,
      "logps/rejected": -244.11181640625,
      "loss": 2094.7482,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09699388593435287,
      "rewards/margins": 0.0718715712428093,
      "rewards/rejected": -0.16886545717716217,
      "step": 710
    },
    {
      "epoch": 1.51,
      "learning_rate": 8.628481651367876e-07,
      "logits/chosen": 0.7129195928573608,
      "logits/rejected": 0.7519146203994751,
      "logps/chosen": -273.88372802734375,
      "logps/rejected": -259.33465576171875,
      "loss": 2053.8975,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10259418189525604,
      "rewards/margins": 0.06398696452379227,
      "rewards/rejected": -0.1665811538696289,
      "step": 720
    },
    {
      "epoch": 1.53,
      "learning_rate": 7.947809564230446e-07,
      "logits/chosen": 0.7380334138870239,
      "logits/rejected": 0.7657966017723083,
      "logps/chosen": -275.5352478027344,
      "logps/rejected": -267.64544677734375,
      "loss": 2090.3867,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.10346021503210068,
      "rewards/margins": 0.06666620075702667,
      "rewards/rejected": -0.17012640833854675,
      "step": 730
    },
    {
      "epoch": 1.55,
      "learning_rate": 7.289996455765749e-07,
      "logits/chosen": 0.694438099861145,
      "logits/rejected": 0.7096751928329468,
      "logps/chosen": -285.7628479003906,
      "logps/rejected": -266.2068176269531,
      "loss": 2179.6143,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1050759106874466,
      "rewards/margins": 0.04846997186541557,
      "rewards/rejected": -0.15354588627815247,
      "step": 740
    },
    {
      "epoch": 1.57,
      "learning_rate": 6.655924144404907e-07,
      "logits/chosen": 0.6481191515922546,
      "logits/rejected": 0.7124420404434204,
      "logps/chosen": -269.7061767578125,
      "logps/rejected": -250.74905395507812,
      "loss": 2143.8697,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09301155805587769,
      "rewards/margins": 0.06966546177864075,
      "rewards/rejected": -0.16267701983451843,
      "step": 750
    },
    {
      "epoch": 1.59,
      "learning_rate": 6.046442623320145e-07,
      "logits/chosen": 0.7415434122085571,
      "logits/rejected": 0.77794349193573,
      "logps/chosen": -267.1199035644531,
      "logps/rejected": -233.93765258789062,
      "loss": 2100.39,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.11892955005168915,
      "rewards/margins": 0.048730865120887756,
      "rewards/rejected": -0.1676604300737381,
      "step": 760
    },
    {
      "epoch": 1.61,
      "learning_rate": 5.462368920983249e-07,
      "logits/chosen": 0.7082683444023132,
      "logits/rejected": 0.7616415023803711,
      "logps/chosen": -252.79940795898438,
      "logps/rejected": -231.410400390625,
      "loss": 2186.8764,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.11736402660608292,
      "rewards/margins": 0.03412212058901787,
      "rewards/rejected": -0.1514861285686493,
      "step": 770
    },
    {
      "epoch": 1.63,
      "learning_rate": 4.904486005914027e-07,
      "logits/chosen": 0.7068939805030823,
      "logits/rejected": 0.7558518052101135,
      "logps/chosen": -293.97174072265625,
      "logps/rejected": -274.1463928222656,
      "loss": 2085.8768,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.0985812246799469,
      "rewards/margins": 0.07491330802440643,
      "rewards/rejected": -0.17349454760551453,
      "step": 780
    },
    {
      "epoch": 1.65,
      "learning_rate": 4.373541737087264e-07,
      "logits/chosen": 0.6898752450942993,
      "logits/rejected": 0.7903083562850952,
      "logps/chosen": -275.10650634765625,
      "logps/rejected": -239.94729614257812,
      "loss": 2145.707,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.09976668655872345,
      "rewards/margins": 0.06850672513246536,
      "rewards/rejected": -0.16827340424060822,
      "step": 790
    },
    {
      "epoch": 1.67,
      "learning_rate": 3.8702478614051353e-07,
      "logits/chosen": 0.668254554271698,
      "logits/rejected": 0.6736531257629395,
      "logps/chosen": -248.01626586914062,
      "logps/rejected": -245.8029327392578,
      "loss": 2096.7854,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10279978811740875,
      "rewards/margins": 0.07554516196250916,
      "rewards/rejected": -0.1783449351787567,
      "step": 800
    },
    {
      "epoch": 1.67,
      "eval_logits/chosen": 0.6641319990158081,
      "eval_logits/rejected": 0.7684468030929565,
      "eval_logps/chosen": -271.1501159667969,
      "eval_logps/rejected": -248.99647521972656,
      "eval_loss": 2159.71044921875,
      "eval_rewards/accuracies": 0.6959999799728394,
      "eval_rewards/chosen": -0.10905998200178146,
      "eval_rewards/margins": 0.06654350459575653,
      "eval_rewards/rejected": -0.1756034791469574,
      "eval_runtime": 325.5727,
      "eval_samples_per_second": 6.143,
      "eval_steps_per_second": 0.384,
      "step": 800
    },
    {
      "epoch": 1.7,
      "learning_rate": 3.3952790595787986e-07,
      "logits/chosen": 0.7833539247512817,
      "logits/rejected": 0.7525036931037903,
      "logps/chosen": -260.330810546875,
      "logps/rejected": -247.55337524414062,
      "loss": 2140.7756,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11206915229558945,
      "rewards/margins": 0.04849858209490776,
      "rewards/rejected": -0.1605677306652069,
      "step": 810
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.9492720416985004e-07,
      "logits/chosen": 0.6317057609558105,
      "logits/rejected": 0.6787868738174438,
      "logps/chosen": -283.4757080078125,
      "logps/rejected": -266.5099182128906,
      "loss": 2079.7037,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.09024739265441895,
      "rewards/margins": 0.06939631700515747,
      "rewards/rejected": -0.15964370965957642,
      "step": 820
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.5328246937043526e-07,
      "logits/chosen": 0.7278770208358765,
      "logits/rejected": 0.7481415867805481,
      "logps/chosen": -248.8776397705078,
      "logps/rejected": -233.3114776611328,
      "loss": 2126.3594,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.12706544995307922,
      "rewards/margins": 0.0513269305229187,
      "rewards/rejected": -0.17839238047599792,
      "step": 830
    },
    {
      "epoch": 1.76,
      "learning_rate": 2.1464952759020857e-07,
      "logits/chosen": 0.711814820766449,
      "logits/rejected": 0.7446814775466919,
      "logps/chosen": -260.2667541503906,
      "logps/rejected": -262.43939208984375,
      "loss": 2154.4643,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.10629498958587646,
      "rewards/margins": 0.06066171079874039,
      "rewards/rejected": -0.16695669293403625,
      "step": 840
    },
    {
      "epoch": 1.78,
      "learning_rate": 1.790801674598186e-07,
      "logits/chosen": 0.6856343746185303,
      "logits/rejected": 0.7043182849884033,
      "logps/chosen": -278.94085693359375,
      "logps/rejected": -251.6495819091797,
      "loss": 2141.4652,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.10617993772029877,
      "rewards/margins": 0.0626487284898758,
      "rewards/rejected": -0.16882868111133575,
      "step": 850
    },
    {
      "epoch": 1.8,
      "learning_rate": 1.4662207078575685e-07,
      "logits/chosen": 0.6835039258003235,
      "logits/rejected": 0.7542312145233154,
      "logps/chosen": -242.60501098632812,
      "logps/rejected": -255.89266967773438,
      "loss": 2120.8357,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10873384773731232,
      "rewards/margins": 0.06333796679973602,
      "rewards/rejected": -0.17207179963588715,
      "step": 860
    },
    {
      "epoch": 1.82,
      "learning_rate": 1.1731874863145143e-07,
      "logits/chosen": 0.7217626571655273,
      "logits/rejected": 0.7742848992347717,
      "logps/chosen": -271.91668701171875,
      "logps/rejected": -239.55130004882812,
      "loss": 2097.9199,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.10101070255041122,
      "rewards/margins": 0.06306958198547363,
      "rewards/rejected": -0.16408027708530426,
      "step": 870
    },
    {
      "epoch": 1.84,
      "learning_rate": 9.120948298936422e-08,
      "logits/chosen": 0.7404865026473999,
      "logits/rejected": 0.7562910914421082,
      "logps/chosen": -260.2977294921875,
      "logps/rejected": -240.30502319335938,
      "loss": 2009.9664,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.10928317159414291,
      "rewards/margins": 0.05941414088010788,
      "rewards/rejected": -0.1686973124742508,
      "step": 880
    },
    {
      "epoch": 1.86,
      "learning_rate": 6.832927412229017e-08,
      "logits/chosen": 0.6707200407981873,
      "logits/rejected": 0.7818647623062134,
      "logps/chosen": -271.86065673828125,
      "logps/rejected": -243.53182983398438,
      "loss": 2081.6182,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.10177429020404816,
      "rewards/margins": 0.06641246378421783,
      "rewards/rejected": -0.16818676888942719,
      "step": 890
    },
    {
      "epoch": 1.88,
      "learning_rate": 4.870879364444109e-08,
      "logits/chosen": 0.6861158609390259,
      "logits/rejected": 0.7574108839035034,
      "logps/chosen": -261.988525390625,
      "logps/rejected": -223.02371215820312,
      "loss": 2094.5041,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.10657148063182831,
      "rewards/margins": 0.07416997849941254,
      "rewards/rejected": -0.18074145913124084,
      "step": 900
    },
    {
      "epoch": 1.88,
      "eval_logits/chosen": 0.6645969748497009,
      "eval_logits/rejected": 0.7690178155899048,
      "eval_logps/chosen": -271.27447509765625,
      "eval_logps/rejected": -249.114013671875,
      "eval_loss": 2158.6298828125,
      "eval_rewards/accuracies": 0.6980000138282776,
      "eval_rewards/chosen": -0.11030303686857224,
      "eval_rewards/margins": 0.06647594273090363,
      "eval_rewards/rejected": -0.17677900195121765,
      "eval_runtime": 325.3231,
      "eval_samples_per_second": 6.148,
      "eval_steps_per_second": 0.384,
      "step": 900
    },
    {
      "epoch": 1.9,
      "learning_rate": 3.237434340521789e-08,
      "logits/chosen": 0.6756221055984497,
      "logits/rejected": 0.7528306245803833,
      "logps/chosen": -272.76104736328125,
      "logps/rejected": -262.8546447753906,
      "loss": 2083.5836,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10101411491632462,
      "rewards/margins": 0.06638970226049423,
      "rewards/rejected": -0.16740381717681885,
      "step": 910
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.93478202307823e-08,
      "logits/chosen": 0.6799421310424805,
      "logits/rejected": 0.7030155658721924,
      "logps/chosen": -252.42404174804688,
      "logps/rejected": -260.2074890136719,
      "loss": 2119.1219,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.11061377823352814,
      "rewards/margins": 0.04686294496059418,
      "rewards/rejected": -0.15747670829296112,
      "step": 920
    },
    {
      "epoch": 1.95,
      "learning_rate": 9.646686570697062e-09,
      "logits/chosen": 0.7364012002944946,
      "logits/rejected": 0.7563687562942505,
      "logps/chosen": -267.6513366699219,
      "logps/rejected": -265.22686767578125,
      "loss": 2177.6246,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.11080427467823029,
      "rewards/margins": 0.05903978273272514,
      "rewards/rejected": -0.16984406113624573,
      "step": 930
    },
    {
      "epoch": 1.97,
      "learning_rate": 3.283947088983663e-09,
      "logits/chosen": 0.7186964750289917,
      "logits/rejected": 0.7220372557640076,
      "logps/chosen": -249.5286407470703,
      "logps/rejected": -258.6728515625,
      "loss": 2177.0199,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12288296222686768,
      "rewards/margins": 0.04801628738641739,
      "rewards/rejected": -0.17089924216270447,
      "step": 940
    },
    {
      "epoch": 1.99,
      "learning_rate": 2.681312309735229e-10,
      "logits/chosen": 0.6878038644790649,
      "logits/rejected": 0.7842324376106262,
      "logps/chosen": -242.4557342529297,
      "logps/rejected": -244.35446166992188,
      "loss": 2182.6682,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.11521060764789581,
      "rewards/margins": 0.04983743280172348,
      "rewards/rejected": -0.1650480479001999,
      "step": 950
    },
    {
      "epoch": 2.0,
      "step": 954,
      "total_flos": 0.0,
      "train_loss": 2246.599348344143,
      "train_runtime": 18130.6033,
      "train_samples_per_second": 3.372,
      "train_steps_per_second": 0.053
    }
  ],
  "logging_steps": 10,
  "max_steps": 954,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}