{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9994666666666666,
  "eval_steps": 500,
  "global_step": 937,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 5.319148936170213e-08,
      "logits/chosen": 0.14563202857971191,
      "logits/rejected": 0.3874575197696686,
      "logps/chosen": -454.742919921875,
      "logps/rejected": -268.25054931640625,
      "loss": 0.3396,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.319148936170213e-07,
      "logits/chosen": 0.07718739658594131,
      "logits/rejected": 0.26869574189186096,
      "logps/chosen": -350.5389709472656,
      "logps/rejected": -281.45709228515625,
      "loss": 0.315,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 2.5761094093468273e-06,
      "rewards/margins": -5.979803609079681e-05,
      "rewards/rejected": 6.237414345378056e-05,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0638297872340427e-06,
      "logits/chosen": 0.051278989762067795,
      "logits/rejected": 0.2582473158836365,
      "logps/chosen": -354.6767578125,
      "logps/rejected": -278.68255615234375,
      "loss": 0.3562,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.00019174082262907177,
      "rewards/margins": 5.2712995966430753e-05,
      "rewards/rejected": 0.00013902783393859863,
      "step": 20
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.595744680851064e-06,
      "logits/chosen": 0.087407685816288,
      "logits/rejected": 0.23584076762199402,
      "logps/chosen": -341.74859619140625,
      "logps/rejected": -292.8842468261719,
      "loss": 0.3276,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.00011408107820898294,
      "rewards/margins": 0.00017452315660193563,
      "rewards/rejected": -6.044206747901626e-05,
      "step": 30
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.1276595744680853e-06,
      "logits/chosen": 0.07952137291431427,
      "logits/rejected": 0.18448325991630554,
      "logps/chosen": -345.06378173828125,
      "logps/rejected": -308.3074951171875,
      "loss": 0.3338,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 6.922694592503831e-05,
      "rewards/margins": 0.0002068312169285491,
      "rewards/rejected": -0.0001376042637275532,
      "step": 40
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6595744680851065e-06,
      "logits/chosen": 0.1114521399140358,
      "logits/rejected": 0.2774251103401184,
      "logps/chosen": -326.83441162109375,
      "logps/rejected": -293.80291748046875,
      "loss": 0.3451,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.00041769607923924923,
      "rewards/margins": 0.0006514322012662888,
      "rewards/rejected": -0.00023373604926746339,
      "step": 50
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.191489361702128e-06,
      "logits/chosen": 0.09482766687870026,
      "logits/rejected": 0.27632468938827515,
      "logps/chosen": -351.2109680175781,
      "logps/rejected": -277.0458068847656,
      "loss": 0.339,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0010114657925441861,
      "rewards/margins": 0.0016303223092108965,
      "rewards/rejected": -0.0006188566330820322,
      "step": 60
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.723404255319149e-06,
      "logits/chosen": 0.02542046643793583,
      "logits/rejected": 0.19503046572208405,
      "logps/chosen": -325.17486572265625,
      "logps/rejected": -285.79205322265625,
      "loss": 0.3265,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.001514864037744701,
      "rewards/margins": 0.002718320582062006,
      "rewards/rejected": -0.0012034563114866614,
      "step": 70
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.255319148936171e-06,
      "logits/chosen": 0.13358162343502045,
      "logits/rejected": 0.31503233313560486,
      "logps/chosen": -368.6209716796875,
      "logps/rejected": -295.80133056640625,
      "loss": 0.3393,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.002275743056088686,
      "rewards/margins": 0.005096759647130966,
      "rewards/rejected": -0.0028210170567035675,
      "step": 80
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.787234042553192e-06,
      "logits/chosen": 0.08928992599248886,
      "logits/rejected": 0.24500127136707306,
      "logps/chosen": -346.7084045410156,
      "logps/rejected": -290.81646728515625,
      "loss": 0.3163,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.004672099836170673,
      "rewards/margins": 0.009218016639351845,
      "rewards/rejected": -0.004545916803181171,
      "step": 90
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999375059004058e-06,
      "logits/chosen": 0.17046232521533966,
      "logits/rejected": 0.22814074158668518,
      "logps/chosen": -332.4991760253906,
      "logps/rejected": -293.03314208984375,
      "loss": 0.3288,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.0032020125072449446,
      "rewards/margins": 0.01331277471035719,
      "rewards/rejected": -0.010110762901604176,
      "step": 100
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9955571065548795e-06,
      "logits/chosen": 0.13835321366786957,
      "logits/rejected": 0.2707987427711487,
      "logps/chosen": -353.07904052734375,
      "logps/rejected": -316.55377197265625,
      "loss": 0.3145,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0036005068104714155,
      "rewards/margins": 0.02047211304306984,
      "rewards/rejected": -0.024072617292404175,
      "step": 110
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9882736864879e-06,
      "logits/chosen": 0.14788897335529327,
      "logits/rejected": 0.2877771258354187,
      "logps/chosen": -368.3328857421875,
      "logps/rejected": -326.0980224609375,
      "loss": 0.3211,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.009556634351611137,
      "rewards/margins": 0.031165510416030884,
      "rewards/rejected": -0.04072214290499687,
      "step": 120
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.977534912960124e-06,
      "logits/chosen": 0.09604962915182114,
      "logits/rejected": 0.29759398102760315,
      "logps/chosen": -394.56146240234375,
      "logps/rejected": -389.3465270996094,
      "loss": 0.2595,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.04469118267297745,
      "rewards/margins": 0.059812031686306,
      "rewards/rejected": -0.10450321435928345,
      "step": 130
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.963355698422092e-06,
      "logits/chosen": 0.09464170038700104,
      "logits/rejected": 0.23635534942150116,
      "logps/chosen": -411.34344482421875,
      "logps/rejected": -472.7259216308594,
      "loss": 0.2809,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.09930221736431122,
      "rewards/margins": 0.08144193887710571,
      "rewards/rejected": -0.18074414134025574,
      "step": 140
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.945755732909625e-06,
      "logits/chosen": 0.15742775797843933,
      "logits/rejected": 0.2725524306297302,
      "logps/chosen": -477.2586975097656,
      "logps/rejected": -517.0968627929688,
      "loss": 0.2592,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.13660530745983124,
      "rewards/margins": 0.11236293613910675,
      "rewards/rejected": -0.2489682137966156,
      "step": 150
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.924759456701167e-06,
      "logits/chosen": 0.19448170065879822,
      "logits/rejected": 0.25956863164901733,
      "logps/chosen": -487.07196044921875,
      "logps/rejected": -597.939208984375,
      "loss": 0.2663,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15577515959739685,
      "rewards/margins": 0.15100589394569397,
      "rewards/rejected": -0.3067810833454132,
      "step": 160
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.900396026378671e-06,
      "logits/chosen": 0.18750979006290436,
      "logits/rejected": 0.326152503490448,
      "logps/chosen": -559.5252685546875,
      "logps/rejected": -645.9837036132812,
      "loss": 0.2719,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.22856616973876953,
      "rewards/margins": 0.12326528877019882,
      "rewards/rejected": -0.35183146595954895,
      "step": 170
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.872699274339169e-06,
      "logits/chosen": 0.16625967621803284,
      "logits/rejected": 0.29693764448165894,
      "logps/chosen": -530.322021484375,
      "logps/rejected": -600.0885009765625,
      "loss": 0.2955,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.18995243310928345,
      "rewards/margins": 0.14340588450431824,
      "rewards/rejected": -0.3333582878112793,
      "step": 180
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8417076618132434e-06,
      "logits/chosen": 0.135554239153862,
      "logits/rejected": 0.26710349321365356,
      "logps/chosen": -582.8619384765625,
      "logps/rejected": -616.3886108398438,
      "loss": 0.2737,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.21700850129127502,
      "rewards/margins": 0.12937961518764496,
      "rewards/rejected": -0.3463881015777588,
      "step": 190
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.807464225455655e-06,
      "logits/chosen": 0.10434029996395111,
      "logits/rejected": 0.28046977519989014,
      "logps/chosen": -585.6417236328125,
      "logps/rejected": -614.5296630859375,
      "loss": 0.2886,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2205405980348587,
      "rewards/margins": 0.12526755034923553,
      "rewards/rejected": -0.34580811858177185,
      "step": 200
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.770016517582283e-06,
      "logits/chosen": 0.1413886696100235,
      "logits/rejected": 0.27238231897354126,
      "logps/chosen": -519.6276245117188,
      "logps/rejected": -587.7799072265625,
      "loss": 0.2557,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.16974905133247375,
      "rewards/margins": 0.1322554051876068,
      "rewards/rejected": -0.30200448632240295,
      "step": 210
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7294165401363616e-06,
      "logits/chosen": 0.1467094123363495,
      "logits/rejected": 0.2754250466823578,
      "logps/chosen": -554.9280395507812,
      "logps/rejected": -615.7232666015625,
      "loss": 0.2471,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.21137747168540955,
      "rewards/margins": 0.1308281123638153,
      "rewards/rejected": -0.34220558404922485,
      "step": 220
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.68572067247573e-06,
      "logits/chosen": 0.1171431764960289,
      "logits/rejected": 0.2890062928199768,
      "logps/chosen": -575.1917724609375,
      "logps/rejected": -651.8801879882812,
      "loss": 0.244,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.24391531944274902,
      "rewards/margins": 0.15722794830799103,
      "rewards/rejected": -0.40114325284957886,
      "step": 230
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.638989593081364e-06,
      "logits/chosen": 0.10296036303043365,
      "logits/rejected": 0.28510022163391113,
      "logps/chosen": -640.1365966796875,
      "logps/rejected": -720.7387084960938,
      "loss": 0.2331,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.2727048397064209,
      "rewards/margins": 0.1723267138004303,
      "rewards/rejected": -0.4450315833091736,
      "step": 240
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5892881952959015e-06,
      "logits/chosen": 0.15176770091056824,
      "logits/rejected": 0.22689513862133026,
      "logps/chosen": -542.8187255859375,
      "logps/rejected": -679.3350830078125,
      "loss": 0.2444,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.23740482330322266,
      "rewards/margins": 0.16511742770671844,
      "rewards/rejected": -0.4025222659111023,
      "step": 250
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.536685497209182e-06,
      "logits/chosen": 0.1848079115152359,
      "logits/rejected": 0.28918254375457764,
      "logps/chosen": -591.1017456054688,
      "logps/rejected": -719.2144775390625,
      "loss": 0.2388,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.23014459013938904,
      "rewards/margins": 0.18921832740306854,
      "rewards/rejected": -0.4193629324436188,
      "step": 260
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.481254545815943e-06,
      "logits/chosen": 0.16476555168628693,
      "logits/rejected": 0.23265871405601501,
      "logps/chosen": -612.1968383789062,
      "logps/rejected": -663.5397338867188,
      "loss": 0.2918,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.24918679893016815,
      "rewards/margins": 0.11979563534259796,
      "rewards/rejected": -0.3689824044704437,
      "step": 270
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.42307231557875e-06,
      "logits/chosen": 0.180597186088562,
      "logits/rejected": 0.3083464503288269,
      "logps/chosen": -560.151123046875,
      "logps/rejected": -635.0821533203125,
      "loss": 0.2582,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2051992118358612,
      "rewards/margins": 0.14893193542957306,
      "rewards/rejected": -0.35413113236427307,
      "step": 280
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3622196015370305e-06,
      "logits/chosen": 0.22149360179901123,
      "logits/rejected": 0.3726951479911804,
      "logps/chosen": -628.3652954101562,
      "logps/rejected": -711.3004150390625,
      "loss": 0.2145,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.24089030921459198,
      "rewards/margins": 0.16530363261699677,
      "rewards/rejected": -0.40619391202926636,
      "step": 290
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.298780907110648e-06,
      "logits/chosen": 0.17435990273952484,
      "logits/rejected": 0.3360474705696106,
      "logps/chosen": -610.1092529296875,
      "logps/rejected": -695.5682983398438,
      "loss": 0.2356,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.26700595021247864,
      "rewards/margins": 0.1659780740737915,
      "rewards/rejected": -0.43298402428627014,
      "step": 300
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.23284432675381e-06,
      "logits/chosen": 0.177028626203537,
      "logits/rejected": 0.2775452136993408,
      "logps/chosen": -601.6631469726562,
      "logps/rejected": -679.7595825195312,
      "loss": 0.2454,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2515362799167633,
      "rewards/margins": 0.15310627222061157,
      "rewards/rejected": -0.4046425223350525,
      "step": 310
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.164501423622277e-06,
      "logits/chosen": 0.17227385938167572,
      "logits/rejected": 0.2626027464866638,
      "logps/chosen": -565.1954956054688,
      "logps/rejected": -653.8951416015625,
      "loss": 0.2601,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.22992494702339172,
      "rewards/margins": 0.14292170107364655,
      "rewards/rejected": -0.3728466331958771,
      "step": 320
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.0938471024237355e-06,
      "logits/chosen": 0.17768487334251404,
      "logits/rejected": 0.33285486698150635,
      "logps/chosen": -610.06201171875,
      "logps/rejected": -670.5873413085938,
      "loss": 0.238,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23991577327251434,
      "rewards/margins": 0.15924175083637238,
      "rewards/rejected": -0.39915746450424194,
      "step": 330
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.020979477627907e-06,
      "logits/chosen": 0.12916894257068634,
      "logits/rejected": 0.2785950303077698,
      "logps/chosen": -571.642578125,
      "logps/rejected": -680.286865234375,
      "loss": 0.2314,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.2410048544406891,
      "rewards/margins": 0.15611061453819275,
      "rewards/rejected": -0.39711546897888184,
      "step": 340
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9459997372194105e-06,
      "logits/chosen": 0.1318448781967163,
      "logits/rejected": 0.2396216094493866,
      "logps/chosen": -577.1714477539062,
      "logps/rejected": -694.001220703125,
      "loss": 0.23,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.25458115339279175,
      "rewards/margins": 0.16680863499641418,
      "rewards/rejected": -0.42138975858688354,
      "step": 350
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.869012002182573e-06,
      "logits/chosen": 0.1819082796573639,
      "logits/rejected": 0.30339935421943665,
      "logps/chosen": -620.7168579101562,
      "logps/rejected": -733.3218994140625,
      "loss": 0.2306,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.24643263220787048,
      "rewards/margins": 0.19664740562438965,
      "rewards/rejected": -0.4430800974369049,
      "step": 360
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7901231819133104e-06,
      "logits/chosen": 0.1591918021440506,
      "logits/rejected": 0.24382638931274414,
      "logps/chosen": -650.5294189453125,
      "logps/rejected": -757.6646118164062,
      "loss": 0.2606,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.29114747047424316,
      "rewards/margins": 0.1727633774280548,
      "rewards/rejected": -0.46391087770462036,
      "step": 370
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.709442825758875e-06,
      "logits/chosen": 0.15119919180870056,
      "logits/rejected": 0.24405035376548767,
      "logps/chosen": -622.4024658203125,
      "logps/rejected": -731.4509887695312,
      "loss": 0.2127,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.2556970715522766,
      "rewards/margins": 0.20232219994068146,
      "rewards/rejected": -0.4580192565917969,
      "step": 380
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6270829708916113e-06,
      "logits/chosen": 0.16059614717960358,
      "logits/rejected": 0.26955538988113403,
      "logps/chosen": -620.3678588867188,
      "logps/rejected": -734.6710205078125,
      "loss": 0.2319,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.26077157258987427,
      "rewards/margins": 0.18923643231391907,
      "rewards/rejected": -0.4500080645084381,
      "step": 390
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.543157986727991e-06,
      "logits/chosen": 0.15299072861671448,
      "logits/rejected": 0.27529269456863403,
      "logps/chosen": -626.7930908203125,
      "logps/rejected": -771.9212646484375,
      "loss": 0.1765,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.25388702750205994,
      "rewards/margins": 0.23886942863464355,
      "rewards/rejected": -0.49275651574134827,
      "step": 400
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4577844161089614e-06,
      "logits/chosen": 0.10297727584838867,
      "logits/rejected": 0.2494010478258133,
      "logps/chosen": -701.2484130859375,
      "logps/rejected": -797.5471801757812,
      "loss": 0.2255,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3326496481895447,
      "rewards/margins": 0.19009152054786682,
      "rewards/rejected": -0.5227411985397339,
      "step": 410
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3710808134621577e-06,
      "logits/chosen": 0.0838174819946289,
      "logits/rejected": 0.24609944224357605,
      "logps/chosen": -691.7960205078125,
      "logps/rejected": -792.8016357421875,
      "loss": 0.2473,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3393723964691162,
      "rewards/margins": 0.17680588364601135,
      "rewards/rejected": -0.5161782503128052,
      "step": 420
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2831675801707126e-06,
      "logits/chosen": 0.13857319951057434,
      "logits/rejected": 0.25091537833213806,
      "logps/chosen": -635.0294189453125,
      "logps/rejected": -770.2885131835938,
      "loss": 0.2029,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.28675633668899536,
      "rewards/margins": 0.20502133667469025,
      "rewards/rejected": -0.4917777180671692,
      "step": 430
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.194166797377289e-06,
      "logits/chosen": 0.09761005640029907,
      "logits/rejected": 0.21226266026496887,
      "logps/chosen": -669.2428588867188,
      "logps/rejected": -749.3931884765625,
      "loss": 0.242,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.3128831088542938,
      "rewards/margins": 0.17808611690998077,
      "rewards/rejected": -0.4909692406654358,
      "step": 440
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.104202056455501e-06,
      "logits/chosen": 0.1553424596786499,
      "logits/rejected": 0.25895553827285767,
      "logps/chosen": -694.4083862304688,
      "logps/rejected": -792.1837158203125,
      "loss": 0.2174,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.3040028214454651,
      "rewards/margins": 0.2048918753862381,
      "rewards/rejected": -0.508894681930542,
      "step": 450
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.013398287384144e-06,
      "logits/chosen": 0.1065811887383461,
      "logits/rejected": 0.25707563757896423,
      "logps/chosen": -613.1231689453125,
      "logps/rejected": -758.7059326171875,
      "loss": 0.2107,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.26900747418403625,
      "rewards/margins": 0.21452918648719788,
      "rewards/rejected": -0.48353663086891174,
      "step": 460
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9218815852625717e-06,
      "logits/chosen": 0.16053031384944916,
      "logits/rejected": 0.2852506935596466,
      "logps/chosen": -637.3067626953125,
      "logps/rejected": -732.3541259765625,
      "loss": 0.2118,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2603939175605774,
      "rewards/margins": 0.18671852350234985,
      "rewards/rejected": -0.44711247086524963,
      "step": 470
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.829779035208113e-06,
      "logits/chosen": 0.1389404833316803,
      "logits/rejected": 0.2467120885848999,
      "logps/chosen": -638.651123046875,
      "logps/rejected": -754.9122314453125,
      "loss": 0.2233,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.26695486903190613,
      "rewards/margins": 0.20951655507087708,
      "rewards/rejected": -0.476471483707428,
      "step": 480
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.737218535878705e-06,
      "logits/chosen": 0.11784114688634872,
      "logits/rejected": 0.18245339393615723,
      "logps/chosen": -603.6224365234375,
      "logps/rejected": -770.082275390625,
      "loss": 0.2182,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.28938090801239014,
      "rewards/margins": 0.2138332575559616,
      "rewards/rejected": -0.5032141804695129,
      "step": 490
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.64432862186579e-06,
      "logits/chosen": 0.11818281561136246,
      "logits/rejected": 0.26330405473709106,
      "logps/chosen": -623.1153564453125,
      "logps/rejected": -809.2369384765625,
      "loss": 0.2143,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3059650957584381,
      "rewards/margins": 0.23646679520606995,
      "rewards/rejected": -0.5424319505691528,
      "step": 500
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.551238285204126e-06,
      "logits/chosen": 0.16755148768424988,
      "logits/rejected": 0.28101032972335815,
      "logps/chosen": -677.8587646484375,
      "logps/rejected": -864.1910400390625,
      "loss": 0.2025,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.32487982511520386,
      "rewards/margins": 0.2524875998497009,
      "rewards/rejected": -0.5773674249649048,
      "step": 510
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4580767962463688e-06,
      "logits/chosen": 0.08524368703365326,
      "logits/rejected": 0.21973176300525665,
      "logps/chosen": -697.4478759765625,
      "logps/rejected": -855.9630737304688,
      "loss": 0.22,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.33033424615859985,
      "rewards/margins": 0.21664047241210938,
      "rewards/rejected": -0.5469747185707092,
      "step": 520
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3649735241511546e-06,
      "logits/chosen": 0.1285276859998703,
      "logits/rejected": 0.2174634039402008,
      "logps/chosen": -610.0903930664062,
      "logps/rejected": -795.0265502929688,
      "loss": 0.1928,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.293308824300766,
      "rewards/margins": 0.21643824875354767,
      "rewards/rejected": -0.5097470879554749,
      "step": 530
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2720577572339914e-06,
      "logits/chosen": 0.14484889805316925,
      "logits/rejected": 0.20977553725242615,
      "logps/chosen": -650.8162231445312,
      "logps/rejected": -814.1326293945312,
      "loss": 0.2019,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.2902231812477112,
      "rewards/margins": 0.24104826152324677,
      "rewards/rejected": -0.5312714576721191,
      "step": 540
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1794585234303995e-06,
      "logits/chosen": 0.11247887462377548,
      "logits/rejected": 0.22003412246704102,
      "logps/chosen": -649.1162719726562,
      "logps/rejected": -767.150634765625,
      "loss": 0.2465,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.3138406574726105,
      "rewards/margins": 0.182392418384552,
      "rewards/rejected": -0.49623313546180725,
      "step": 550
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0873044111206407e-06,
      "logits/chosen": 0.057500071823596954,
      "logits/rejected": 0.19665978848934174,
      "logps/chosen": -651.9990844726562,
      "logps/rejected": -783.0087890625,
      "loss": 0.2255,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.3106940984725952,
      "rewards/margins": 0.20310230553150177,
      "rewards/rejected": -0.5137964487075806,
      "step": 560
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9957233905648293e-06,
      "logits/chosen": 0.11110985279083252,
      "logits/rejected": 0.2312009632587433,
      "logps/chosen": -621.1902465820312,
      "logps/rejected": -777.6902465820312,
      "loss": 0.1934,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.25909268856048584,
      "rewards/margins": 0.2260764092206955,
      "rewards/rejected": -0.48516911268234253,
      "step": 570
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.904842636196402e-06,
      "logits/chosen": 0.14112098515033722,
      "logits/rejected": 0.2636359930038452,
      "logps/chosen": -635.9463500976562,
      "logps/rejected": -735.6661376953125,
      "loss": 0.2452,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.266891747713089,
      "rewards/margins": 0.19298222661018372,
      "rewards/rejected": -0.4598739743232727,
      "step": 580
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.814788350020726e-06,
      "logits/chosen": 0.11148792505264282,
      "logits/rejected": 0.25395482778549194,
      "logps/chosen": -586.4432373046875,
      "logps/rejected": -725.523193359375,
      "loss": 0.224,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2619563043117523,
      "rewards/margins": 0.19483323395252228,
      "rewards/rejected": -0.4567895531654358,
      "step": 590
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.725685586364051e-06,
      "logits/chosen": 0.09673704952001572,
      "logits/rejected": 0.20130054652690887,
      "logps/chosen": -615.4747314453125,
      "logps/rejected": -788.6925659179688,
      "loss": 0.2763,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.27522048354148865,
      "rewards/margins": 0.21255996823310852,
      "rewards/rejected": -0.48778048157691956,
      "step": 600
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6376580782162172e-06,
      "logits/chosen": 0.07239886373281479,
      "logits/rejected": 0.18439838290214539,
      "logps/chosen": -564.4202880859375,
      "logps/rejected": -704.7264404296875,
      "loss": 0.2415,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.25343528389930725,
      "rewards/margins": 0.1700516790151596,
      "rewards/rejected": -0.42348694801330566,
      "step": 610
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.550828065408227e-06,
      "logits/chosen": 0.10070653259754181,
      "logits/rejected": 0.19296300411224365,
      "logps/chosen": -595.9661865234375,
      "logps/rejected": -724.99658203125,
      "loss": 0.2326,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.23254573345184326,
      "rewards/margins": 0.19659695029258728,
      "rewards/rejected": -0.42914265394210815,
      "step": 620
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4653161248633053e-06,
      "logits/chosen": 0.08602897822856903,
      "logits/rejected": 0.1648799628019333,
      "logps/chosen": -553.3023681640625,
      "logps/rejected": -709.7655639648438,
      "loss": 0.2185,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.2514170706272125,
      "rewards/margins": 0.1915900856256485,
      "rewards/rejected": -0.4430071711540222,
      "step": 630
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.381241003157162e-06,
      "logits/chosen": 0.0444771870970726,
      "logits/rejected": 0.16466517746448517,
      "logps/chosen": -611.7086181640625,
      "logps/rejected": -756.202392578125,
      "loss": 0.2093,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.27020856738090515,
      "rewards/margins": 0.20864149928092957,
      "rewards/rejected": -0.4788500666618347,
      "step": 640
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.298719451619979e-06,
      "logits/chosen": 0.08262890577316284,
      "logits/rejected": 0.21953897178173065,
      "logps/chosen": -657.6599731445312,
      "logps/rejected": -753.6175537109375,
      "loss": 0.2635,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.30637097358703613,
      "rewards/margins": 0.1610550731420517,
      "rewards/rejected": -0.46742600202560425,
      "step": 650
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2178660642091036e-06,
      "logits/chosen": 0.08547943085432053,
      "logits/rejected": 0.22080206871032715,
      "logps/chosen": -639.3602905273438,
      "logps/rejected": -765.0423583984375,
      "loss": 0.2281,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2679855227470398,
      "rewards/margins": 0.2209259271621704,
      "rewards/rejected": -0.4889114797115326,
      "step": 660
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1387931183775821e-06,
      "logits/chosen": 0.10255439579486847,
      "logits/rejected": 0.17642661929130554,
      "logps/chosen": -590.3636474609375,
      "logps/rejected": -765.87890625,
      "loss": 0.2285,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.2688559293746948,
      "rewards/margins": 0.2031112015247345,
      "rewards/rejected": -0.47196707129478455,
      "step": 670
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.061610419159532e-06,
      "logits/chosen": 0.09613371640443802,
      "logits/rejected": 0.20681801438331604,
      "logps/chosen": -639.8070678710938,
      "logps/rejected": -783.1290283203125,
      "loss": 0.2284,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.28017768263816833,
      "rewards/margins": 0.2119220495223999,
      "rewards/rejected": -0.4920997619628906,
      "step": 680
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.864251466888364e-07,
      "logits/chosen": 0.10619207471609116,
      "logits/rejected": 0.19468547403812408,
      "logps/chosen": -697.66162109375,
      "logps/rejected": -777.8093872070312,
      "loss": 0.2456,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.31186243891716003,
      "rewards/margins": 0.1904953420162201,
      "rewards/rejected": -0.5023577809333801,
      "step": 690
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.133417073629288e-07,
      "logits/chosen": 0.12791021168231964,
      "logits/rejected": 0.18563024699687958,
      "logps/chosen": -692.7999267578125,
      "logps/rejected": -838.0908203125,
      "loss": 0.2424,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.30099743604660034,
      "rewards/margins": 0.24136948585510254,
      "rewards/rejected": -0.5423668622970581,
      "step": 700
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.424615888583332e-07,
      "logits/chosen": 0.10475460439920425,
      "logits/rejected": 0.17532803118228912,
      "logps/chosen": -574.7904663085938,
      "logps/rejected": -732.5684814453125,
      "loss": 0.2116,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.27657538652420044,
      "rewards/margins": 0.19332227110862732,
      "rewards/rejected": -0.46989765763282776,
      "step": 710
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.738832191993092e-07,
      "logits/chosen": 0.11339239031076431,
      "logits/rejected": 0.23391477763652802,
      "logps/chosen": -614.44384765625,
      "logps/rejected": -739.0775756835938,
      "loss": 0.2129,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.26443395018577576,
      "rewards/margins": 0.20620107650756836,
      "rewards/rejected": -0.47063499689102173,
      "step": 720
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.077018300752917e-07,
      "logits/chosen": 0.11114220321178436,
      "logits/rejected": 0.1741330623626709,
      "logps/chosen": -649.84423828125,
      "logps/rejected": -796.5662841796875,
      "loss": 0.249,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.30554836988449097,
      "rewards/margins": 0.18621903657913208,
      "rewards/rejected": -0.49176740646362305,
      "step": 730
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.440093245969342e-07,
      "logits/chosen": 0.05583947151899338,
      "logits/rejected": 0.208854079246521,
      "logps/chosen": -677.1410522460938,
      "logps/rejected": -806.2962646484375,
      "loss": 0.2157,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.3019706606864929,
      "rewards/margins": 0.21764162182807922,
      "rewards/rejected": -0.5196123123168945,
      "step": 740
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.828941496744075e-07,
      "logits/chosen": 0.11103875935077667,
      "logits/rejected": 0.23960928618907928,
      "logps/chosen": -651.9210205078125,
      "logps/rejected": -767.1956176757812,
      "loss": 0.2196,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.28146857023239136,
      "rewards/margins": 0.21090731024742126,
      "rewards/rejected": -0.492375910282135,
      "step": 750
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.244411731951671e-07,
      "logits/chosen": 0.08522719889879227,
      "logits/rejected": 0.18019166588783264,
      "logps/chosen": -693.9295043945312,
      "logps/rejected": -834.8268432617188,
      "loss": 0.2044,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.3021830916404724,
      "rewards/margins": 0.22599780559539795,
      "rewards/rejected": -0.5281809568405151,
      "step": 760
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.6873156617173594e-07,
      "logits/chosen": 0.10727125406265259,
      "logits/rejected": 0.2462559938430786,
      "logps/chosen": -690.4157104492188,
      "logps/rejected": -837.8204956054688,
      "loss": 0.2228,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.3111000657081604,
      "rewards/margins": 0.22689378261566162,
      "rewards/rejected": -0.5379937887191772,
      "step": 770
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1584269002318653e-07,
      "logits/chosen": 0.10468622297048569,
      "logits/rejected": 0.21340981125831604,
      "logps/chosen": -664.1920166015625,
      "logps/rejected": -791.6151123046875,
      "loss": 0.2025,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.28642627596855164,
      "rewards/margins": 0.2306082546710968,
      "rewards/rejected": -0.5170345306396484,
      "step": 780
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.658479891468258e-07,
      "logits/chosen": 0.11628569662570953,
      "logits/rejected": 0.23161634802818298,
      "logps/chosen": -672.2249755859375,
      "logps/rejected": -801.2711181640625,
      "loss": 0.2464,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3202977776527405,
      "rewards/margins": 0.2009088099002838,
      "rewards/rejected": -0.5212065577507019,
      "step": 790
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.18816888929272e-07,
      "logits/chosen": 0.06959172338247299,
      "logits/rejected": 0.2281877100467682,
      "logps/chosen": -652.8377685546875,
      "logps/rejected": -795.1046752929688,
      "loss": 0.2284,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2909380793571472,
      "rewards/margins": 0.22871951758861542,
      "rewards/rejected": -0.5196576714515686,
      "step": 800
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.748146993385484e-07,
      "logits/chosen": 0.11237098276615143,
      "logits/rejected": 0.21487274765968323,
      "logps/chosen": -660.4642944335938,
      "logps/rejected": -802.35009765625,
      "loss": 0.196,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.31749480962753296,
      "rewards/margins": 0.21312642097473145,
      "rewards/rejected": -0.5306212306022644,
      "step": 810
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3390252423108077e-07,
      "logits/chosen": 0.04496614262461662,
      "logits/rejected": 0.25041455030441284,
      "logps/chosen": -633.7536010742188,
      "logps/rejected": -752.9474487304688,
      "loss": 0.2326,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.303285151720047,
      "rewards/margins": 0.18882021307945251,
      "rewards/rejected": -0.4921053349971771,
      "step": 820
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.961371764995243e-07,
      "logits/chosen": 0.07390692830085754,
      "logits/rejected": 0.17046715319156647,
      "logps/chosen": -650.1712646484375,
      "logps/rejected": -773.189697265625,
      "loss": 0.2534,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.32713738083839417,
      "rewards/margins": 0.16583597660064697,
      "rewards/rejected": -0.4929733872413635,
      "step": 830
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.61571099179261e-07,
      "logits/chosen": 0.08455213159322739,
      "logits/rejected": 0.2394600361585617,
      "logps/chosen": -624.288818359375,
      "logps/rejected": -786.5145263671875,
      "loss": 0.2184,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.28965824842453003,
      "rewards/margins": 0.20828434824943542,
      "rewards/rejected": -0.49794262647628784,
      "step": 840
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.3025229262312367e-07,
      "logits/chosen": 0.08053167164325714,
      "logits/rejected": 0.1703474223613739,
      "logps/chosen": -638.90966796875,
      "logps/rejected": -760.4950561523438,
      "loss": 0.2568,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.29267439246177673,
      "rewards/margins": 0.18851785361766815,
      "rewards/rejected": -0.48119229078292847,
      "step": 850
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0222424784546853e-07,
      "logits/chosen": 0.08862993866205215,
      "logits/rejected": 0.2355298101902008,
      "logps/chosen": -659.1481323242188,
      "logps/rejected": -788.7034912109375,
      "loss": 0.2149,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.28642982244491577,
      "rewards/margins": 0.22406980395317078,
      "rewards/rejected": -0.5104995965957642,
      "step": 860
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.752588612816553e-08,
      "logits/chosen": 0.09468914568424225,
      "logits/rejected": 0.17423442006111145,
      "logps/chosen": -628.0216064453125,
      "logps/rejected": -736.0591430664062,
      "loss": 0.2635,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.29823777079582214,
      "rewards/margins": 0.16002070903778076,
      "rewards/rejected": -0.4582585394382477,
      "step": 870
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.619150497236991e-08,
      "logits/chosen": 0.1822114735841751,
      "logits/rejected": 0.25673800706863403,
      "logps/chosen": -647.0616455078125,
      "logps/rejected": -785.24853515625,
      "loss": 0.2055,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2835825979709625,
      "rewards/margins": 0.21645894646644592,
      "rewards/rejected": -0.5000415444374084,
      "step": 880
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.825073047112743e-08,
      "logits/chosen": 0.08051988482475281,
      "logits/rejected": 0.22623100876808167,
      "logps/chosen": -661.945068359375,
      "logps/rejected": -801.7554321289062,
      "loss": 0.279,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.3187974691390991,
      "rewards/margins": 0.20359888672828674,
      "rewards/rejected": -0.5223963260650635,
      "step": 890
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.372847616895685e-08,
      "logits/chosen": 0.09252520650625229,
      "logits/rejected": 0.2231016904115677,
      "logps/chosen": -594.2275390625,
      "logps/rejected": -788.0599975585938,
      "loss": 0.2146,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2893867492675781,
      "rewards/margins": 0.22611062228679657,
      "rewards/rejected": -0.5154973864555359,
      "step": 900
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.264490846553279e-08,
      "logits/chosen": 0.06350111961364746,
      "logits/rejected": 0.20778074860572815,
      "logps/chosen": -707.4957275390625,
      "logps/rejected": -834.1343994140625,
      "loss": 0.2248,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.3360898196697235,
      "rewards/margins": 0.18199169635772705,
      "rewards/rejected": -0.518081545829773,
      "step": 910
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.015418611516165e-09,
      "logits/chosen": 0.08399736881256104,
      "logits/rejected": 0.18887805938720703,
      "logps/chosen": -649.8248291015625,
      "logps/rejected": -780.5101318359375,
      "loss": 0.212,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.2871829569339752,
      "rewards/margins": 0.21405375003814697,
      "rewards/rejected": -0.5012367367744446,
      "step": 920
    },
    {
      "epoch": 0.99,
      "learning_rate": 8.506013354186993e-10,
      "logits/chosen": 0.10902540385723114,
      "logits/rejected": 0.24318072199821472,
      "logps/chosen": -606.8221435546875,
      "logps/rejected": -784.5928955078125,
      "loss": 0.2001,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.284158855676651,
      "rewards/margins": 0.23260316252708435,
      "rewards/rejected": -0.5167620778083801,
      "step": 930
    },
    {
      "epoch": 1.0,
      "step": 937,
      "total_flos": 0.0,
      "train_loss": 0.00839476689712215,
      "train_runtime": 334.0046,
      "train_samples_per_second": 44.91,
      "train_steps_per_second": 2.805
    }
  ],
  "logging_steps": 10,
  "max_steps": 937,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}