{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 1875,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 2.6595744680851065e-08,
      "logits/chosen": 0.09666420519351959,
      "logits/rejected": 0.3057119846343994,
      "logps/chosen": -431.574951171875,
      "logps/rejected": -283.99554443359375,
      "loss": 0.3643,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.6595744680851066e-07,
      "logits/chosen": 0.06902924180030823,
      "logits/rejected": 0.19579637050628662,
      "logps/chosen": -345.7933654785156,
      "logps/rejected": -284.0511779785156,
      "loss": 0.3431,
      "rewards/accuracies": 0.4027777910232544,
      "rewards/chosen": -5.811791925225407e-05,
      "rewards/margins": -0.00014306220691651106,
      "rewards/rejected": 8.494428766425699e-05,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.319148936170213e-07,
      "logits/chosen": 0.07102914154529572,
      "logits/rejected": 0.24960489571094513,
      "logps/chosen": -328.02667236328125,
      "logps/rejected": -273.9169006347656,
      "loss": 0.3699,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -2.8161617592559196e-05,
      "rewards/margins": -6.676381599390879e-05,
      "rewards/rejected": 3.860218566842377e-05,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.97872340425532e-07,
      "logits/chosen": 0.10551998764276505,
      "logits/rejected": 0.2433021515607834,
      "logps/chosen": -354.22650146484375,
      "logps/rejected": -284.6506042480469,
      "loss": 0.3402,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": 0.00011991789506282657,
      "rewards/margins": 2.2664942207484273e-06,
      "rewards/rejected": 0.00011765138333430514,
      "step": 30
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0638297872340427e-06,
      "logits/chosen": 0.12176469713449478,
      "logits/rejected": 0.3288267254829407,
      "logps/chosen": -371.8106384277344,
      "logps/rejected": -293.2636413574219,
      "loss": 0.3463,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.0001882923679659143,
      "rewards/margins": 0.00011801043001469225,
      "rewards/rejected": 7.028193067526445e-05,
      "step": 40
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3297872340425533e-06,
      "logits/chosen": 0.1286633312702179,
      "logits/rejected": 0.13883765041828156,
      "logps/chosen": -318.04205322265625,
      "logps/rejected": -258.83013916015625,
      "loss": 0.3691,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": 0.0002136583934770897,
      "rewards/margins": 6.237901106942445e-05,
      "rewards/rejected": 0.0001512793533038348,
      "step": 50
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.595744680851064e-06,
      "logits/chosen": 0.11209549754858017,
      "logits/rejected": 0.25248101353645325,
      "logps/chosen": -362.73858642578125,
      "logps/rejected": -315.79681396484375,
      "loss": 0.3559,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00025600154185667634,
      "rewards/margins": 0.00027534778928384185,
      "rewards/rejected": -1.9346229237271473e-05,
      "step": 60
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8617021276595745e-06,
      "logits/chosen": 0.11582938581705093,
      "logits/rejected": 0.23768457770347595,
      "logps/chosen": -365.4197998046875,
      "logps/rejected": -307.07269287109375,
      "loss": 0.3196,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0007645456935279071,
      "rewards/margins": 0.000753869186155498,
      "rewards/rejected": 1.0676437341317069e-05,
      "step": 70
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.1276595744680853e-06,
      "logits/chosen": 0.09230409562587738,
      "logits/rejected": 0.23419447243213654,
      "logps/chosen": -309.068603515625,
      "logps/rejected": -264.81463623046875,
      "loss": 0.3248,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0010737198172137141,
      "rewards/margins": 0.0011370908468961716,
      "rewards/rejected": -6.337100057862699e-05,
      "step": 80
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.393617021276596e-06,
      "logits/chosen": 0.10440310090780258,
      "logits/rejected": 0.23364253342151642,
      "logps/chosen": -349.4715881347656,
      "logps/rejected": -306.0603332519531,
      "loss": 0.3274,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0017630963120609522,
      "rewards/margins": 0.0020842214580625296,
      "rewards/rejected": -0.00032112523331306875,
      "step": 90
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6595744680851065e-06,
      "logits/chosen": 0.11755403131246567,
      "logits/rejected": 0.20094576478004456,
      "logps/chosen": -346.9091796875,
      "logps/rejected": -281.81396484375,
      "loss": 0.3318,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.002608776092529297,
      "rewards/margins": 0.0030841627158224583,
      "rewards/rejected": -0.00047538644867017865,
      "step": 100
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.9255319148936174e-06,
      "logits/chosen": 0.0515226311981678,
      "logits/rejected": 0.22483673691749573,
      "logps/chosen": -371.9663391113281,
      "logps/rejected": -313.88739013671875,
      "loss": 0.3365,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.0030859257094562054,
      "rewards/margins": 0.003779615741223097,
      "rewards/rejected": -0.0006936899153515697,
      "step": 110
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.191489361702128e-06,
      "logits/chosen": 0.07034258544445038,
      "logits/rejected": 0.16451817750930786,
      "logps/chosen": -297.6949768066406,
      "logps/rejected": -275.7294616699219,
      "loss": 0.3283,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.0033937774132937193,
      "rewards/margins": 0.004883460234850645,
      "rewards/rejected": -0.0014896828215569258,
      "step": 120
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.457446808510639e-06,
      "logits/chosen": 0.08784133940935135,
      "logits/rejected": 0.23549315333366394,
      "logps/chosen": -326.3392639160156,
      "logps/rejected": -306.0093994140625,
      "loss": 0.3241,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.0068435766734182835,
      "rewards/margins": 0.011338241398334503,
      "rewards/rejected": -0.00449466472491622,
      "step": 130
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.723404255319149e-06,
      "logits/chosen": 0.03845987841486931,
      "logits/rejected": 0.23602671921253204,
      "logps/chosen": -300.4671630859375,
      "logps/rejected": -283.1863708496094,
      "loss": 0.3267,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.007321356330066919,
      "rewards/margins": 0.014485524967312813,
      "rewards/rejected": -0.007164168171584606,
      "step": 140
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.98936170212766e-06,
      "logits/chosen": 0.14375978708267212,
      "logits/rejected": 0.21812143921852112,
      "logps/chosen": -331.6351318359375,
      "logps/rejected": -298.9814453125,
      "loss": 0.3286,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.008922559209167957,
      "rewards/margins": 0.023942312225699425,
      "rewards/rejected": -0.015019752085208893,
      "step": 150
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.255319148936171e-06,
      "logits/chosen": 0.1490752249956131,
      "logits/rejected": 0.27425771951675415,
      "logps/chosen": -365.9402160644531,
      "logps/rejected": -357.152587890625,
      "loss": 0.297,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.005502852611243725,
      "rewards/margins": 0.04779756814241409,
      "rewards/rejected": -0.05330042913556099,
      "step": 160
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.521276595744681e-06,
      "logits/chosen": 0.14248177409172058,
      "logits/rejected": 0.24203705787658691,
      "logps/chosen": -390.8301696777344,
      "logps/rejected": -387.36309814453125,
      "loss": 0.2796,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.05510077625513077,
      "rewards/margins": 0.06211589649319649,
      "rewards/rejected": -0.11721666902303696,
      "step": 170
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.787234042553192e-06,
      "logits/chosen": 0.10283932834863663,
      "logits/rejected": 0.1948317587375641,
      "logps/chosen": -523.36962890625,
      "logps/rejected": -596.3675537109375,
      "loss": 0.2893,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.17172859609127045,
      "rewards/margins": 0.10832329094409943,
      "rewards/rejected": -0.2800518572330475,
      "step": 180
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999982660399688e-06,
      "logits/chosen": 0.13617555797100067,
      "logits/rejected": 0.2513347566127777,
      "logps/chosen": -615.2536010742188,
      "logps/rejected": -724.0734252929688,
      "loss": 0.2269,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.24617168307304382,
      "rewards/margins": 0.17905732989311218,
      "rewards/rejected": -0.425229012966156,
      "step": 190
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.99937579964398e-06,
      "logits/chosen": 0.11373931169509888,
      "logits/rejected": 0.21447698771953583,
      "logps/chosen": -669.2640380859375,
      "logps/rejected": -688.9365844726562,
      "loss": 0.3081,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.27371013164520264,
      "rewards/margins": 0.124347984790802,
      "rewards/rejected": -0.3980581760406494,
      "step": 200
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.9979021993870645e-06,
      "logits/chosen": 0.17223823070526123,
      "logits/rejected": 0.25994282960891724,
      "logps/chosen": -524.0916137695312,
      "logps/rejected": -528.10595703125,
      "loss": 0.3009,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1624583899974823,
      "rewards/margins": 0.07922794669866562,
      "rewards/rejected": -0.2416863739490509,
      "step": 210
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995562370647553e-06,
      "logits/chosen": 0.13899263739585876,
      "logits/rejected": 0.2618511915206909,
      "logps/chosen": -438.03125,
      "logps/rejected": -467.33013916015625,
      "loss": 0.2569,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10613300651311874,
      "rewards/margins": 0.091960109770298,
      "rewards/rejected": -0.19809311628341675,
      "step": 220
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.992357124836838e-06,
      "logits/chosen": 0.13999655842781067,
      "logits/rejected": 0.26916781067848206,
      "logps/chosen": -510.631103515625,
      "logps/rejected": -519.5391845703125,
      "loss": 0.2849,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.14092272520065308,
      "rewards/margins": 0.08064567297697067,
      "rewards/rejected": -0.22156842052936554,
      "step": 230
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9882875734777044e-06,
      "logits/chosen": 0.15458917617797852,
      "logits/rejected": 0.21037283539772034,
      "logps/chosen": -470.0352478027344,
      "logps/rejected": -535.9059448242188,
      "loss": 0.236,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.14089417457580566,
      "rewards/margins": 0.12119980901479721,
      "rewards/rejected": -0.2620939612388611,
      "step": 240
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.983355127818882e-06,
      "logits/chosen": 0.17321957647800446,
      "logits/rejected": 0.3360803425312042,
      "logps/chosen": -578.07177734375,
      "logps/rejected": -678.3271484375,
      "loss": 0.25,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.23716862499713898,
      "rewards/margins": 0.15940351784229279,
      "rewards/rejected": -0.39657214283943176,
      "step": 250
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.977561498345639e-06,
      "logits/chosen": 0.16144976019859314,
      "logits/rejected": 0.2567829489707947,
      "logps/chosen": -631.9638061523438,
      "logps/rejected": -725.3414916992188,
      "loss": 0.2686,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2788200080394745,
      "rewards/margins": 0.14786966145038605,
      "rewards/rejected": -0.42668968439102173,
      "step": 260
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.970908694186624e-06,
      "logits/chosen": 0.16097748279571533,
      "logits/rejected": 0.22262544929981232,
      "logps/chosen": -553.2183837890625,
      "logps/rejected": -640.4689331054688,
      "loss": 0.2476,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.219456285238266,
      "rewards/margins": 0.1420574188232422,
      "rewards/rejected": -0.3615136742591858,
      "step": 270
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9633990224171305e-06,
      "logits/chosen": 0.14573153853416443,
      "logits/rejected": 0.23508696258068085,
      "logps/chosen": -539.7109375,
      "logps/rejected": -651.232421875,
      "loss": 0.2535,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.2373492270708084,
      "rewards/margins": 0.14303575456142426,
      "rewards/rejected": -0.38038498163223267,
      "step": 280
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.955035087259046e-06,
      "logits/chosen": 0.1654827892780304,
      "logits/rejected": 0.2800630033016205,
      "logps/chosen": -603.4498901367188,
      "logps/rejected": -686.7757568359375,
      "loss": 0.2537,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2507944703102112,
      "rewards/margins": 0.16016040742397308,
      "rewards/rejected": -0.41095486283302307,
      "step": 290
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.945819789177756e-06,
      "logits/chosen": 0.11868251860141754,
      "logits/rejected": 0.29606184363365173,
      "logps/chosen": -587.1047973632812,
      "logps/rejected": -650.8994750976562,
      "loss": 0.2541,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.23289060592651367,
      "rewards/margins": 0.13534070551395416,
      "rewards/rejected": -0.36823132634162903,
      "step": 300
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.935756323876306e-06,
      "logits/chosen": 0.11233693361282349,
      "logits/rejected": 0.2625276446342468,
      "logps/chosen": -580.7044677734375,
      "logps/rejected": -654.3244018554688,
      "loss": 0.2557,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.2328997552394867,
      "rewards/margins": 0.13923317193984985,
      "rewards/rejected": -0.37213292717933655,
      "step": 310
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.924848181187199e-06,
      "logits/chosen": 0.14452582597732544,
      "logits/rejected": 0.2437099665403366,
      "logps/chosen": -606.4398803710938,
      "logps/rejected": -706.3465576171875,
      "loss": 0.2552,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.24929067492485046,
      "rewards/margins": 0.1445845663547516,
      "rewards/rejected": -0.39387524127960205,
      "step": 320
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.913099143862173e-06,
      "logits/chosen": 0.09590379893779755,
      "logits/rejected": 0.15467569231987,
      "logps/chosen": -597.2806396484375,
      "logps/rejected": -679.9869384765625,
      "loss": 0.2518,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.23923709988594055,
      "rewards/margins": 0.15891839563846588,
      "rewards/rejected": -0.3981555104255676,
      "step": 330
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.900513286260416e-06,
      "logits/chosen": 0.11927614361047745,
      "logits/rejected": 0.1604013293981552,
      "logps/chosen": -640.5142822265625,
      "logps/rejected": -841.5134887695312,
      "loss": 0.2283,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.3347136974334717,
      "rewards/margins": 0.2090570628643036,
      "rewards/rejected": -0.5437707901000977,
      "step": 340
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.887094972935645e-06,
      "logits/chosen": 0.12204344570636749,
      "logits/rejected": 0.2500421106815338,
      "logps/chosen": -715.3956298828125,
      "logps/rejected": -785.560791015625,
      "loss": 0.2628,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.34101438522338867,
      "rewards/margins": 0.1506863236427307,
      "rewards/rejected": -0.4917007386684418,
      "step": 350
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.87284885712256e-06,
      "logits/chosen": 0.15500317513942719,
      "logits/rejected": 0.29695573449134827,
      "logps/chosen": -633.2789306640625,
      "logps/rejected": -729.1109619140625,
      "loss": 0.2473,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2560732960700989,
      "rewards/margins": 0.17503255605697632,
      "rewards/rejected": -0.4311058521270752,
      "step": 360
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.857779879123181e-06,
      "logits/chosen": 0.0852101594209671,
      "logits/rejected": 0.1848641186952591,
      "logps/chosen": -561.0950927734375,
      "logps/rejected": -703.2201538085938,
      "loss": 0.2509,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.25699612498283386,
      "rewards/margins": 0.1455649882555008,
      "rewards/rejected": -0.40256109833717346,
      "step": 370
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.841893264593643e-06,
      "logits/chosen": 0.12055132538080215,
      "logits/rejected": 0.22479526698589325,
      "logps/chosen": -564.7427978515625,
      "logps/rejected": -672.8660888671875,
      "loss": 0.1755,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2189377099275589,
      "rewards/margins": 0.18233411014080048,
      "rewards/rejected": -0.4012718200683594,
      "step": 380
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.825194522732023e-06,
      "logits/chosen": 0.07654919475317001,
      "logits/rejected": 0.18854263424873352,
      "logps/chosen": -660.2512817382812,
      "logps/rejected": -773.058837890625,
      "loss": 0.2193,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2836855351924896,
      "rewards/margins": 0.19571776688098907,
      "rewards/rejected": -0.4794033169746399,
      "step": 390
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.807689444367853e-06,
      "logits/chosen": 0.09550791233778,
      "logits/rejected": 0.19389700889587402,
      "logps/chosen": -669.1429443359375,
      "logps/rejected": -838.7755126953125,
      "loss": 0.2165,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.31033799052238464,
      "rewards/margins": 0.2328120917081833,
      "rewards/rejected": -0.5431500673294067,
      "step": 400
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.78938409995396e-06,
      "logits/chosen": 0.001080992049537599,
      "logits/rejected": 0.14185592532157898,
      "logps/chosen": -594.972900390625,
      "logps/rejected": -736.0914306640625,
      "loss": 0.2722,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.2792796492576599,
      "rewards/margins": 0.17098276317119598,
      "rewards/rejected": -0.4502624571323395,
      "step": 410
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.770284837461342e-06,
      "logits/chosen": 0.08192798495292664,
      "logits/rejected": 0.138724684715271,
      "logps/chosen": -556.79345703125,
      "logps/rejected": -625.5200805664062,
      "loss": 0.2453,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.21958652138710022,
      "rewards/margins": 0.12120404094457626,
      "rewards/rejected": -0.3407905697822571,
      "step": 420
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7503982801778015e-06,
      "logits/chosen": 0.08842509239912033,
      "logits/rejected": 0.12143252789974213,
      "logps/chosen": -575.1492919921875,
      "logps/rejected": -664.153564453125,
      "loss": 0.2486,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.24815413355827332,
      "rewards/margins": 0.13290396332740784,
      "rewards/rejected": -0.38105809688568115,
      "step": 430
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.729731324411104e-06,
      "logits/chosen": 0.0016645323485136032,
      "logits/rejected": 0.1351255476474762,
      "logps/chosen": -573.5245971679688,
      "logps/rejected": -703.4102783203125,
      "loss": 0.2377,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.24630823731422424,
      "rewards/margins": 0.16149748861789703,
      "rewards/rejected": -0.40780574083328247,
      "step": 440
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.7082911370974645e-06,
      "logits/chosen": 0.04974130541086197,
      "logits/rejected": 0.10218667984008789,
      "logps/chosen": -575.5032958984375,
      "logps/rejected": -679.7342529296875,
      "loss": 0.2552,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.2367144078016281,
      "rewards/margins": 0.15034009516239166,
      "rewards/rejected": -0.38705453276634216,
      "step": 450
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.68608515331618e-06,
      "logits/chosen": 0.03959808871150017,
      "logits/rejected": 0.16889765858650208,
      "logps/chosen": -601.1052856445312,
      "logps/rejected": -735.5942993164062,
      "loss": 0.2556,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.2658396363258362,
      "rewards/margins": 0.17807722091674805,
      "rewards/rejected": -0.44391685724258423,
      "step": 460
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.663121073711269e-06,
      "logits/chosen": 0.060259442776441574,
      "logits/rejected": 0.17572340369224548,
      "logps/chosen": -625.7338256835938,
      "logps/rejected": -786.9043579101562,
      "loss": 0.2364,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.29917049407958984,
      "rewards/margins": 0.1899183690547943,
      "rewards/rejected": -0.48908883333206177,
      "step": 470
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.63940686182103e-06,
      "logits/chosen": 0.05524193122982979,
      "logits/rejected": 0.1667497456073761,
      "logps/chosen": -640.2739868164062,
      "logps/rejected": -716.3453369140625,
      "loss": 0.2418,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.28584688901901245,
      "rewards/margins": 0.1426083743572235,
      "rewards/rejected": -0.42845526337623596,
      "step": 480
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.614950741316425e-06,
      "logits/chosen": 0.05645556375384331,
      "logits/rejected": 0.1303979605436325,
      "logps/chosen": -594.4942016601562,
      "logps/rejected": -682.8673095703125,
      "loss": 0.2491,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.25856301188468933,
      "rewards/margins": 0.15196993947029114,
      "rewards/rejected": -0.41053295135498047,
      "step": 490
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.589761193149254e-06,
      "logits/chosen": 0.07052268087863922,
      "logits/rejected": 0.13902485370635986,
      "logps/chosen": -585.3961181640625,
      "logps/rejected": -691.311767578125,
      "loss": 0.2537,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.2726908326148987,
      "rewards/margins": 0.1488717794418335,
      "rewards/rejected": -0.4215625822544098,
      "step": 500
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.563846952611112e-06,
      "logits/chosen": 0.05198805406689644,
      "logits/rejected": 0.10118089616298676,
      "logps/chosen": -598.3322143554688,
      "logps/rejected": -739.3489990234375,
      "loss": 0.2355,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.25981515645980835,
      "rewards/margins": 0.2083590030670166,
      "rewards/rejected": -0.46817412972450256,
      "step": 510
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.537217006304141e-06,
      "logits/chosen": 0.09554968029260635,
      "logits/rejected": 0.22383460402488708,
      "logps/chosen": -621.1278076171875,
      "logps/rejected": -703.5054931640625,
      "loss": 0.2346,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.2537866532802582,
      "rewards/margins": 0.1655985414981842,
      "rewards/rejected": -0.4193851947784424,
      "step": 520
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.50988058902464e-06,
      "logits/chosen": 0.07675488293170929,
      "logits/rejected": 0.22268608212471008,
      "logps/chosen": -649.4010009765625,
      "logps/rejected": -696.96337890625,
      "loss": 0.2337,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.24650602042675018,
      "rewards/margins": 0.1643296480178833,
      "rewards/rejected": -0.4108356535434723,
      "step": 530
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.481847180560593e-06,
      "logits/chosen": -0.009524239227175713,
      "logits/rejected": 0.13664120435714722,
      "logps/chosen": -608.6438598632812,
      "logps/rejected": -744.18701171875,
      "loss": 0.2511,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.2557665705680847,
      "rewards/margins": 0.1905471682548523,
      "rewards/rejected": -0.4463137090206146,
      "step": 540
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.453126502404253e-06,
      "logits/chosen": 0.04823983460664749,
      "logits/rejected": 0.21495167911052704,
      "logps/chosen": -668.1390991210938,
      "logps/rejected": -764.8587036132812,
      "loss": 0.2063,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.2886661887168884,
      "rewards/margins": 0.20503409206867218,
      "rewards/rejected": -0.4937002658843994,
      "step": 550
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.423728514380892e-06,
      "logits/chosen": 0.14659680426120758,
      "logits/rejected": 0.19975139200687408,
      "logps/chosen": -691.0910034179688,
      "logps/rejected": -818.7584228515625,
      "loss": 0.216,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.33451852202415466,
      "rewards/margins": 0.21299397945404053,
      "rewards/rejected": -0.5475125312805176,
      "step": 560
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.393663411194918e-06,
      "logits/chosen": 0.10968673229217529,
      "logits/rejected": 0.2234468162059784,
      "logps/chosen": -732.4443359375,
      "logps/rejected": -899.6062622070312,
      "loss": 0.2183,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.33378008008003235,
      "rewards/margins": 0.2530228793621063,
      "rewards/rejected": -0.5868029594421387,
      "step": 570
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.362941618894523e-06,
      "logits/chosen": 0.10406269133090973,
      "logits/rejected": 0.15065929293632507,
      "logps/chosen": -659.9013671875,
      "logps/rejected": -766.6561279296875,
      "loss": 0.2346,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.29572516679763794,
      "rewards/margins": 0.1765248328447342,
      "rewards/rejected": -0.47224998474121094,
      "step": 580
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.331573791256116e-06,
      "logits/chosen": 0.0664634108543396,
      "logits/rejected": 0.16555871069431305,
      "logps/chosen": -600.4996948242188,
      "logps/rejected": -713.78466796875,
      "loss": 0.2461,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.2576759457588196,
      "rewards/margins": 0.1549091637134552,
      "rewards/rejected": -0.4125851094722748,
      "step": 590
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.299570806089786e-06,
      "logits/chosen": 0.16024425625801086,
      "logits/rejected": 0.22183863818645477,
      "logps/chosen": -619.4130859375,
      "logps/rejected": -696.956787109375,
      "loss": 0.2457,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.24661366641521454,
      "rewards/margins": 0.1785852015018463,
      "rewards/rejected": -0.42519888281822205,
      "step": 600
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.266943761467057e-06,
      "logits/chosen": 0.07459579408168793,
      "logits/rejected": 0.20276884734630585,
      "logps/chosen": -657.4212646484375,
      "logps/rejected": -731.3056640625,
      "loss": 0.231,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.2757970690727234,
      "rewards/margins": 0.1686132848262787,
      "rewards/rejected": -0.44441038370132446,
      "step": 610
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.233703971872287e-06,
      "logits/chosen": 0.056164585053920746,
      "logits/rejected": 0.22885584831237793,
      "logps/chosen": -652.3689575195312,
      "logps/rejected": -749.9366455078125,
      "loss": 0.227,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.29479923844337463,
      "rewards/margins": 0.16815118491649628,
      "rewards/rejected": -0.4629504680633545,
      "step": 620
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1998629642789925e-06,
      "logits/chosen": 0.10070328414440155,
      "logits/rejected": 0.16475458443164825,
      "logps/chosen": -708.42529296875,
      "logps/rejected": -811.713134765625,
      "loss": 0.2415,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.33783578872680664,
      "rewards/margins": 0.18223688006401062,
      "rewards/rejected": -0.5200726389884949,
      "step": 630
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.165432474152505e-06,
      "logits/chosen": 0.10311365127563477,
      "logits/rejected": 0.15643465518951416,
      "logps/chosen": -583.4866333007812,
      "logps/rejected": -729.7852783203125,
      "loss": 0.2016,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2622414231300354,
      "rewards/margins": 0.19530150294303894,
      "rewards/rejected": -0.45754295587539673,
      "step": 640
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.130424441380308e-06,
      "logits/chosen": 0.10025729984045029,
      "logits/rejected": 0.2146105021238327,
      "logps/chosen": -681.4263916015625,
      "logps/rejected": -823.4031372070312,
      "loss": 0.2163,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3132783770561218,
      "rewards/margins": 0.21751663088798523,
      "rewards/rejected": -0.5307949781417847,
      "step": 650
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.09485100613151e-06,
      "logits/chosen": 0.10832244157791138,
      "logits/rejected": 0.20945067703723907,
      "logps/chosen": -660.5101318359375,
      "logps/rejected": -768.7022705078125,
      "loss": 0.2215,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.30939123034477234,
      "rewards/margins": 0.18488983809947968,
      "rewards/rejected": -0.49428099393844604,
      "step": 660
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.058724504646834e-06,
      "logits/chosen": 0.08686210215091705,
      "logits/rejected": 0.2045850306749344,
      "logps/chosen": -723.7673950195312,
      "logps/rejected": -883.3922119140625,
      "loss": 0.2512,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.38144949078559875,
      "rewards/margins": 0.195582315325737,
      "rewards/rejected": -0.5770317912101746,
      "step": 670
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.022057464960632e-06,
      "logits/chosen": 0.09274314343929291,
      "logits/rejected": 0.1920190155506134,
      "logps/chosen": -618.1563720703125,
      "logps/rejected": -740.8644409179688,
      "loss": 0.2629,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.3001014292240143,
      "rewards/margins": 0.1581810563802719,
      "rewards/rejected": -0.458282470703125,
      "step": 680
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.984862602556383e-06,
      "logits/chosen": 0.021902620792388916,
      "logits/rejected": 0.11925847828388214,
      "logps/chosen": -595.6156005859375,
      "logps/rejected": -711.0296630859375,
      "loss": 0.2626,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.2705914080142975,
      "rewards/margins": 0.16306385397911072,
      "rewards/rejected": -0.4336552619934082,
      "step": 690
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.947152815957187e-06,
      "logits/chosen": 0.05186104774475098,
      "logits/rejected": 0.20855280756950378,
      "logps/chosen": -584.5855102539062,
      "logps/rejected": -713.7994384765625,
      "loss": 0.1958,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.25383633375167847,
      "rewards/margins": 0.1767672598361969,
      "rewards/rejected": -0.43060359358787537,
      "step": 700
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.908941182252785e-06,
      "logits/chosen": 0.022304801270365715,
      "logits/rejected": 0.14363619685173035,
      "logps/chosen": -643.5025634765625,
      "logps/rejected": -748.3091430664062,
      "loss": 0.2237,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.29282766580581665,
      "rewards/margins": 0.17381563782691956,
      "rewards/rejected": -0.4666433334350586,
      "step": 710
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.8702409525646535e-06,
      "logits/chosen": 0.09528463333845139,
      "logits/rejected": 0.18656165897846222,
      "logps/chosen": -692.3035888671875,
      "logps/rejected": -819.2098388671875,
      "loss": 0.2097,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.31982243061065674,
      "rewards/margins": 0.20729240775108337,
      "rewards/rejected": -0.5271148085594177,
      "step": 720
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8310655474507495e-06,
      "logits/chosen": 0.04973004013299942,
      "logits/rejected": 0.19504304230213165,
      "logps/chosen": -688.0816650390625,
      "logps/rejected": -852.9588012695312,
      "loss": 0.2432,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.36149436235427856,
      "rewards/margins": 0.1907968968153,
      "rewards/rejected": -0.5522913336753845,
      "step": 730
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7914285522515002e-06,
      "logits/chosen": 0.09976281225681305,
      "logits/rejected": 0.19117242097854614,
      "logps/chosen": -680.7174072265625,
      "logps/rejected": -797.1549072265625,
      "loss": 0.2341,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.3415672183036804,
      "rewards/margins": 0.1845674216747284,
      "rewards/rejected": -0.5261346101760864,
      "step": 740
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.751343712378639e-06,
      "logits/chosen": 0.07142799347639084,
      "logits/rejected": 0.22169947624206543,
      "logps/chosen": -626.5697021484375,
      "logps/rejected": -787.6502075195312,
      "loss": 0.2383,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2981518507003784,
      "rewards/margins": 0.206342414021492,
      "rewards/rejected": -0.504494309425354,
      "step": 750
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.710824928548546e-06,
      "logits/chosen": 0.1079743355512619,
      "logits/rejected": 0.13973113894462585,
      "logps/chosen": -665.4769287109375,
      "logps/rejected": -744.9660034179688,
      "loss": 0.2612,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.3199656009674072,
      "rewards/margins": 0.16452810168266296,
      "rewards/rejected": -0.4844936728477478,
      "step": 760
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6698862519617225e-06,
      "logits/chosen": 0.075187087059021,
      "logits/rejected": 0.15009218454360962,
      "logps/chosen": -644.90478515625,
      "logps/rejected": -790.0805053710938,
      "loss": 0.2429,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.30303993821144104,
      "rewards/margins": 0.19713596999645233,
      "rewards/rejected": -0.500175952911377,
      "step": 770
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6285418794300793e-06,
      "logits/chosen": 0.06037820130586624,
      "logits/rejected": 0.12935443222522736,
      "logps/chosen": -659.0757446289062,
      "logps/rejected": -834.96533203125,
      "loss": 0.207,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.31061092019081116,
      "rewards/margins": 0.22491443157196045,
      "rewards/rejected": -0.535525381565094,
      "step": 780
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5868061484537365e-06,
      "logits/chosen": 0.062171898782253265,
      "logits/rejected": 0.20288972556591034,
      "logps/chosen": -650.5526733398438,
      "logps/rejected": -812.0040893554688,
      "loss": 0.2295,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3222615122795105,
      "rewards/margins": 0.2239852249622345,
      "rewards/rejected": -0.5462467670440674,
      "step": 790
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5446935322490285e-06,
      "logits/chosen": 0.07936866581439972,
      "logits/rejected": 0.23025266826152802,
      "logps/chosen": -730.1929321289062,
      "logps/rejected": -875.7835693359375,
      "loss": 0.1995,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3494511842727661,
      "rewards/margins": 0.2297430783510208,
      "rewards/rejected": -0.5791942477226257,
      "step": 800
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.502218634729447e-06,
      "logits/chosen": 0.11655671894550323,
      "logits/rejected": 0.1666901409626007,
      "logps/chosen": -654.1530151367188,
      "logps/rejected": -815.29296875,
      "loss": 0.2193,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.30700844526290894,
      "rewards/margins": 0.21489982306957245,
      "rewards/rejected": -0.5219083428382874,
      "step": 810
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.459396185441265e-06,
      "logits/chosen": 0.11936108767986298,
      "logits/rejected": 0.18859058618545532,
      "logps/chosen": -630.1988525390625,
      "logps/rejected": -762.0612182617188,
      "loss": 0.2079,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.25710731744766235,
      "rewards/margins": 0.2074425220489502,
      "rewards/rejected": -0.46454983949661255,
      "step": 820
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4162410344555834e-06,
      "logits/chosen": 0.05629768967628479,
      "logits/rejected": 0.1666196882724762,
      "logps/chosen": -649.73046875,
      "logps/rejected": -818.6950073242188,
      "loss": 0.1792,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.2980211675167084,
      "rewards/margins": 0.23819315433502197,
      "rewards/rejected": -0.5362143516540527,
      "step": 830
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3727681472185937e-06,
      "logits/chosen": 0.04574333503842354,
      "logits/rejected": 0.17197957634925842,
      "logps/chosen": -702.0049438476562,
      "logps/rejected": -867.20947265625,
      "loss": 0.2038,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.35734453797340393,
      "rewards/margins": 0.22116133570671082,
      "rewards/rejected": -0.5785058736801147,
      "step": 840
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3289925993618217e-06,
      "logits/chosen": 0.055036939680576324,
      "logits/rejected": 0.18666990101337433,
      "logps/chosen": -772.3389282226562,
      "logps/rejected": -934.3049926757812,
      "loss": 0.2081,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.3967970013618469,
      "rewards/margins": 0.2379632443189621,
      "rewards/rejected": -0.6347602605819702,
      "step": 850
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2849295714741643e-06,
      "logits/chosen": 0.06958106905221939,
      "logits/rejected": 0.1442635953426361,
      "logps/chosen": -669.5227661132812,
      "logps/rejected": -853.2138671875,
      "loss": 0.207,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3451857566833496,
      "rewards/margins": 0.2383837252855301,
      "rewards/rejected": -0.5835694074630737,
      "step": 860
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2405943438375287e-06,
      "logits/chosen": 0.0513598695397377,
      "logits/rejected": 0.13928762078285217,
      "logps/chosen": -653.1226806640625,
      "logps/rejected": -795.4114379882812,
      "loss": 0.2438,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.327537477016449,
      "rewards/margins": 0.18883965909481049,
      "rewards/rejected": -0.5163770914077759,
      "step": 870
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1960022911279036e-06,
      "logits/chosen": 0.07376402616500854,
      "logits/rejected": 0.17226745188236237,
      "logps/chosen": -704.4319458007812,
      "logps/rejected": -898.2445068359375,
      "loss": 0.1766,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.33583196997642517,
      "rewards/margins": 0.25600650906562805,
      "rewards/rejected": -0.5918384790420532,
      "step": 880
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1511688770836844e-06,
      "logits/chosen": 0.04906494915485382,
      "logits/rejected": 0.1545862853527069,
      "logps/chosen": -670.8369140625,
      "logps/rejected": -803.107421875,
      "loss": 0.2318,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3352956175804138,
      "rewards/margins": 0.2003110647201538,
      "rewards/rejected": -0.5356066823005676,
      "step": 890
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1061096491431307e-06,
      "logits/chosen": 0.12640194594860077,
      "logits/rejected": 0.219802588224411,
      "logps/chosen": -742.6751098632812,
      "logps/rejected": -895.0465698242188,
      "loss": 0.1921,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.3457973599433899,
      "rewards/margins": 0.24558237195014954,
      "rewards/rejected": -0.591379702091217,
      "step": 900
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0608402330527796e-06,
      "logits/chosen": -0.01623944379389286,
      "logits/rejected": 0.16946300864219666,
      "logps/chosen": -649.9053955078125,
      "logps/rejected": -832.1677856445312,
      "loss": 0.1958,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.3138830065727234,
      "rewards/margins": 0.23707744479179382,
      "rewards/rejected": -0.5509604215621948,
      "step": 910
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0153763274487176e-06,
      "logits/chosen": 0.05111056566238403,
      "logits/rejected": 0.14114698767662048,
      "logps/chosen": -662.3326416015625,
      "logps/rejected": -784.4324951171875,
      "loss": 0.2388,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.32007449865341187,
      "rewards/margins": 0.1715754121541977,
      "rewards/rejected": -0.49164995551109314,
      "step": 920
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9697336984125683e-06,
      "logits/chosen": -0.006915999110788107,
      "logits/rejected": 0.11657528579235077,
      "logps/chosen": -666.0967407226562,
      "logps/rejected": -806.9496459960938,
      "loss": 0.2169,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.30281534790992737,
      "rewards/margins": 0.22429080307483673,
      "rewards/rejected": -0.5271061062812805,
      "step": 930
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.923928174004094e-06,
      "logits/chosen": 0.056006837636232376,
      "logits/rejected": 0.17199097573757172,
      "logps/chosen": -652.8565673828125,
      "logps/rejected": -816.8966674804688,
      "loss": 0.2207,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.27928978204727173,
      "rewards/margins": 0.23044073581695557,
      "rewards/rejected": -0.5097305178642273,
      "step": 940
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8779756387723036e-06,
      "logits/chosen": 0.06072944402694702,
      "logits/rejected": 0.13912425935268402,
      "logps/chosen": -622.7828369140625,
      "logps/rejected": -809.6715698242188,
      "loss": 0.2059,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.2813548147678375,
      "rewards/margins": 0.23472395539283752,
      "rewards/rejected": -0.5160787105560303,
      "step": 950
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.831892028246968e-06,
      "logits/chosen": -0.0010059855412691832,
      "logits/rejected": 0.11881496012210846,
      "logps/chosen": -682.4347534179688,
      "logps/rejected": -841.9749755859375,
      "loss": 0.2177,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3291272521018982,
      "rewards/margins": 0.23086945712566376,
      "rewards/rejected": -0.5599966645240784,
      "step": 960
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7856933234124617e-06,
      "logits/chosen": 0.05394923686981201,
      "logits/rejected": 0.16719438135623932,
      "logps/chosen": -690.3303833007812,
      "logps/rejected": -856.0970458984375,
      "loss": 0.2469,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.3471989035606384,
      "rewards/margins": 0.21049991250038147,
      "rewards/rejected": -0.5576988458633423,
      "step": 970
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7393955451658387e-06,
      "logits/chosen": 0.043146662414073944,
      "logits/rejected": 0.08542972058057785,
      "logps/chosen": -734.964599609375,
      "logps/rejected": -885.3410034179688,
      "loss": 0.2057,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3733413815498352,
      "rewards/margins": 0.21346192061901093,
      "rewards/rejected": -0.5868033170700073,
      "step": 980
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6930147487610667e-06,
      "logits/chosen": 0.01986471749842167,
      "logits/rejected": 0.1354392021894455,
      "logps/chosen": -722.3784790039062,
      "logps/rejected": -825.0411376953125,
      "loss": 0.2134,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.33837828040122986,
      "rewards/margins": 0.21411871910095215,
      "rewards/rejected": -0.5524970293045044,
      "step": 990
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6465670182413487e-06,
      "logits/chosen": 0.04280179738998413,
      "logits/rejected": 0.16137579083442688,
      "logps/chosen": -761.4157104492188,
      "logps/rejected": -863.3331909179688,
      "loss": 0.2193,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3615965247154236,
      "rewards/margins": 0.20891360938549042,
      "rewards/rejected": -0.5705101490020752,
      "step": 1000
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6000684608614594e-06,
      "logits/chosen": 0.028211206197738647,
      "logits/rejected": 0.09348297864198685,
      "logps/chosen": -724.6305541992188,
      "logps/rejected": -880.3488159179688,
      "loss": 0.2079,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.36276954412460327,
      "rewards/margins": 0.22737035155296326,
      "rewards/rejected": -0.5901398658752441,
      "step": 1010
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5535352015020338e-06,
      "logits/chosen": 0.05322765186429024,
      "logits/rejected": 0.13077041506767273,
      "logps/chosen": -760.7427978515625,
      "logps/rejected": -912.9437255859375,
      "loss": 0.2429,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.41152581572532654,
      "rewards/margins": 0.21826231479644775,
      "rewards/rejected": -0.6297882199287415,
      "step": 1020
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.506983377077741e-06,
      "logits/chosen": -0.0053452602587640285,
      "logits/rejected": 0.112821564078331,
      "logps/chosen": -674.3397827148438,
      "logps/rejected": -873.40087890625,
      "loss": 0.1812,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.34911829233169556,
      "rewards/margins": 0.24309155344963074,
      "rewards/rejected": -0.5922098159790039,
      "step": 1030
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.460429130941289e-06,
      "logits/chosen": 0.0023634755052626133,
      "logits/rejected": 0.12815037369728088,
      "logps/chosen": -758.9469604492188,
      "logps/rejected": -931.2481689453125,
      "loss": 0.2024,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.37650245428085327,
      "rewards/margins": 0.25763195753097534,
      "rewards/rejected": -0.6341344118118286,
      "step": 1040
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.413888607285192e-06,
      "logits/chosen": 0.035070545971393585,
      "logits/rejected": 0.12803152203559875,
      "logps/chosen": -736.931396484375,
      "logps/rejected": -923.0338745117188,
      "loss": 0.184,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.36943623423576355,
      "rewards/margins": 0.2507568597793579,
      "rewards/rejected": -0.6201931238174438,
      "step": 1050
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.367377945543249e-06,
      "logits/chosen": 5.705729199689813e-05,
      "logits/rejected": 0.1263124793767929,
      "logps/chosen": -706.70068359375,
      "logps/rejected": -814.0146484375,
      "loss": 0.2331,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.3658052086830139,
      "rewards/margins": 0.1808270514011383,
      "rewards/rejected": -0.5466322898864746,
      "step": 1060
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.320913274793676e-06,
      "logits/chosen": 0.015436547808349133,
      "logits/rejected": 0.10843756049871445,
      "logps/chosen": -703.6624755859375,
      "logps/rejected": -858.7509765625,
      "loss": 0.1987,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.31511586904525757,
      "rewards/margins": 0.2355271279811859,
      "rewards/rejected": -0.5506430268287659,
      "step": 1070
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.27451070816582e-06,
      "logits/chosen": 0.012071605771780014,
      "logits/rejected": 0.11224579811096191,
      "logps/chosen": -666.1785888671875,
      "logps/rejected": -801.8255004882812,
      "loss": 0.2259,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.34429460763931274,
      "rewards/margins": 0.19534102082252502,
      "rewards/rejected": -0.5396356582641602,
      "step": 1080
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.228186337252414e-06,
      "logits/chosen": -0.010557098314166069,
      "logits/rejected": 0.06913888454437256,
      "logps/chosen": -734.2570190429688,
      "logps/rejected": -892.15625,
      "loss": 0.2044,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.36991119384765625,
      "rewards/margins": 0.2224087417125702,
      "rewards/rejected": -0.5923199653625488,
      "step": 1090
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1819562265292946e-06,
      "logits/chosen": -0.006013315636664629,
      "logits/rejected": 0.08018209785223007,
      "logps/chosen": -695.9881591796875,
      "logps/rejected": -833.2312622070312,
      "loss": 0.2152,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.36053889989852905,
      "rewards/margins": 0.20316271483898163,
      "rewards/rejected": -0.5637016296386719,
      "step": 1100
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1358364077845236e-06,
      "logits/chosen": -0.008084287866950035,
      "logits/rejected": 0.0774918869137764,
      "logps/chosen": -716.2960815429688,
      "logps/rejected": -869.6041259765625,
      "loss": 0.2309,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3821774125099182,
      "rewards/margins": 0.20834413170814514,
      "rewards/rejected": -0.5905215740203857,
      "step": 1110
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.089842874558849e-06,
      "logits/chosen": 0.007766352035105228,
      "logits/rejected": 0.030095476657152176,
      "logps/chosen": -695.3117065429688,
      "logps/rejected": -871.33837890625,
      "loss": 0.1987,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.3631244897842407,
      "rewards/margins": 0.22518320381641388,
      "rewards/rejected": -0.5883077383041382,
      "step": 1120
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0439915765994242e-06,
      "logits/chosen": -0.024640878662467003,
      "logits/rejected": 0.09226598590612411,
      "logps/chosen": -703.4818115234375,
      "logps/rejected": -877.7662353515625,
      "loss": 0.2056,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.3533559739589691,
      "rewards/margins": 0.22752220928668976,
      "rewards/rejected": -0.5808781385421753,
      "step": 1130
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9982984143287186e-06,
      "logits/chosen": 0.009741904214024544,
      "logits/rejected": 0.17097178101539612,
      "logps/chosen": -810.0487670898438,
      "logps/rejected": -879.8194580078125,
      "loss": 0.1982,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.3632999062538147,
      "rewards/margins": 0.2137196958065033,
      "rewards/rejected": -0.5770195722579956,
      "step": 1140
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.95277923333053e-06,
      "logits/chosen": 0.012245150282979012,
      "logits/rejected": 0.12731657922267914,
      "logps/chosen": -676.3506469726562,
      "logps/rejected": -802.7940063476562,
      "loss": 0.2275,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.33614057302474976,
      "rewards/margins": 0.19641144573688507,
      "rewards/rejected": -0.5325520634651184,
      "step": 1150
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9074498188550156e-06,
      "logits/chosen": 0.02413441613316536,
      "logits/rejected": 0.19431506097316742,
      "logps/chosen": -700.3193359375,
      "logps/rejected": -871.38232421875,
      "loss": 0.1795,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.32141950726509094,
      "rewards/margins": 0.23818500339984894,
      "rewards/rejected": -0.5596045255661011,
      "step": 1160
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.862325890344643e-06,
      "logits/chosen": 0.041377656161785126,
      "logits/rejected": 0.15879663825035095,
      "logps/chosen": -665.6008911132812,
      "logps/rejected": -830.9329223632812,
      "loss": 0.2267,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.342972993850708,
      "rewards/margins": 0.20079226791858673,
      "rewards/rejected": -0.5437651872634888,
      "step": 1170
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.817423095982972e-06,
      "logits/chosen": 0.022486714646220207,
      "logits/rejected": 0.146213561296463,
      "logps/chosen": -672.7999877929688,
      "logps/rejected": -847.8994140625,
      "loss": 0.1819,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.3093092739582062,
      "rewards/margins": 0.2545778453350067,
      "rewards/rejected": -0.5638871192932129,
      "step": 1180
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7727570072681293e-06,
      "logits/chosen": 0.04687700420618057,
      "logits/rejected": 0.12050887197256088,
      "logps/chosen": -672.9434814453125,
      "logps/rejected": -819.170654296875,
      "loss": 0.2352,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.3423709273338318,
      "rewards/margins": 0.20281191170215607,
      "rewards/rejected": -0.5451828241348267,
      "step": 1190
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7283431136128961e-06,
      "logits/chosen": 0.045673031359910965,
      "logits/rejected": 0.11997739970684052,
      "logps/chosen": -611.8385009765625,
      "logps/rejected": -820.4505615234375,
      "loss": 0.2063,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.29818734526634216,
      "rewards/margins": 0.22259333729743958,
      "rewards/rejected": -0.5207806825637817,
      "step": 1200
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6841968169732478e-06,
      "logits/chosen": 0.06439246237277985,
      "logits/rejected": 0.10217195749282837,
      "logps/chosen": -665.1314697265625,
      "logps/rejected": -801.7761840820312,
      "loss": 0.2283,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3135600686073303,
      "rewards/margins": 0.20070858299732208,
      "rewards/rejected": -0.514268696308136,
      "step": 1210
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6403334265072284e-06,
      "logits/chosen": 0.03967912867665291,
      "logits/rejected": 0.1717870980501175,
      "logps/chosen": -683.311767578125,
      "logps/rejected": -844.0208129882812,
      "loss": 0.2371,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3242947459220886,
      "rewards/margins": 0.2034534513950348,
      "rewards/rejected": -0.5277482271194458,
      "step": 1220
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5967681532660066e-06,
      "logits/chosen": 0.013939693570137024,
      "logits/rejected": 0.12744280695915222,
      "logps/chosen": -689.4755859375,
      "logps/rejected": -812.0987548828125,
      "loss": 0.2429,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3416514992713928,
      "rewards/margins": 0.1776062697172165,
      "rewards/rejected": -0.5192577838897705,
      "step": 1230
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5535161049189463e-06,
      "logits/chosen": 0.06095554679632187,
      "logits/rejected": 0.14684705436229706,
      "logps/chosen": -654.1356811523438,
      "logps/rejected": -810.3081665039062,
      "loss": 0.2229,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.3114492893218994,
      "rewards/margins": 0.1974024623632431,
      "rewards/rejected": -0.5088517665863037,
      "step": 1240
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5105922805145356e-06,
      "logits/chosen": 0.05724464729428291,
      "logits/rejected": 0.14580979943275452,
      "logps/chosen": -657.7913818359375,
      "logps/rejected": -799.1402587890625,
      "loss": 0.2055,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.308960497379303,
      "rewards/margins": 0.21290500462055206,
      "rewards/rejected": -0.5218654870986938,
      "step": 1250
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4680115652789823e-06,
      "logits/chosen": 0.04883120581507683,
      "logits/rejected": 0.11742043495178223,
      "logps/chosen": -645.9979248046875,
      "logps/rejected": -819.2106323242188,
      "loss": 0.1943,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.3161103129386902,
      "rewards/margins": 0.21235115826129913,
      "rewards/rejected": -0.5284615159034729,
      "step": 1260
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4257887254542767e-06,
      "logits/chosen": 0.08411203324794769,
      "logits/rejected": 0.14393514394760132,
      "logps/chosen": -695.6365966796875,
      "logps/rejected": -878.0914306640625,
      "loss": 0.2218,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.34891873598098755,
      "rewards/margins": 0.24423813819885254,
      "rewards/rejected": -0.5931568741798401,
      "step": 1270
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3839384031775227e-06,
      "logits/chosen": 0.045291513204574585,
      "logits/rejected": 0.13728824257850647,
      "logps/chosen": -707.1044311523438,
      "logps/rejected": -854.4798583984375,
      "loss": 0.2101,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3448121249675751,
      "rewards/margins": 0.23182924091815948,
      "rewards/rejected": -0.5766413807868958,
      "step": 1280
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.342475111403298e-06,
      "logits/chosen": 0.029178116470575333,
      "logits/rejected": 0.17620958387851715,
      "logps/chosen": -674.8609008789062,
      "logps/rejected": -858.0894775390625,
      "loss": 0.1804,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.3337288498878479,
      "rewards/margins": 0.24131789803504944,
      "rewards/rejected": -0.5750466585159302,
      "step": 1290
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3014132288708209e-06,
      "logits/chosen": 0.038364656269550323,
      "logits/rejected": 0.15578509867191315,
      "logps/chosen": -692.9742431640625,
      "logps/rejected": -857.2974853515625,
      "loss": 0.2094,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3217311501502991,
      "rewards/margins": 0.2241102159023285,
      "rewards/rejected": -0.54584139585495,
      "step": 1300
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2607669951176549e-06,
      "logits/chosen": 0.01667640171945095,
      "logits/rejected": 0.10969080030918121,
      "logps/chosen": -627.9285888671875,
      "logps/rejected": -815.8270263671875,
      "loss": 0.2083,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3187568783760071,
      "rewards/margins": 0.22442683577537537,
      "rewards/rejected": -0.5431837439537048,
      "step": 1310
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2205505055416891e-06,
      "logits/chosen": 0.04231597110629082,
      "logits/rejected": 0.14465740323066711,
      "logps/chosen": -721.646484375,
      "logps/rejected": -825.8064575195312,
      "loss": 0.2264,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3264230787754059,
      "rewards/margins": 0.21109537780284882,
      "rewards/rejected": -0.5375184416770935,
      "step": 1320
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1807777065131002e-06,
      "logits/chosen": 0.0638522133231163,
      "logits/rejected": 0.14387431740760803,
      "logps/chosen": -669.5394287109375,
      "logps/rejected": -838.8292236328125,
      "loss": 0.1997,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.33911409974098206,
      "rewards/margins": 0.22880196571350098,
      "rewards/rejected": -0.5679160356521606,
      "step": 1330
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1414623905380012e-06,
      "logits/chosen": 0.014098724350333214,
      "logits/rejected": 0.13527414202690125,
      "logps/chosen": -669.8994750976562,
      "logps/rejected": -806.6534423828125,
      "loss": 0.2604,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.34625038504600525,
      "rewards/margins": 0.17370744049549103,
      "rewards/rejected": -0.5199578404426575,
      "step": 1340
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1026181914754388e-06,
      "logits/chosen": 0.027355173602700233,
      "logits/rejected": 0.12094994634389877,
      "logps/chosen": -686.2830200195312,
      "logps/rejected": -793.8912353515625,
      "loss": 0.2006,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3198310434818268,
      "rewards/margins": 0.2038339078426361,
      "rewards/rejected": -0.5236650109291077,
      "step": 1350
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0642585798094136e-06,
      "logits/chosen": 0.011197770945727825,
      "logits/rejected": 0.1233413964509964,
      "logps/chosen": -663.2740478515625,
      "logps/rejected": -827.7921142578125,
      "loss": 0.2023,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.31737273931503296,
      "rewards/margins": 0.2423689365386963,
      "rewards/rejected": -0.5597416162490845,
      "step": 1360
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0263968579775522e-06,
      "logits/chosen": 0.012365484610199928,
      "logits/rejected": 0.14399036765098572,
      "logps/chosen": -723.01611328125,
      "logps/rejected": -888.5731201171875,
      "loss": 0.2107,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.3377194106578827,
      "rewards/margins": 0.25036248564720154,
      "rewards/rejected": -0.588081955909729,
      "step": 1370
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.89046155758058e-07,
      "logits/chosen": 0.03630852699279785,
      "logits/rejected": 0.1697433888912201,
      "logps/chosen": -692.0089111328125,
      "logps/rejected": -841.7423706054688,
      "loss": 0.2225,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.33607909083366394,
      "rewards/margins": 0.2240186482667923,
      "rewards/rejected": -0.5600977540016174,
      "step": 1380
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.52219425716534e-07,
      "logits/chosen": 0.06855352222919464,
      "logits/rejected": 0.16431955993175507,
      "logps/chosen": -735.5272216796875,
      "logps/rejected": -859.2053833007812,
      "loss": 0.2056,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3497410714626312,
      "rewards/margins": 0.21472422778606415,
      "rewards/rejected": -0.564465343952179,
      "step": 1390
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.15929438714262e-07,
      "logits/chosen": -0.0006916738930158317,
      "logits/rejected": 0.11941851675510406,
      "logps/chosen": -708.4884643554688,
      "logps/rejected": -896.8206787109375,
      "loss": 0.1853,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.35898059606552124,
      "rewards/margins": 0.240576833486557,
      "rewards/rejected": -0.5995574593544006,
      "step": 1400
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.801887794794911e-07,
      "logits/chosen": 0.039211105555295944,
      "logits/rejected": 0.11498727649450302,
      "logps/chosen": -711.9271850585938,
      "logps/rejected": -853.4754638671875,
      "loss": 0.2018,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3567042946815491,
      "rewards/margins": 0.21128018200397491,
      "rewards/rejected": -0.567984402179718,
      "step": 1410
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.450098422432787e-07,
      "logits/chosen": 0.02443738654255867,
      "logits/rejected": 0.05825776979327202,
      "logps/chosen": -668.7024536132812,
      "logps/rejected": -816.4102172851562,
      "loss": 0.2303,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.332243412733078,
      "rewards/margins": 0.21151785552501678,
      "rewards/rejected": -0.5437612533569336,
      "step": 1420
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.104048264413858e-07,
      "logits/chosen": 0.04351818934082985,
      "logits/rejected": 0.10868921130895615,
      "logps/chosen": -673.7401123046875,
      "logps/rejected": -815.3800659179688,
      "loss": 0.2565,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.35063451528549194,
      "rewards/margins": 0.17919650673866272,
      "rewards/rejected": -0.529831051826477,
      "step": 1430
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.763857324837321e-07,
      "logits/chosen": 0.035469502210617065,
      "logits/rejected": 0.10574381053447723,
      "logps/chosen": -697.9148559570312,
      "logps/rejected": -829.6324462890625,
      "loss": 0.191,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.3119833469390869,
      "rewards/margins": 0.24292249977588654,
      "rewards/rejected": -0.5549058318138123,
      "step": 1440
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.429643575928605e-07,
      "logits/chosen": -0.019414827227592468,
      "logits/rejected": 0.18325455486774445,
      "logps/chosen": -718.9652099609375,
      "logps/rejected": -853.3225708007812,
      "loss": 0.2266,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35121822357177734,
      "rewards/margins": 0.2009017914533615,
      "rewards/rejected": -0.5521200299263,
      "step": 1450
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.101522917128709e-07,
      "logits/chosen": 0.023696620017290115,
      "logits/rejected": 0.10542090982198715,
      "logps/chosen": -635.32373046875,
      "logps/rejected": -819.9827880859375,
      "loss": 0.177,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.30631598830223083,
      "rewards/margins": 0.2354573756456375,
      "rewards/rejected": -0.5417734384536743,
      "step": 1460
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.779609134902312e-07,
      "logits/chosen": 0.0738513395190239,
      "logits/rejected": 0.12634918093681335,
      "logps/chosen": -659.0910034179688,
      "logps/rejected": -865.4376831054688,
      "loss": 0.2054,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.3390515446662903,
      "rewards/margins": 0.23071372509002686,
      "rewards/rejected": -0.5697652697563171,
      "step": 1470
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.464013863278629e-07,
      "logits/chosen": 0.0322534441947937,
      "logits/rejected": 0.1811327189207077,
      "logps/chosen": -716.4608764648438,
      "logps/rejected": -811.6767578125,
      "loss": 0.231,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3598640263080597,
      "rewards/margins": 0.19128559529781342,
      "rewards/rejected": -0.5511496663093567,
      "step": 1480
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.154846545138696e-07,
      "logits/chosen": -0.00073561369208619,
      "logits/rejected": 0.0718742161989212,
      "logps/chosen": -638.3681640625,
      "logps/rejected": -820.8011474609375,
      "loss": 0.2099,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.3302134573459625,
      "rewards/margins": 0.2071600705385208,
      "rewards/rejected": -0.5373734831809998,
      "step": 1490
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.852214394262515e-07,
      "logits/chosen": 0.00898817926645279,
      "logits/rejected": 0.10342226177453995,
      "logps/chosen": -722.0171508789062,
      "logps/rejected": -861.2794189453125,
      "loss": 0.2237,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3676268458366394,
      "rewards/margins": 0.21328619122505188,
      "rewards/rejected": -0.5809130072593689,
      "step": 1500
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.556222358149191e-07,
      "logits/chosen": 0.01620279997587204,
      "logits/rejected": 0.12123896926641464,
      "logps/chosen": -617.8934326171875,
      "logps/rejected": -802.05859375,
      "loss": 0.2168,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.299948513507843,
      "rewards/margins": 0.219092458486557,
      "rewards/rejected": -0.5190409421920776,
      "step": 1510
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.266973081622992e-07,
      "logits/chosen": 0.0352124460041523,
      "logits/rejected": 0.1431238204240799,
      "logps/chosen": -697.825927734375,
      "logps/rejected": -875.0059814453125,
      "loss": 0.2137,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3430352807044983,
      "rewards/margins": 0.21984973549842834,
      "rewards/rejected": -0.562885046005249,
      "step": 1520
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.984566871237942e-07,
      "logits/chosen": 0.04245247319340706,
      "logits/rejected": 0.13526387512683868,
      "logps/chosen": -697.025390625,
      "logps/rejected": -813.5718994140625,
      "loss": 0.2386,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.33000361919403076,
      "rewards/margins": 0.21133120357990265,
      "rewards/rejected": -0.5413348078727722,
      "step": 1530
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.709101660493251e-07,
      "logits/chosen": 0.02729245461523533,
      "logits/rejected": 0.14574703574180603,
      "logps/chosen": -676.8394775390625,
      "logps/rejected": -825.9822998046875,
      "loss": 0.2333,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3324502110481262,
      "rewards/margins": 0.20352134108543396,
      "rewards/rejected": -0.5359715223312378,
      "step": 1540
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.440672975871743e-07,
      "logits/chosen": 0.04532911255955696,
      "logits/rejected": 0.12752528488636017,
      "logps/chosen": -633.0133056640625,
      "logps/rejected": -786.7761840820312,
      "loss": 0.2063,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.29472070932388306,
      "rewards/margins": 0.21680741012096405,
      "rewards/rejected": -0.5115281343460083,
      "step": 1550
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1793739037129134e-07,
      "logits/chosen": 0.07257268577814102,
      "logits/rejected": 0.11432480812072754,
      "logps/chosen": -690.2648315429688,
      "logps/rejected": -863.9197998046875,
      "loss": 0.2096,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.3343331217765808,
      "rewards/margins": 0.23166866600513458,
      "rewards/rejected": -0.5660017728805542,
      "step": 1560
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9252950579322405e-07,
      "logits/chosen": 0.06963463127613068,
      "logits/rejected": 0.12501554191112518,
      "logps/chosen": -634.1485595703125,
      "logps/rejected": -776.090576171875,
      "loss": 0.2422,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.32659441232681274,
      "rewards/margins": 0.19114482402801514,
      "rewards/rejected": -0.5177391767501831,
      "step": 1570
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.6785245485978864e-07,
      "logits/chosen": 0.0035268261563032866,
      "logits/rejected": 0.03672543540596962,
      "logps/chosen": -623.8313598632812,
      "logps/rejected": -851.9779052734375,
      "loss": 0.2306,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3241678476333618,
      "rewards/margins": 0.22523315250873566,
      "rewards/rejected": -0.5494009852409363,
      "step": 1580
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.43914795137566e-07,
      "logits/chosen": 0.03905852884054184,
      "logits/rejected": 0.0981665775179863,
      "logps/chosen": -628.8436889648438,
      "logps/rejected": -775.3961791992188,
      "loss": 0.2274,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.3110875189304352,
      "rewards/margins": 0.2002953737974167,
      "rewards/rejected": -0.5113828778266907,
      "step": 1590
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.207248277852901e-07,
      "logits/chosen": 0.06323795020580292,
      "logits/rejected": 0.1348869502544403,
      "logps/chosen": -645.1129150390625,
      "logps/rejected": -793.5126953125,
      "loss": 0.1864,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3031459450721741,
      "rewards/margins": 0.22209510207176208,
      "rewards/rejected": -0.5252410173416138,
      "step": 1600
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.9829059467515074e-07,
      "logits/chosen": 0.07657162100076675,
      "logits/rejected": 0.11810870468616486,
      "logps/chosen": -679.9815673828125,
      "logps/rejected": -822.6639404296875,
      "loss": 0.2164,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.342891126871109,
      "rewards/margins": 0.2010626345872879,
      "rewards/rejected": -0.5439537167549133,
      "step": 1610
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.766198756040153e-07,
      "logits/chosen": 0.065867580473423,
      "logits/rejected": 0.15143200755119324,
      "logps/chosen": -630.640869140625,
      "logps/rejected": -778.2337036132812,
      "loss": 0.1918,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3079833388328552,
      "rewards/margins": 0.2207847535610199,
      "rewards/rejected": -0.5287681221961975,
      "step": 1620
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5572018559553155e-07,
      "logits/chosen": 0.07310919463634491,
      "logits/rejected": 0.12045961618423462,
      "logps/chosen": -711.3140869140625,
      "logps/rejected": -828.66015625,
      "loss": 0.2413,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.34624183177948,
      "rewards/margins": 0.1804010570049286,
      "rewards/rejected": -0.526642918586731,
      "step": 1630
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3559877229404864e-07,
      "logits/chosen": 0.038972992449998856,
      "logits/rejected": 0.1098276823759079,
      "logps/chosen": -679.78271484375,
      "logps/rejected": -838.59765625,
      "loss": 0.2205,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3377990126609802,
      "rewards/margins": 0.21535663306713104,
      "rewards/rejected": -0.5531556606292725,
      "step": 1640
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1626261345126576e-07,
      "logits/chosen": 0.05230052396655083,
      "logits/rejected": 0.14600692689418793,
      "logps/chosen": -679.2337036132812,
      "logps/rejected": -837.9591674804688,
      "loss": 0.2071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3297780156135559,
      "rewards/margins": 0.22520343959331512,
      "rewards/rejected": -0.5549814701080322,
      "step": 1650
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.9771841450646505e-07,
      "logits/chosen": 0.09260080009698868,
      "logits/rejected": 0.1418459266424179,
      "logps/chosen": -681.7306518554688,
      "logps/rejected": -802.3380126953125,
      "loss": 0.2068,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.31467029452323914,
      "rewards/margins": 0.21341577172279358,
      "rewards/rejected": -0.5280860662460327,
      "step": 1660
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7997260626118758e-07,
      "logits/chosen": 0.01892709732055664,
      "logits/rejected": 0.10615235567092896,
      "logps/chosen": -638.789794921875,
      "logps/rejected": -835.4816284179688,
      "loss": 0.2051,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3251439332962036,
      "rewards/margins": 0.21179835498332977,
      "rewards/rejected": -0.5369422435760498,
      "step": 1670
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6303134264914365e-07,
      "logits/chosen": 0.07935929298400879,
      "logits/rejected": 0.2099631130695343,
      "logps/chosen": -729.1849365234375,
      "logps/rejected": -823.9190673828125,
      "loss": 0.2305,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.32880938053131104,
      "rewards/margins": 0.21041882038116455,
      "rewards/rejected": -0.5392282009124756,
      "step": 1680
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.469004986021355e-07,
      "logits/chosen": 0.056362926959991455,
      "logits/rejected": 0.13600429892539978,
      "logps/chosen": -694.4349975585938,
      "logps/rejected": -821.8929443359375,
      "loss": 0.2202,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3222964107990265,
      "rewards/margins": 0.22275909781455994,
      "rewards/rejected": -0.5450555086135864,
      "step": 1690
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.315856680127367e-07,
      "logits/chosen": 0.07223354279994965,
      "logits/rejected": 0.13364526629447937,
      "logps/chosen": -760.1466674804688,
      "logps/rejected": -892.9208984375,
      "loss": 0.1978,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.37695029377937317,
      "rewards/margins": 0.20259904861450195,
      "rewards/rejected": -0.5795493721961975,
      "step": 1700
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1709216179442817e-07,
      "logits/chosen": 0.004451720509678125,
      "logits/rejected": 0.1476518213748932,
      "logps/chosen": -679.5355224609375,
      "logps/rejected": -841.3785400390625,
      "loss": 0.2127,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3298358917236328,
      "rewards/margins": 0.2245192527770996,
      "rewards/rejected": -0.5543551445007324,
      "step": 1710
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0342500603986421e-07,
      "logits/chosen": 0.008707459084689617,
      "logits/rejected": 0.09261924773454666,
      "logps/chosen": -672.3406982421875,
      "logps/rejected": -848.3216552734375,
      "loss": 0.1905,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3172846734523773,
      "rewards/margins": 0.24588418006896973,
      "rewards/rejected": -0.5631688833236694,
      "step": 1720
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.058894027791643e-08,
      "logits/chosen": 0.0769379660487175,
      "logits/rejected": 0.10007413476705551,
      "logps/chosen": -699.5477294921875,
      "logps/rejected": -884.2034301757812,
      "loss": 0.1939,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.3175678253173828,
      "rewards/margins": 0.25022247433662415,
      "rewards/rejected": -0.5677902698516846,
      "step": 1730
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.858841583008592e-08,
      "logits/chosen": 0.055988211184740067,
      "logits/rejected": 0.1475806087255478,
      "logps/chosen": -652.2313842773438,
      "logps/rejected": -836.2672119140625,
      "loss": 0.1983,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3141983151435852,
      "rewards/margins": 0.24290838837623596,
      "rewards/rejected": -0.5571067333221436,
      "step": 1740
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.742759426686313e-08,
      "logits/chosen": 0.04174142703413963,
      "logits/rejected": 0.14616337418556213,
      "logps/chosen": -674.041015625,
      "logps/rejected": -825.32470703125,
      "loss": 0.1955,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.31099969148635864,
      "rewards/margins": 0.22915931046009064,
      "rewards/rejected": -0.5401589870452881,
      "step": 1750
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.7110345964571104e-08,
      "logits/chosen": 0.057897310703992844,
      "logits/rejected": 0.157648503780365,
      "logps/chosen": -703.7222290039062,
      "logps/rejected": -847.68115234375,
      "loss": 0.2134,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3606089949607849,
      "rewards/margins": 0.21621307730674744,
      "rewards/rejected": -0.5768221616744995,
      "step": 1760
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.764024876318357e-08,
      "logits/chosen": 0.06264643371105194,
      "logits/rejected": 0.14495016634464264,
      "logps/chosen": -730.0210571289062,
      "logps/rejected": -846.86474609375,
      "loss": 0.2213,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.35915127396583557,
      "rewards/margins": 0.19883516430854797,
      "rewards/rejected": -0.5579863786697388,
      "step": 1770
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.902058672559633e-08,
      "logits/chosen": 0.07288487255573273,
      "logits/rejected": 0.1731829047203064,
      "logps/chosen": -698.0350341796875,
      "logps/rejected": -848.1388549804688,
      "loss": 0.1737,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.31695953011512756,
      "rewards/margins": 0.2423754632472992,
      "rewards/rejected": -0.5593350529670715,
      "step": 1780
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.125434899876933e-08,
      "logits/chosen": 0.06433729827404022,
      "logits/rejected": 0.13777321577072144,
      "logps/chosen": -716.5806884765625,
      "logps/rejected": -871.9490356445312,
      "loss": 0.2003,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3185502290725708,
      "rewards/margins": 0.23725280165672302,
      "rewards/rejected": -0.5558030009269714,
      "step": 1790
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4344228777145873e-08,
      "logits/chosen": 0.038981080055236816,
      "logits/rejected": 0.1305188238620758,
      "logps/chosen": -728.6278076171875,
      "logps/rejected": -865.8338012695312,
      "loss": 0.2074,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.34264928102493286,
      "rewards/margins": 0.23139305412769318,
      "rewards/rejected": -0.5740423202514648,
      "step": 1800
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.829262236869772e-08,
      "logits/chosen": 0.06916898488998413,
      "logits/rejected": 0.18797878921031952,
      "logps/chosen": -702.5946044921875,
      "logps/rejected": -872.0211791992188,
      "loss": 0.2173,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.34207233786582947,
      "rewards/margins": 0.2262537032365799,
      "rewards/rejected": -0.5683261156082153,
      "step": 1810
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.3101628363929586e-08,
      "logits/chosen": 0.05317959189414978,
      "logits/rejected": 0.11542042344808578,
      "logps/chosen": -715.3739624023438,
      "logps/rejected": -858.2398681640625,
      "loss": 0.2227,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.34564128518104553,
      "rewards/margins": 0.21440550684928894,
      "rewards/rejected": -0.5600467920303345,
      "step": 1820
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.773046908123195e-09,
      "logits/chosen": 0.07714705169200897,
      "logits/rejected": 0.1786491423845291,
      "logps/chosen": -677.5328979492188,
      "logps/rejected": -854.1397705078125,
      "loss": 0.2085,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.32333892583847046,
      "rewards/margins": 0.22118084132671356,
      "rewards/rejected": -0.5445197820663452,
      "step": 1830
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.308379077080817e-09,
      "logits/chosen": 0.09814608097076416,
      "logits/rejected": 0.09552597999572754,
      "logps/chosen": -678.3449096679688,
      "logps/rejected": -837.9964599609375,
      "loss": 0.2152,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3395656943321228,
      "rewards/margins": 0.21091222763061523,
      "rewards/rejected": -0.5504779815673828,
      "step": 1840
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.7088263565760996e-09,
      "logits/chosen": 0.08845305442810059,
      "logits/rejected": 0.13582393527030945,
      "logps/chosen": -649.8729248046875,
      "logps/rejected": -840.5818481445312,
      "loss": 0.2022,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.32875552773475647,
      "rewards/margins": 0.22628939151763916,
      "rewards/rejected": -0.555044949054718,
      "step": 1850
    },
    {
      "epoch": 0.99,
      "learning_rate": 9.752902257023633e-10,
      "logits/chosen": 0.041667476296424866,
      "logits/rejected": 0.1802738904953003,
      "logps/chosen": -713.4054565429688,
      "logps/rejected": -848.8229370117188,
      "loss": 0.2413,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3549792766571045,
      "rewards/margins": 0.2078593522310257,
      "rewards/rejected": -0.5628384947776794,
      "step": 1860
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.083718442532189e-10,
      "logits/chosen": 0.07584629207849503,
      "logits/rejected": 0.15728022158145905,
      "logps/chosen": -741.0794677734375,
      "logps/rejected": -853.7579956054688,
      "loss": 0.2253,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3356108069419861,
      "rewards/margins": 0.20086529850959778,
      "rewards/rejected": -0.5364760756492615,
      "step": 1870
    },
    {
      "epoch": 1.0,
      "step": 1875,
      "total_flos": 0.0,
      "train_loss": 0.2332912155151367,
      "train_runtime": 17312.5985,
      "train_samples_per_second": 1.733,
      "train_steps_per_second": 0.108
    }
  ],
  "logging_steps": 10,
  "max_steps": 1875,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}