{
  "best_metric": 0.3238605260848999,
  "best_model_checkpoint": "/share/project/zhaolulu/LLama-factory/saves/qwen-7B/full/med_Aquila3_dpo_2e-7_0.03_v1/checkpoint-350",
  "epoch": 1.9553072625698324,
  "eval_steps": 50,
  "global_step": 350,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "grad_norm": 13.13675436076791,
      "learning_rate": 1e-08,
      "logits/chosen": -2.169825315475464,
      "logits/rejected": -2.3517727851867676,
      "logps/chosen": -370.6702575683594,
      "logps/rejected": -186.08995056152344,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "grad_norm": 10.37360079104283,
      "learning_rate": 2e-08,
      "logits/chosen": -2.1483254432678223,
      "logits/rejected": -2.422952890396118,
      "logps/chosen": -364.8175048828125,
      "logps/rejected": -104.55035400390625,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.02,
      "grad_norm": 15.677882625329767,
      "learning_rate": 3e-08,
      "logits/chosen": -2.1419529914855957,
      "logits/rejected": -2.258122682571411,
      "logps/chosen": -282.2867431640625,
      "logps/rejected": -143.6208953857422,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0066271936520934105,
      "rewards/margins": -0.009020809084177017,
      "rewards/rejected": 0.0023936154320836067,
      "step": 3
    },
    {
      "epoch": 0.02,
      "grad_norm": 12.428617731690887,
      "learning_rate": 4e-08,
      "logits/chosen": -2.110610008239746,
      "logits/rejected": -2.4376320838928223,
      "logps/chosen": -369.72955322265625,
      "logps/rejected": -118.06014251708984,
      "loss": 0.6919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.003459886647760868,
      "rewards/margins": 0.0005237694713287055,
      "rewards/rejected": -0.003983656410127878,
      "step": 4
    },
    {
      "epoch": 0.03,
      "grad_norm": 13.025493935888877,
      "learning_rate": 5e-08,
      "logits/chosen": -2.2498154640197754,
      "logits/rejected": -2.409123420715332,
      "logps/chosen": -144.5589599609375,
      "logps/rejected": -99.77885437011719,
      "loss": 0.6914,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006283703725785017,
      "rewards/margins": -0.003716606879606843,
      "rewards/rejected": -0.0025670973118394613,
      "step": 5
    },
    {
      "epoch": 0.03,
      "grad_norm": 11.265681159744263,
      "learning_rate": 6e-08,
      "logits/chosen": -2.1781773567199707,
      "logits/rejected": -2.3208632469177246,
      "logps/chosen": -194.74285888671875,
      "logps/rejected": -97.00983428955078,
      "loss": 0.693,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0076182084158062935,
      "rewards/margins": -0.010748159140348434,
      "rewards/rejected": 0.00312995002605021,
      "step": 6
    },
    {
      "epoch": 0.04,
      "grad_norm": 13.347084629630716,
      "learning_rate": 6.999999999999999e-08,
      "logits/chosen": -2.0979971885681152,
      "logits/rejected": -2.405662775039673,
      "logps/chosen": -313.389404296875,
      "logps/rejected": -66.95777130126953,
      "loss": 0.6945,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.004818655550479889,
      "rewards/margins": -0.0025399725418537855,
      "rewards/rejected": 0.007358627859503031,
      "step": 7
    },
    {
      "epoch": 0.04,
      "grad_norm": 13.350533701450699,
      "learning_rate": 8e-08,
      "logits/chosen": -2.0918493270874023,
      "logits/rejected": -2.3058016300201416,
      "logps/chosen": -431.85345458984375,
      "logps/rejected": -151.113037109375,
      "loss": 0.6919,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002554692327976227,
      "rewards/margins": 0.0028948320541530848,
      "rewards/rejected": -0.00034014007542282343,
      "step": 8
    },
    {
      "epoch": 0.05,
      "grad_norm": 12.289403141621039,
      "learning_rate": 9e-08,
      "logits/chosen": -2.180878162384033,
      "logits/rejected": -2.5295894145965576,
      "logps/chosen": -463.82879638671875,
      "logps/rejected": -202.85638427734375,
      "loss": 0.6933,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.004108843859285116,
      "rewards/margins": 0.004055410623550415,
      "rewards/rejected": 5.343282828107476e-05,
      "step": 9
    },
    {
      "epoch": 0.06,
      "grad_norm": 12.99458204799066,
      "learning_rate": 1e-07,
      "logits/chosen": -2.042452573776245,
      "logits/rejected": -2.2264420986175537,
      "logps/chosen": -298.3495788574219,
      "logps/rejected": -61.18866729736328,
      "loss": 0.6935,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.0035140065010637045,
      "rewards/margins": -0.0031997794285416603,
      "rewards/rejected": 0.0067137861624360085,
      "step": 10
    },
    {
      "epoch": 0.06,
      "grad_norm": 14.055290984256162,
      "learning_rate": 1.1e-07,
      "logits/chosen": -2.081472873687744,
      "logits/rejected": -2.1692185401916504,
      "logps/chosen": -245.54763793945312,
      "logps/rejected": -73.15451049804688,
      "loss": 0.6947,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005330131854861975,
      "rewards/margins": 0.00029961264226585627,
      "rewards/rejected": 0.00503051932901144,
      "step": 11
    },
    {
      "epoch": 0.07,
      "grad_norm": 11.702245206286774,
      "learning_rate": 1.2e-07,
      "logits/chosen": -2.1000940799713135,
      "logits/rejected": -2.440720558166504,
      "logps/chosen": -283.462646484375,
      "logps/rejected": -72.38239288330078,
      "loss": 0.6907,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003204283071681857,
      "rewards/margins": 0.002218131674453616,
      "rewards/rejected": 0.0009861512808129191,
      "step": 12
    },
    {
      "epoch": 0.07,
      "grad_norm": 13.763456208726748,
      "learning_rate": 1.3e-07,
      "logits/chosen": -2.158162832260132,
      "logits/rejected": -2.407245635986328,
      "logps/chosen": -393.5798645019531,
      "logps/rejected": -142.84457397460938,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0038590808399021626,
      "rewards/margins": 0.0019199349917471409,
      "rewards/rejected": 0.0019391458481550217,
      "step": 13
    },
    {
      "epoch": 0.08,
      "grad_norm": 10.270738607964903,
      "learning_rate": 1.3999999999999998e-07,
      "logits/chosen": -2.1504528522491455,
      "logits/rejected": -2.278064250946045,
      "logps/chosen": -228.805908203125,
      "logps/rejected": -110.54583740234375,
      "loss": 0.6939,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0015889224596321583,
      "rewards/margins": -0.0030970978550612926,
      "rewards/rejected": 0.001508174929767847,
      "step": 14
    },
    {
      "epoch": 0.08,
      "grad_norm": 15.508438946473573,
      "learning_rate": 1.5e-07,
      "logits/chosen": -2.2077159881591797,
      "logits/rejected": -2.457319736480713,
      "logps/chosen": -420.102294921875,
      "logps/rejected": -168.63453674316406,
      "loss": 0.6926,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0016078853514045477,
      "rewards/margins": -0.007479728199541569,
      "rewards/rejected": 0.005871844012290239,
      "step": 15
    },
    {
      "epoch": 0.09,
      "grad_norm": 13.409686768799023,
      "learning_rate": 1.6e-07,
      "logits/chosen": -2.0918636322021484,
      "logits/rejected": -2.2362842559814453,
      "logps/chosen": -361.9263916015625,
      "logps/rejected": -204.3945770263672,
      "loss": 0.6885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.01383258868008852,
      "rewards/margins": 0.010981044732034206,
      "rewards/rejected": 0.0028515439480543137,
      "step": 16
    },
    {
      "epoch": 0.09,
      "grad_norm": 16.153489802534672,
      "learning_rate": 1.7e-07,
      "logits/chosen": -2.137455701828003,
      "logits/rejected": -2.4219908714294434,
      "logps/chosen": -340.22601318359375,
      "logps/rejected": -149.10215759277344,
      "loss": 0.6886,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0055633848533034325,
      "rewards/margins": 0.007908429950475693,
      "rewards/rejected": -0.0023450437001883984,
      "step": 17
    },
    {
      "epoch": 0.1,
      "grad_norm": 11.924094161067577,
      "learning_rate": 1.8e-07,
      "logits/chosen": -2.3819501399993896,
      "logits/rejected": -2.3850955963134766,
      "logps/chosen": -257.6195373535156,
      "logps/rejected": -229.90989685058594,
      "loss": 0.6921,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.005492844618856907,
      "rewards/margins": -0.01068685669451952,
      "rewards/rejected": 0.0051940125413239,
      "step": 18
    },
    {
      "epoch": 0.11,
      "grad_norm": 14.629325842518503,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -2.1740121841430664,
      "logits/rejected": -2.4789979457855225,
      "logps/chosen": -357.2406311035156,
      "logps/rejected": -125.6826400756836,
      "loss": 0.6908,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005801108665764332,
      "rewards/margins": 0.0024439399130642414,
      "rewards/rejected": 0.003357168985530734,
      "step": 19
    },
    {
      "epoch": 0.11,
      "grad_norm": 12.509652455926858,
      "learning_rate": 2e-07,
      "logits/chosen": -2.1090314388275146,
      "logits/rejected": -2.3186936378479004,
      "logps/chosen": -437.3341979980469,
      "logps/rejected": -127.853759765625,
      "loss": 0.6904,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00899372436106205,
      "rewards/margins": 0.005673188716173172,
      "rewards/rejected": 0.0033205365762114525,
      "step": 20
    },
    {
      "epoch": 0.12,
      "grad_norm": 9.868342718304051,
      "learning_rate": 1.999956805025437e-07,
      "logits/chosen": -2.0439090728759766,
      "logits/rejected": -2.2628588676452637,
      "logps/chosen": -329.084716796875,
      "logps/rejected": -135.65786743164062,
      "loss": 0.6891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00382827827706933,
      "rewards/margins": -0.00935549009591341,
      "rewards/rejected": 0.005527212284505367,
      "step": 21
    },
    {
      "epoch": 0.12,
      "grad_norm": 15.025885684471918,
      "learning_rate": 1.9998272238333603e-07,
      "logits/chosen": -2.1477479934692383,
      "logits/rejected": -2.210273504257202,
      "logps/chosen": -313.54058837890625,
      "logps/rejected": -290.3620300292969,
      "loss": 0.6825,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.014177151955664158,
      "rewards/margins": 0.014184605330228806,
      "rewards/rejected": -7.453141734004021e-06,
      "step": 22
    },
    {
      "epoch": 0.13,
      "grad_norm": 13.632297626115403,
      "learning_rate": 1.9996112676182827e-07,
      "logits/chosen": -2.164405345916748,
      "logits/rejected": -2.393259286880493,
      "logps/chosen": -378.54864501953125,
      "logps/rejected": -167.57174682617188,
      "loss": 0.681,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0553203821182251,
      "rewards/margins": 0.05659153684973717,
      "rewards/rejected": -0.0012711524032056332,
      "step": 23
    },
    {
      "epoch": 0.13,
      "grad_norm": 13.912168029436412,
      "learning_rate": 1.9993089550366496e-07,
      "logits/chosen": -2.123502016067505,
      "logits/rejected": -2.4436779022216797,
      "logps/chosen": -382.0030822753906,
      "logps/rejected": -100.3667221069336,
      "loss": 0.6823,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04434547945857048,
      "rewards/margins": 0.044589102268218994,
      "rewards/rejected": -0.00024362141266465187,
      "step": 24
    },
    {
      "epoch": 0.14,
      "grad_norm": 11.126485544662991,
      "learning_rate": 1.9989203122052308e-07,
      "logits/chosen": -2.1825976371765137,
      "logits/rejected": -2.349247455596924,
      "logps/chosen": -336.5179443359375,
      "logps/rejected": -156.41336059570312,
      "loss": 0.6825,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.012656629085540771,
      "rewards/margins": 0.01318385824561119,
      "rewards/rejected": -0.0005272297421470284,
      "step": 25
    },
    {
      "epoch": 0.15,
      "grad_norm": 12.881133768022176,
      "learning_rate": 1.9984453726988597e-07,
      "logits/chosen": -2.162121057510376,
      "logits/rejected": -2.342599630355835,
      "logps/chosen": -351.52069091796875,
      "logps/rejected": -105.91226959228516,
      "loss": 0.6763,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.061001330614089966,
      "rewards/margins": 0.07135318219661713,
      "rewards/rejected": -0.010351852513849735,
      "step": 26
    },
    {
      "epoch": 0.15,
      "grad_norm": 11.920957867570188,
      "learning_rate": 1.9978841775475365e-07,
      "logits/chosen": -2.109081745147705,
      "logits/rejected": -2.3539679050445557,
      "logps/chosen": -291.2568664550781,
      "logps/rejected": -106.35763549804688,
      "loss": 0.6765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.040990542620420456,
      "rewards/margins": 0.0412316657602787,
      "rewards/rejected": -0.0002411194145679474,
      "step": 27
    },
    {
      "epoch": 0.16,
      "grad_norm": 14.475145205567406,
      "learning_rate": 1.9972367752328823e-07,
      "logits/chosen": -2.0869147777557373,
      "logits/rejected": -2.354045867919922,
      "logps/chosen": -456.2073669433594,
      "logps/rejected": -169.84324645996094,
      "loss": 0.6687,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.055279482156038284,
      "rewards/margins": 0.05866669863462448,
      "rewards/rejected": -0.0033872141502797604,
      "step": 28
    },
    {
      "epoch": 0.16,
      "grad_norm": 12.532121666063755,
      "learning_rate": 1.996503221683949e-07,
      "logits/chosen": -2.0160179138183594,
      "logits/rejected": -2.3588764667510986,
      "logps/chosen": -390.41290283203125,
      "logps/rejected": -133.29124450683594,
      "loss": 0.6729,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.038773227483034134,
      "rewards/margins": 0.042331479489803314,
      "rewards/rejected": -0.0035582496784627438,
      "step": 29
    },
    {
      "epoch": 0.17,
      "grad_norm": 13.419020536226967,
      "learning_rate": 1.9956835802723916e-07,
      "logits/chosen": -2.209843873977661,
      "logits/rejected": -2.4064574241638184,
      "logps/chosen": -405.7186279296875,
      "logps/rejected": -167.26657104492188,
      "loss": 0.6741,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05138365924358368,
      "rewards/margins": 0.04216768592596054,
      "rewards/rejected": 0.009215973317623138,
      "step": 30
    },
    {
      "epoch": 0.17,
      "grad_norm": 10.86765582737344,
      "learning_rate": 1.9947779218069888e-07,
      "logits/chosen": -2.1761481761932373,
      "logits/rejected": -2.1804559230804443,
      "logps/chosen": -173.52847290039062,
      "logps/rejected": -113.20588684082031,
      "loss": 0.6715,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04976405203342438,
      "rewards/margins": 0.04497087001800537,
      "rewards/rejected": 0.004793173633515835,
      "step": 31
    },
    {
      "epoch": 0.18,
      "grad_norm": 11.502969061621119,
      "learning_rate": 1.9937863245275302e-07,
      "logits/chosen": -2.07767915725708,
      "logits/rejected": -2.260467052459717,
      "logps/chosen": -349.59149169921875,
      "logps/rejected": -194.00537109375,
      "loss": 0.6609,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.054776448756456375,
      "rewards/margins": 0.057643141597509384,
      "rewards/rejected": -0.002866692841053009,
      "step": 32
    },
    {
      "epoch": 0.18,
      "grad_norm": 12.857022449597613,
      "learning_rate": 1.9927088740980536e-07,
      "logits/chosen": -2.1136598587036133,
      "logits/rejected": -2.4379379749298096,
      "logps/chosen": -431.517822265625,
      "logps/rejected": -155.318603515625,
      "loss": 0.6523,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09774884581565857,
      "rewards/margins": 0.09220996499061584,
      "rewards/rejected": 0.00553888501599431,
      "step": 33
    },
    {
      "epoch": 0.19,
      "grad_norm": 11.458981035263896,
      "learning_rate": 1.991545663599448e-07,
      "logits/chosen": -2.1781678199768066,
      "logits/rejected": -2.398679494857788,
      "logps/chosen": -308.6724548339844,
      "logps/rejected": -104.43415069580078,
      "loss": 0.6517,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11533545702695847,
      "rewards/margins": 0.11893679946660995,
      "rewards/rejected": -0.003601343370974064,
      "step": 34
    },
    {
      "epoch": 0.2,
      "grad_norm": 10.628647934096392,
      "learning_rate": 1.990296793521408e-07,
      "logits/chosen": -1.9812856912612915,
      "logits/rejected": -2.246541738510132,
      "logps/chosen": -308.601806640625,
      "logps/rejected": -106.56471252441406,
      "loss": 0.6504,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1276446282863617,
      "rewards/margins": 0.12667663395404816,
      "rewards/rejected": 0.0009679919457994401,
      "step": 35
    },
    {
      "epoch": 0.2,
      "grad_norm": 9.989241900876351,
      "learning_rate": 1.9889623717537563e-07,
      "logits/chosen": -2.0774359703063965,
      "logits/rejected": -2.271974802017212,
      "logps/chosen": -310.50506591796875,
      "logps/rejected": -144.49606323242188,
      "loss": 0.6491,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1192081868648529,
      "rewards/margins": 0.11847703158855438,
      "rewards/rejected": 0.0007311587687581778,
      "step": 36
    },
    {
      "epoch": 0.21,
      "grad_norm": 12.048092750050337,
      "learning_rate": 1.9875425135771217e-07,
      "logits/chosen": -2.0446596145629883,
      "logits/rejected": -2.348402738571167,
      "logps/chosen": -317.9515075683594,
      "logps/rejected": -67.82337951660156,
      "loss": 0.64,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11614331603050232,
      "rewards/margins": 0.116356760263443,
      "rewards/rejected": -0.00021345657296478748,
      "step": 37
    },
    {
      "epoch": 0.21,
      "grad_norm": 9.41812256063119,
      "learning_rate": 1.9860373416529801e-07,
      "logits/chosen": -2.1628098487854004,
      "logits/rejected": -2.5593316555023193,
      "logps/chosen": -365.1604309082031,
      "logps/rejected": -121.07073974609375,
      "loss": 0.6505,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09028297662734985,
      "rewards/margins": 0.0999036654829979,
      "rewards/rejected": -0.009620685130357742,
      "step": 38
    },
    {
      "epoch": 0.22,
      "grad_norm": 10.66925837158948,
      "learning_rate": 1.9844469860130572e-07,
      "logits/chosen": -2.1404829025268555,
      "logits/rejected": -2.406595468521118,
      "logps/chosen": -387.9655456542969,
      "logps/rejected": -159.52334594726562,
      "loss": 0.6426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13599559664726257,
      "rewards/margins": 0.12349364906549454,
      "rewards/rejected": 0.012501951307058334,
      "step": 39
    },
    {
      "epoch": 0.22,
      "grad_norm": 8.314112135425724,
      "learning_rate": 1.9827715840480958e-07,
      "logits/chosen": -2.1365151405334473,
      "logits/rejected": -2.225376605987549,
      "logps/chosen": -170.3944091796875,
      "logps/rejected": -76.3488540649414,
      "loss": 0.6576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0686836838722229,
      "rewards/margins": 0.08776295185089111,
      "rewards/rejected": -0.019079256802797318,
      "step": 40
    },
    {
      "epoch": 0.23,
      "grad_norm": 10.050564274359012,
      "learning_rate": 1.9810112804959865e-07,
      "logits/chosen": -2.1081488132476807,
      "logits/rejected": -2.4641435146331787,
      "logps/chosen": -432.20794677734375,
      "logps/rejected": -157.08131408691406,
      "loss": 0.6394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1458844095468521,
      "rewards/margins": 0.15924030542373657,
      "rewards/rejected": -0.013355905190110207,
      "step": 41
    },
    {
      "epoch": 0.23,
      "grad_norm": 10.532623517273787,
      "learning_rate": 1.9791662274292634e-07,
      "logits/chosen": -2.1743876934051514,
      "logits/rejected": -2.4913887977600098,
      "logps/chosen": -322.1396484375,
      "logps/rejected": -71.1148681640625,
      "loss": 0.636,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15435950458049774,
      "rewards/margins": 0.1565544456243515,
      "rewards/rejected": -0.002194945001974702,
      "step": 42
    },
    {
      "epoch": 0.24,
      "grad_norm": 11.51412373772502,
      "learning_rate": 1.9772365842419675e-07,
      "logits/chosen": -1.9813337326049805,
      "logits/rejected": -2.4765429496765137,
      "logps/chosen": -403.84197998046875,
      "logps/rejected": -107.24089050292969,
      "loss": 0.6274,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12991374731063843,
      "rewards/margins": 0.14337915182113647,
      "rewards/rejected": -0.01346538309007883,
      "step": 43
    },
    {
      "epoch": 0.25,
      "grad_norm": 13.245694792329783,
      "learning_rate": 1.9752225176358755e-07,
      "logits/chosen": -2.0392189025878906,
      "logits/rejected": -2.2406013011932373,
      "logps/chosen": -236.0980682373047,
      "logps/rejected": -62.11045455932617,
      "loss": 0.6376,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08092227578163147,
      "rewards/margins": 0.082358218729496,
      "rewards/rejected": -0.001435947371646762,
      "step": 44
    },
    {
      "epoch": 0.25,
      "grad_norm": 11.762023866110251,
      "learning_rate": 1.9731242016060984e-07,
      "logits/chosen": -2.055626153945923,
      "logits/rejected": -2.515719175338745,
      "logps/chosen": -431.6874694824219,
      "logps/rejected": -88.8346176147461,
      "loss": 0.6234,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17793263494968414,
      "rewards/margins": 0.18582700192928314,
      "rewards/rejected": -0.007894383743405342,
      "step": 45
    },
    {
      "epoch": 0.26,
      "grad_norm": 8.28929712626817,
      "learning_rate": 1.970941817426052e-07,
      "logits/chosen": -1.9879474639892578,
      "logits/rejected": -2.3104357719421387,
      "logps/chosen": -211.4118194580078,
      "logps/rejected": -30.24435043334961,
      "loss": 0.6419,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07762061804533005,
      "rewards/margins": 0.08949979394674301,
      "rewards/rejected": -0.01187918335199356,
      "step": 46
    },
    {
      "epoch": 0.26,
      "grad_norm": 8.941394429559,
      "learning_rate": 1.9686755536317942e-07,
      "logits/chosen": -1.982191562652588,
      "logits/rejected": -2.0665199756622314,
      "logps/chosen": -139.63328552246094,
      "logps/rejected": -49.644195556640625,
      "loss": 0.6371,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0728769525885582,
      "rewards/margins": 0.07204664498567581,
      "rewards/rejected": 0.0008303160429932177,
      "step": 47
    },
    {
      "epoch": 0.27,
      "grad_norm": 10.284161862377836,
      "learning_rate": 1.9663256060057392e-07,
      "logits/chosen": -2.2932796478271484,
      "logits/rejected": -2.4093167781829834,
      "logps/chosen": -330.951416015625,
      "logps/rejected": -176.99734497070312,
      "loss": 0.6306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13722312450408936,
      "rewards/margins": 0.1075534000992775,
      "rewards/rejected": 0.02966972626745701,
      "step": 48
    },
    {
      "epoch": 0.27,
      "grad_norm": 10.587404352257455,
      "learning_rate": 1.9638921775597422e-07,
      "logits/chosen": -1.95592200756073,
      "logits/rejected": -2.303375482559204,
      "logps/chosen": -498.4439697265625,
      "logps/rejected": -218.24583435058594,
      "loss": 0.6245,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15323582291603088,
      "rewards/margins": 0.1492384374141693,
      "rewards/rejected": 0.00399737898260355,
      "step": 49
    },
    {
      "epoch": 0.28,
      "grad_norm": 10.723511661626334,
      "learning_rate": 1.9613754785175638e-07,
      "logits/chosen": -2.0556998252868652,
      "logits/rejected": -2.219463586807251,
      "logps/chosen": -224.0782012939453,
      "logps/rejected": -103.49295806884766,
      "loss": 0.616,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08105921745300293,
      "rewards/margins": 0.06892888247966766,
      "rewards/rejected": 0.01213033776730299,
      "step": 50
    },
    {
      "epoch": 0.28,
      "eval_logits/chosen": -2.063699722290039,
      "eval_logits/rejected": -2.314316987991333,
      "eval_logps/chosen": -331.750244140625,
      "eval_logps/rejected": -116.3936538696289,
      "eval_loss": 0.6081522703170776,
      "eval_rewards/accuracies": 0.875,
      "eval_rewards/chosen": 0.1798781156539917,
      "eval_rewards/margins": 0.19427311420440674,
      "eval_rewards/rejected": -0.014395004138350487,
      "eval_runtime": 186.2235,
      "eval_samples_per_second": 6.836,
      "eval_steps_per_second": 0.859,
      "step": 50
    },
    {
      "epoch": 0.28,
      "grad_norm": 10.09005570072562,
      "learning_rate": 1.9587757262967054e-07,
      "logits/chosen": -2.181702136993408,
      "logits/rejected": -2.3205502033233643,
      "logps/chosen": -341.22662353515625,
      "logps/rejected": -122.17086791992188,
      "loss": 0.5977,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24404488503932953,
      "rewards/margins": 0.24747334420681,
      "rewards/rejected": -0.0034284451976418495,
      "step": 51
    },
    {
      "epoch": 0.29,
      "grad_norm": 8.81518303930105,
      "learning_rate": 1.9560931454896297e-07,
      "logits/chosen": -2.1127026081085205,
      "logits/rejected": -2.3931002616882324,
      "logps/chosen": -210.6508026123047,
      "logps/rejected": -48.36506652832031,
      "loss": 0.5966,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15659283101558685,
      "rewards/margins": 0.1508672684431076,
      "rewards/rejected": 0.00572555884718895,
      "step": 52
    },
    {
      "epoch": 0.3,
      "grad_norm": 9.208140684891914,
      "learning_rate": 1.9533279678443557e-07,
      "logits/chosen": -2.1042847633361816,
      "logits/rejected": -2.1663877964019775,
      "logps/chosen": -252.5467987060547,
      "logps/rejected": -164.3186492919922,
      "loss": 0.5877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.128390371799469,
      "rewards/margins": 0.13332779705524445,
      "rewards/rejected": -0.004937426187098026,
      "step": 53
    },
    {
      "epoch": 0.3,
      "grad_norm": 7.509168689756416,
      "learning_rate": 1.95048043224444e-07,
      "logits/chosen": -2.0268847942352295,
      "logits/rejected": -2.2786049842834473,
      "logps/chosen": -317.0534973144531,
      "logps/rejected": -171.33995056152344,
      "loss": 0.5985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21348123252391815,
      "rewards/margins": 0.20053228735923767,
      "rewards/rejected": 0.012948956340551376,
      "step": 54
    },
    {
      "epoch": 0.31,
      "grad_norm": 8.171851534420124,
      "learning_rate": 1.9475507846883373e-07,
      "logits/chosen": -2.0638015270233154,
      "logits/rejected": -2.220968723297119,
      "logps/chosen": -384.039794921875,
      "logps/rejected": -282.49658203125,
      "loss": 0.5687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24091266095638275,
      "rewards/margins": 0.24727410078048706,
      "rewards/rejected": -0.0063614556565880775,
      "step": 55
    },
    {
      "epoch": 0.31,
      "grad_norm": 8.56365071461843,
      "learning_rate": 1.944539278268152e-07,
      "logits/chosen": -1.9981962442398071,
      "logits/rejected": -2.3010456562042236,
      "logps/chosen": -343.3883361816406,
      "logps/rejected": -97.42654418945312,
      "loss": 0.5552,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24590067565441132,
      "rewards/margins": 0.2593224346637726,
      "rewards/rejected": -0.013421745970845222,
      "step": 56
    },
    {
      "epoch": 0.32,
      "grad_norm": 7.892289862886692,
      "learning_rate": 1.94144617314777e-07,
      "logits/chosen": -2.0351200103759766,
      "logits/rejected": -2.301356792449951,
      "logps/chosen": -372.68609619140625,
      "logps/rejected": -174.614013671875,
      "loss": 0.5728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2827102541923523,
      "rewards/margins": 0.2959131598472595,
      "rewards/rejected": -0.013202919624745846,
      "step": 57
    },
    {
      "epoch": 0.32,
      "grad_norm": 7.3420110886371885,
      "learning_rate": 1.938271736540385e-07,
      "logits/chosen": -2.204134464263916,
      "logits/rejected": -2.3048620223999023,
      "logps/chosen": -346.56146240234375,
      "logps/rejected": -271.0859375,
      "loss": 0.573,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22849659621715546,
      "rewards/margins": 0.20856241881847382,
      "rewards/rejected": 0.01993417926132679,
      "step": 58
    },
    {
      "epoch": 0.33,
      "grad_norm": 6.00666118956556,
      "learning_rate": 1.935016242685415e-07,
      "logits/chosen": -2.0446014404296875,
      "logits/rejected": -2.142392635345459,
      "logps/chosen": -238.8208770751953,
      "logps/rejected": -118.88178253173828,
      "loss": 0.5918,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.22309349477291107,
      "rewards/margins": 0.1636679619550705,
      "rewards/rejected": 0.05942551791667938,
      "step": 59
    },
    {
      "epoch": 0.34,
      "grad_norm": 7.656028095081895,
      "learning_rate": 1.931679972824807e-07,
      "logits/chosen": -2.0129806995391846,
      "logits/rejected": -2.261230230331421,
      "logps/chosen": -305.9075927734375,
      "logps/rejected": -101.50321197509766,
      "loss": 0.5517,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.25226718187332153,
      "rewards/margins": 0.2614496052265167,
      "rewards/rejected": -0.00918243546038866,
      "step": 60
    },
    {
      "epoch": 0.34,
      "grad_norm": 12.171466631603687,
      "learning_rate": 1.928263215178746e-07,
      "logits/chosen": -1.9546587467193604,
      "logits/rejected": -2.1961617469787598,
      "logps/chosen": -279.5615539550781,
      "logps/rejected": -115.06743621826172,
      "loss": 0.5543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17554552853107452,
      "rewards/margins": 0.18559010326862335,
      "rewards/rejected": -0.010044600814580917,
      "step": 61
    },
    {
      "epoch": 0.35,
      "grad_norm": 6.610218016041575,
      "learning_rate": 1.9247662649207506e-07,
      "logits/chosen": -2.1121904850006104,
      "logits/rejected": -2.3557066917419434,
      "logps/chosen": -336.6860046386719,
      "logps/rejected": -159.52703857421875,
      "loss": 0.5681,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.32165953516960144,
      "rewards/margins": 0.3067389726638794,
      "rewards/rejected": 0.014920572750270367,
      "step": 62
    },
    {
      "epoch": 0.35,
      "grad_norm": 7.941038400658689,
      "learning_rate": 1.9211894241521757e-07,
      "logits/chosen": -2.0181961059570312,
      "logits/rejected": -2.23909592628479,
      "logps/chosen": -240.93853759765625,
      "logps/rejected": -104.80455780029297,
      "loss": 0.5661,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19158579409122467,
      "rewards/margins": 0.2334558218717575,
      "rewards/rejected": -0.041870009154081345,
      "step": 63
    },
    {
      "epoch": 0.36,
      "grad_norm": 6.351871394862074,
      "learning_rate": 1.917533001876113e-07,
      "logits/chosen": -2.025303840637207,
      "logits/rejected": -2.2152905464172363,
      "logps/chosen": -288.76513671875,
      "logps/rejected": -166.4485321044922,
      "loss": 0.5722,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18773403763771057,
      "rewards/margins": 0.14920204877853394,
      "rewards/rejected": 0.038532011210918427,
      "step": 64
    },
    {
      "epoch": 0.36,
      "grad_norm": 6.998604694555862,
      "learning_rate": 1.913797313970697e-07,
      "logits/chosen": -1.9997467994689941,
      "logits/rejected": -2.1551594734191895,
      "logps/chosen": -210.7803955078125,
      "logps/rejected": -86.51407623291016,
      "loss": 0.5568,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1796235293149948,
      "rewards/margins": 0.19531936943531036,
      "rewards/rejected": -0.0156958419829607,
      "step": 65
    },
    {
      "epoch": 0.37,
      "grad_norm": 6.577421777934189,
      "learning_rate": 1.9099826831618167e-07,
      "logits/chosen": -2.1465611457824707,
      "logits/rejected": -2.345120906829834,
      "logps/chosen": -377.3128662109375,
      "logps/rejected": -223.61959838867188,
      "loss": 0.5599,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39910346269607544,
      "rewards/margins": 0.4542591869831085,
      "rewards/rejected": -0.055155716836452484,
      "step": 66
    },
    {
      "epoch": 0.37,
      "grad_norm": 6.2250131829631865,
      "learning_rate": 1.9060894389952328e-07,
      "logits/chosen": -2.024362802505493,
      "logits/rejected": -2.0789616107940674,
      "logps/chosen": -199.9416961669922,
      "logps/rejected": -98.576416015625,
      "loss": 0.5584,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2663131654262543,
      "rewards/margins": 0.2556077837944031,
      "rewards/rejected": 0.010705405846238136,
      "step": 67
    },
    {
      "epoch": 0.38,
      "grad_norm": 7.189726097166299,
      "learning_rate": 1.9021179178081103e-07,
      "logits/chosen": -2.0059540271759033,
      "logits/rejected": -2.2304420471191406,
      "logps/chosen": -380.8197326660156,
      "logps/rejected": -169.16343688964844,
      "loss": 0.543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4966823160648346,
      "rewards/margins": 0.5529358386993408,
      "rewards/rejected": -0.05625356361269951,
      "step": 68
    },
    {
      "epoch": 0.39,
      "grad_norm": 6.249636636104176,
      "learning_rate": 1.8980684626999638e-07,
      "logits/chosen": -2.033315896987915,
      "logits/rejected": -2.303514242172241,
      "logps/chosen": -260.1401672363281,
      "logps/rejected": -92.7634506225586,
      "loss": 0.5591,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2530212700366974,
      "rewards/margins": 0.29110509157180786,
      "rewards/rejected": -0.03808382526040077,
      "step": 69
    },
    {
      "epoch": 0.39,
      "grad_norm": 6.437391045028375,
      "learning_rate": 1.8939414235030132e-07,
      "logits/chosen": -1.8933184146881104,
      "logits/rejected": -2.15160870552063,
      "logps/chosen": -208.13137817382812,
      "logps/rejected": -48.8433952331543,
      "loss": 0.5606,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22771571576595306,
      "rewards/margins": 0.2676602900028229,
      "rewards/rejected": -0.0399446040391922,
      "step": 70
    },
    {
      "epoch": 0.4,
      "grad_norm": 7.0182488830117675,
      "learning_rate": 1.889737156751965e-07,
      "logits/chosen": -1.9940942525863647,
      "logits/rejected": -2.2075793743133545,
      "logps/chosen": -337.55322265625,
      "logps/rejected": -134.0843505859375,
      "loss": 0.5322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3136310279369354,
      "rewards/margins": 0.3688744008541107,
      "rewards/rejected": -0.0552433617413044,
      "step": 71
    },
    {
      "epoch": 0.4,
      "grad_norm": 6.875925492294041,
      "learning_rate": 1.8854560256532097e-07,
      "logits/chosen": -2.1553783416748047,
      "logits/rejected": -2.242018938064575,
      "logps/chosen": -217.90322875976562,
      "logps/rejected": -135.96380615234375,
      "loss": 0.5281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2514539659023285,
      "rewards/margins": 0.24648477137088776,
      "rewards/rejected": 0.004969220608472824,
      "step": 72
    },
    {
      "epoch": 0.41,
      "grad_norm": 6.420556289383338,
      "learning_rate": 1.8810984000534455e-07,
      "logits/chosen": -2.171513319015503,
      "logits/rejected": -2.4142439365386963,
      "logps/chosen": -372.5497741699219,
      "logps/rejected": -107.42914581298828,
      "loss": 0.5392,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3514203429222107,
      "rewards/margins": 0.3896327614784241,
      "rewards/rejected": -0.03821243345737457,
      "step": 73
    },
    {
      "epoch": 0.41,
      "grad_norm": 6.375186295982661,
      "learning_rate": 1.8766646564077262e-07,
      "logits/chosen": -1.966844081878662,
      "logits/rejected": -2.2431657314300537,
      "logps/chosen": -319.42822265625,
      "logps/rejected": -84.1899185180664,
      "loss": 0.5415,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3488491177558899,
      "rewards/margins": 0.4105728566646576,
      "rewards/rejected": -0.06172379106283188,
      "step": 74
    },
    {
      "epoch": 0.42,
      "grad_norm": 5.838393126841468,
      "learning_rate": 1.8721551777469395e-07,
      "logits/chosen": -2.1016697883605957,
      "logits/rejected": -2.328932762145996,
      "logps/chosen": -336.9084777832031,
      "logps/rejected": -195.07691955566406,
      "loss": 0.5424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3171999156475067,
      "rewards/margins": 0.34964245557785034,
      "rewards/rejected": -0.03244255110621452,
      "step": 75
    },
    {
      "epoch": 0.42,
      "grad_norm": 6.149492969231708,
      "learning_rate": 1.8675703536447177e-07,
      "logits/chosen": -2.071046829223633,
      "logits/rejected": -2.1905810832977295,
      "logps/chosen": -261.36724853515625,
      "logps/rejected": -151.13186645507812,
      "loss": 0.5439,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2710087299346924,
      "rewards/margins": 0.234308123588562,
      "rewards/rejected": 0.03670059144496918,
      "step": 76
    },
    {
      "epoch": 0.43,
      "grad_norm": 6.008863437719162,
      "learning_rate": 1.8629105801837816e-07,
      "logits/chosen": -2.0235931873321533,
      "logits/rejected": -2.2131049633026123,
      "logps/chosen": -299.67791748046875,
      "logps/rejected": -113.56851196289062,
      "loss": 0.5466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16502772271633148,
      "rewards/margins": 0.21942166984081268,
      "rewards/rejected": -0.0543939545750618,
      "step": 77
    },
    {
      "epoch": 0.44,
      "grad_norm": 5.378526341274013,
      "learning_rate": 1.8581762599217238e-07,
      "logits/chosen": -1.9962934255599976,
      "logits/rejected": -2.1097159385681152,
      "logps/chosen": -151.64990234375,
      "logps/rejected": -54.187625885009766,
      "loss": 0.5518,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16601191461086273,
      "rewards/margins": 0.19980715215206146,
      "rewards/rejected": -0.03379523381590843,
      "step": 78
    },
    {
      "epoch": 0.44,
      "grad_norm": 6.104689013268376,
      "learning_rate": 1.8533678018562307e-07,
      "logits/chosen": -1.9824721813201904,
      "logits/rejected": -2.10068678855896,
      "logps/chosen": -222.87936401367188,
      "logps/rejected": -121.28485870361328,
      "loss": 0.5419,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19363561272621155,
      "rewards/margins": 0.1619020700454712,
      "rewards/rejected": 0.031733546406030655,
      "step": 79
    },
    {
      "epoch": 0.45,
      "grad_norm": 5.874550794444151,
      "learning_rate": 1.8484856213897496e-07,
      "logits/chosen": -2.178605079650879,
      "logits/rejected": -2.4736976623535156,
      "logps/chosen": -378.0633544921875,
      "logps/rejected": -142.17327880859375,
      "loss": 0.5344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.556702196598053,
      "rewards/margins": 0.6439647674560547,
      "rewards/rejected": -0.08726256340742111,
      "step": 80
    },
    {
      "epoch": 0.45,
      "grad_norm": 5.569584190243061,
      "learning_rate": 1.843530140293603e-07,
      "logits/chosen": -2.123403787612915,
      "logits/rejected": -2.3839967250823975,
      "logps/chosen": -307.99310302734375,
      "logps/rejected": -94.0133285522461,
      "loss": 0.5344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4116560220718384,
      "rewards/margins": 0.46545499563217163,
      "rewards/rejected": -0.053798969835042953,
      "step": 81
    },
    {
      "epoch": 0.46,
      "grad_norm": 6.280242196576865,
      "learning_rate": 1.8385017866715505e-07,
      "logits/chosen": -1.9856494665145874,
      "logits/rejected": -2.2972702980041504,
      "logps/chosen": -420.359619140625,
      "logps/rejected": -149.61878967285156,
      "loss": 0.5357,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41265326738357544,
      "rewards/margins": 0.4080853760242462,
      "rewards/rejected": 0.00456790579482913,
      "step": 82
    },
    {
      "epoch": 0.46,
      "grad_norm": 6.313182239745635,
      "learning_rate": 1.833400994922806e-07,
      "logits/chosen": -1.9181156158447266,
      "logits/rejected": -2.2001190185546875,
      "logps/chosen": -341.0047302246094,
      "logps/rejected": -157.48458862304688,
      "loss": 0.5293,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3060517907142639,
      "rewards/margins": 0.2681809067726135,
      "rewards/rejected": 0.037870921194553375,
      "step": 83
    },
    {
      "epoch": 0.47,
      "grad_norm": 6.27246004101099,
      "learning_rate": 1.8282282057045086e-07,
      "logits/chosen": -2.0791337490081787,
      "logits/rejected": -2.3352248668670654,
      "logps/chosen": -395.2298278808594,
      "logps/rejected": -159.21627807617188,
      "loss": 0.5273,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3726465106010437,
      "rewards/margins": 0.3795578181743622,
      "rewards/rejected": -0.006911378353834152,
      "step": 84
    },
    {
      "epoch": 0.47,
      "grad_norm": 5.5921705697506665,
      "learning_rate": 1.8229838658936564e-07,
      "logits/chosen": -2.108044385910034,
      "logits/rejected": -2.3148159980773926,
      "logps/chosen": -296.7893981933594,
      "logps/rejected": -119.67386627197266,
      "loss": 0.5438,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4559445381164551,
      "rewards/margins": 0.5166485905647278,
      "rewards/rejected": -0.06070411577820778,
      "step": 85
    },
    {
      "epoch": 0.48,
      "grad_norm": 6.011168548073458,
      "learning_rate": 1.8176684285484982e-07,
      "logits/chosen": -2.0125370025634766,
      "logits/rejected": -2.4413373470306396,
      "logps/chosen": -489.0357360839844,
      "logps/rejected": -143.520751953125,
      "loss": 0.5276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46227729320526123,
      "rewards/margins": 0.5473066568374634,
      "rewards/rejected": -0.08502937108278275,
      "step": 86
    },
    {
      "epoch": 0.49,
      "grad_norm": 6.544525494627247,
      "learning_rate": 1.8122823528693966e-07,
      "logits/chosen": -2.0247550010681152,
      "logits/rejected": -2.2515745162963867,
      "logps/chosen": -409.3858947753906,
      "logps/rejected": -185.4442596435547,
      "loss": 0.506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.37112706899642944,
      "rewards/margins": 0.4832947850227356,
      "rewards/rejected": -0.11216770112514496,
      "step": 87
    },
    {
      "epoch": 0.49,
      "grad_norm": 5.225956930747786,
      "learning_rate": 1.8068261041591546e-07,
      "logits/chosen": -1.9530370235443115,
      "logits/rejected": -2.0214812755584717,
      "logps/chosen": -193.56881713867188,
      "logps/rejected": -113.12052154541016,
      "loss": 0.5437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1621086150407791,
      "rewards/margins": 0.21262691915035248,
      "rewards/rejected": -0.05051829293370247,
      "step": 88
    },
    {
      "epoch": 0.5,
      "grad_norm": 6.362015290793359,
      "learning_rate": 1.8013001537828212e-07,
      "logits/chosen": -1.9773004055023193,
      "logits/rejected": -2.4064035415649414,
      "logps/chosen": -473.7355651855469,
      "logps/rejected": -108.94012451171875,
      "loss": 0.4837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7071879506111145,
      "rewards/margins": 0.8128292560577393,
      "rewards/rejected": -0.10564135015010834,
      "step": 89
    },
    {
      "epoch": 0.5,
      "grad_norm": 5.418254357850609,
      "learning_rate": 1.7957049791269684e-07,
      "logits/chosen": -1.9675734043121338,
      "logits/rejected": -2.1122231483459473,
      "logps/chosen": -188.61734008789062,
      "logps/rejected": -89.61772155761719,
      "loss": 0.511,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26492881774902344,
      "rewards/margins": 0.3512020409107208,
      "rewards/rejected": -0.08627324551343918,
      "step": 90
    },
    {
      "epoch": 0.51,
      "grad_norm": 5.872257218178851,
      "learning_rate": 1.7900410635584497e-07,
      "logits/chosen": -2.0316319465637207,
      "logits/rejected": -2.253246784210205,
      "logps/chosen": -251.0762176513672,
      "logps/rejected": -61.86017990112305,
      "loss": 0.5069,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29612407088279724,
      "rewards/margins": 0.37898051738739014,
      "rewards/rejected": -0.0828564465045929,
      "step": 91
    },
    {
      "epoch": 0.51,
      "grad_norm": 5.652130957766253,
      "learning_rate": 1.7843088963826433e-07,
      "logits/chosen": -2.0624449253082275,
      "logits/rejected": -2.2764549255371094,
      "logps/chosen": -370.9137268066406,
      "logps/rejected": -250.69602966308594,
      "loss": 0.5014,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4017767012119293,
      "rewards/margins": 0.43597471714019775,
      "rewards/rejected": -0.034198030829429626,
      "step": 92
    },
    {
      "epoch": 0.52,
      "grad_norm": 5.102972695442928,
      "learning_rate": 1.7785089728011794e-07,
      "logits/chosen": -1.9869798421859741,
      "logits/rejected": -2.2077856063842773,
      "logps/chosen": -236.51177978515625,
      "logps/rejected": -61.656253814697266,
      "loss": 0.4948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3908085227012634,
      "rewards/margins": 0.5067812204360962,
      "rewards/rejected": -0.11597264558076859,
      "step": 93
    },
    {
      "epoch": 0.53,
      "grad_norm": 5.370818540058152,
      "learning_rate": 1.7726417938691619e-07,
      "logits/chosen": -1.967153787612915,
      "logits/rejected": -2.141418933868408,
      "logps/chosen": -231.7236328125,
      "logps/rejected": -116.03726196289062,
      "loss": 0.4724,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22416304051876068,
      "rewards/margins": 0.2950986623764038,
      "rewards/rejected": -0.07093562185764313,
      "step": 94
    },
    {
      "epoch": 0.53,
      "grad_norm": 5.189949899066027,
      "learning_rate": 1.7667078664518793e-07,
      "logits/chosen": -2.074751853942871,
      "logits/rejected": -2.0922069549560547,
      "logps/chosen": -173.9927978515625,
      "logps/rejected": -123.99789428710938,
      "loss": 0.4758,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22283044457435608,
      "rewards/margins": 0.32154157757759094,
      "rewards/rejected": -0.09871112555265427,
      "step": 95
    },
    {
      "epoch": 0.54,
      "grad_norm": 5.085022408643686,
      "learning_rate": 1.76070770318102e-07,
      "logits/chosen": -2.1210179328918457,
      "logits/rejected": -2.4016335010528564,
      "logps/chosen": -315.16082763671875,
      "logps/rejected": -154.37620544433594,
      "loss": 0.5037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5534517765045166,
      "rewards/margins": 0.6325445175170898,
      "rewards/rejected": -0.07909276336431503,
      "step": 96
    },
    {
      "epoch": 0.54,
      "grad_norm": 4.8993009503155225,
      "learning_rate": 1.7546418224103835e-07,
      "logits/chosen": -1.9710700511932373,
      "logits/rejected": -2.1767797470092773,
      "logps/chosen": -309.2714538574219,
      "logps/rejected": -161.23086547851562,
      "loss": 0.4713,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.45312583446502686,
      "rewards/margins": 0.6297840476036072,
      "rewards/rejected": -0.1766582429409027,
      "step": 97
    },
    {
      "epoch": 0.55,
      "grad_norm": 4.374782063627258,
      "learning_rate": 1.748510748171101e-07,
      "logits/chosen": -1.9851081371307373,
      "logits/rejected": -2.043236494064331,
      "logps/chosen": -135.60049438476562,
      "logps/rejected": -68.2064437866211,
      "loss": 0.481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18936334550380707,
      "rewards/margins": 0.26729437708854675,
      "rewards/rejected": -0.07793103158473969,
      "step": 98
    },
    {
      "epoch": 0.55,
      "grad_norm": 4.392980085406151,
      "learning_rate": 1.7423150101263642e-07,
      "logits/chosen": -2.0348260402679443,
      "logits/rejected": -2.1746668815612793,
      "logps/chosen": -214.05282592773438,
      "logps/rejected": -100.34738159179688,
      "loss": 0.4838,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2590511441230774,
      "rewards/margins": 0.48322322964668274,
      "rewards/rejected": -0.22417207062244415,
      "step": 99
    },
    {
      "epoch": 0.56,
      "grad_norm": 4.821003899193818,
      "learning_rate": 1.7360551435256673e-07,
      "logits/chosen": -2.0046215057373047,
      "logits/rejected": -2.2741754055023193,
      "logps/chosen": -451.2037353515625,
      "logps/rejected": -181.55972290039062,
      "loss": 0.4623,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.43423500657081604,
      "rewards/margins": 0.6116418838500977,
      "rewards/rejected": -0.17740684747695923,
      "step": 100
    },
    {
      "epoch": 0.56,
      "eval_logits/chosen": -1.9775569438934326,
      "eval_logits/rejected": -2.2316997051239014,
      "eval_logps/chosen": -320.51092529296875,
      "eval_logps/rejected": -122.61784362792969,
      "eval_loss": 0.4467245638370514,
      "eval_rewards/accuracies": 0.918749988079071,
      "eval_rewards/chosen": 0.517056405544281,
      "eval_rewards/margins": 0.7181770205497742,
      "eval_rewards/rejected": -0.20112057030200958,
      "eval_runtime": 186.4668,
      "eval_samples_per_second": 6.827,
      "eval_steps_per_second": 0.858,
      "step": 100
    },
    {
      "epoch": 0.56,
      "grad_norm": 4.15265430176453,
      "learning_rate": 1.7297316891585675e-07,
      "logits/chosen": -1.9079805612564087,
      "logits/rejected": -2.1030519008636475,
      "logps/chosen": -268.8660888671875,
      "logps/rejected": -115.76235961914062,
      "loss": 0.482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4167216420173645,
      "rewards/margins": 0.5568986535072327,
      "rewards/rejected": -0.14017705619335175,
      "step": 101
    },
    {
      "epoch": 0.57,
      "grad_norm": 4.0504682508727035,
      "learning_rate": 1.723345193307966e-07,
      "logits/chosen": -2.1078760623931885,
      "logits/rejected": -2.110827922821045,
      "logps/chosen": -220.6822967529297,
      "logps/rejected": -237.88125610351562,
      "loss": 0.4896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.31573280692100525,
      "rewards/margins": 0.257499635219574,
      "rewards/rejected": 0.058233220130205154,
      "step": 102
    },
    {
      "epoch": 0.58,
      "grad_norm": 4.073084534105835,
      "learning_rate": 1.7168962077029147e-07,
      "logits/chosen": -2.0834879875183105,
      "logits/rejected": -2.268869400024414,
      "logps/chosen": -368.4338684082031,
      "logps/rejected": -146.4982147216797,
      "loss": 0.4469,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6909863948822021,
      "rewards/margins": 0.9143600463867188,
      "rewards/rejected": -0.2233736366033554,
      "step": 103
    },
    {
      "epoch": 0.58,
      "grad_norm": 4.37978318257449,
      "learning_rate": 1.7103852894709516e-07,
      "logits/chosen": -1.9810847043991089,
      "logits/rejected": -2.2168128490448,
      "logps/chosen": -315.2268981933594,
      "logps/rejected": -112.0322494506836,
      "loss": 0.4335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5689502358436584,
      "rewards/margins": 0.8497422933578491,
      "rewards/rejected": -0.2807920277118683,
      "step": 104
    },
    {
      "epoch": 0.59,
      "grad_norm": 3.889756281915941,
      "learning_rate": 1.7038130010899715e-07,
      "logits/chosen": -1.9811582565307617,
      "logits/rejected": -2.092210292816162,
      "logps/chosen": -289.2423095703125,
      "logps/rejected": -178.71807861328125,
      "loss": 0.4823,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3411446213722229,
      "rewards/margins": 0.4420354962348938,
      "rewards/rejected": -0.10089084506034851,
      "step": 105
    },
    {
      "epoch": 0.59,
      "grad_norm": 4.020472458246672,
      "learning_rate": 1.6971799103396333e-07,
      "logits/chosen": -1.9673709869384766,
      "logits/rejected": -2.158069372177124,
      "logps/chosen": -243.9493408203125,
      "logps/rejected": -94.94026947021484,
      "loss": 0.4691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5125857591629028,
      "rewards/margins": 0.6839098930358887,
      "rewards/rejected": -0.1713240146636963,
      "step": 106
    },
    {
      "epoch": 0.6,
      "grad_norm": 3.7354805196661727,
      "learning_rate": 1.6904865902523097e-07,
      "logits/chosen": -1.9753053188323975,
      "logits/rejected": -2.1016111373901367,
      "logps/chosen": -220.32826232910156,
      "logps/rejected": -112.85079193115234,
      "loss": 0.4533,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.31250494718551636,
      "rewards/margins": 0.6757313013076782,
      "rewards/rejected": -0.36322641372680664,
      "step": 107
    },
    {
      "epoch": 0.6,
      "grad_norm": 3.9245064384013704,
      "learning_rate": 1.6837336190635822e-07,
      "logits/chosen": -2.059147357940674,
      "logits/rejected": -2.1772873401641846,
      "logps/chosen": -324.65899658203125,
      "logps/rejected": -223.98416137695312,
      "loss": 0.4925,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.38603684306144714,
      "rewards/margins": 0.4854813516139984,
      "rewards/rejected": -0.09944455325603485,
      "step": 108
    },
    {
      "epoch": 0.61,
      "grad_norm": 4.239398591437667,
      "learning_rate": 1.6769215801622881e-07,
      "logits/chosen": -1.8967094421386719,
      "logits/rejected": -2.27612566947937,
      "logps/chosen": -385.6495666503906,
      "logps/rejected": -123.2912826538086,
      "loss": 0.4126,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5274486541748047,
      "rewards/margins": 0.695229172706604,
      "rewards/rejected": -0.1677805632352829,
      "step": 109
    },
    {
      "epoch": 0.61,
      "grad_norm": 3.4050976316519455,
      "learning_rate": 1.6700510620401222e-07,
      "logits/chosen": -1.9731296300888062,
      "logits/rejected": -2.123116970062256,
      "logps/chosen": -230.08668518066406,
      "logps/rejected": -102.91403198242188,
      "loss": 0.4687,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.45512545108795166,
      "rewards/margins": 0.5183148980140686,
      "rewards/rejected": -0.06318947672843933,
      "step": 110
    },
    {
      "epoch": 0.62,
      "grad_norm": 3.722592336469093,
      "learning_rate": 1.6631226582407951e-07,
      "logits/chosen": -1.9364876747131348,
      "logits/rejected": -2.2068028450012207,
      "logps/chosen": -327.61737060546875,
      "logps/rejected": -150.93841552734375,
      "loss": 0.4683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.530191957950592,
      "rewards/margins": 0.799619197845459,
      "rewards/rejected": -0.26942718029022217,
      "step": 111
    },
    {
      "epoch": 0.63,
      "grad_norm": 3.9447811103480306,
      "learning_rate": 1.6561369673087585e-07,
      "logits/chosen": -1.9829171895980835,
      "logits/rejected": -2.2110044956207275,
      "logps/chosen": -412.00341796875,
      "logps/rejected": -145.0166778564453,
      "loss": 0.38,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8012962937355042,
      "rewards/margins": 1.1100391149520874,
      "rewards/rejected": -0.3087427020072937,
      "step": 112
    },
    {
      "epoch": 0.63,
      "grad_norm": 4.484517066813305,
      "learning_rate": 1.6490945927374968e-07,
      "logits/chosen": -1.9895436763763428,
      "logits/rejected": -2.232736587524414,
      "logps/chosen": -420.51300048828125,
      "logps/rejected": -231.12796020507812,
      "loss": 0.3833,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6339386105537415,
      "rewards/margins": 0.8945336937904358,
      "rewards/rejected": -0.26059508323669434,
      "step": 113
    },
    {
      "epoch": 0.64,
      "grad_norm": 3.8362793915513818,
      "learning_rate": 1.641996142917391e-07,
      "logits/chosen": -2.0275871753692627,
      "logits/rejected": -2.1543703079223633,
      "logps/chosen": -336.6465759277344,
      "logps/rejected": -169.4688262939453,
      "loss": 0.4334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5943139791488647,
      "rewards/margins": 0.9754756093025208,
      "rewards/rejected": -0.3811616003513336,
      "step": 114
    },
    {
      "epoch": 0.64,
      "grad_norm": 4.288478315089954,
      "learning_rate": 1.6348422310831595e-07,
      "logits/chosen": -2.013012170791626,
      "logits/rejected": -2.29827618598938,
      "logps/chosen": -322.6732177734375,
      "logps/rejected": -139.569091796875,
      "loss": 0.4019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5506273508071899,
      "rewards/margins": 0.8705551624298096,
      "rewards/rejected": -0.31992778182029724,
      "step": 115
    },
    {
      "epoch": 0.65,
      "grad_norm": 2.932854863653483,
      "learning_rate": 1.627633475260882e-07,
      "logits/chosen": -1.8620115518569946,
      "logits/rejected": -2.178684711456299,
      "logps/chosen": -432.8457946777344,
      "logps/rejected": -142.8162841796875,
      "loss": 0.4383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7144624590873718,
      "rewards/margins": 1.1024161577224731,
      "rewards/rejected": -0.38795360922813416,
      "step": 116
    },
    {
      "epoch": 0.65,
      "grad_norm": 3.5258705477452312,
      "learning_rate": 1.620370498214607e-07,
      "logits/chosen": -1.9928277730941772,
      "logits/rejected": -2.294769525527954,
      "logps/chosen": -327.63983154296875,
      "logps/rejected": -107.83856201171875,
      "loss": 0.4785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5440024137496948,
      "rewards/margins": 0.7528430223464966,
      "rewards/rejected": -0.20884062349796295,
      "step": 117
    },
    {
      "epoch": 0.66,
      "grad_norm": 2.9842534931020355,
      "learning_rate": 1.6130539273925528e-07,
      "logits/chosen": -2.0068583488464355,
      "logits/rejected": -2.042741060256958,
      "logps/chosen": -169.5665740966797,
      "logps/rejected": -145.59188842773438,
      "loss": 0.4719,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.28490278124809265,
      "rewards/margins": 0.25545331835746765,
      "rewards/rejected": 0.029449433088302612,
      "step": 118
    },
    {
      "epoch": 0.66,
      "grad_norm": 3.2633771102193707,
      "learning_rate": 1.6056843948728997e-07,
      "logits/chosen": -2.04612135887146,
      "logits/rejected": -2.1641111373901367,
      "logps/chosen": -304.9841003417969,
      "logps/rejected": -259.2249755859375,
      "loss": 0.4496,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4365397095680237,
      "rewards/margins": 0.5547477006912231,
      "rewards/rejected": -0.11820797622203827,
      "step": 119
    },
    {
      "epoch": 0.67,
      "grad_norm": 3.2661747416858713,
      "learning_rate": 1.5982625373091874e-07,
      "logits/chosen": -1.9853219985961914,
      "logits/rejected": -2.247044801712036,
      "logps/chosen": -274.5014953613281,
      "logps/rejected": -98.39961242675781,
      "loss": 0.4412,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3941829204559326,
      "rewards/margins": 0.7147579193115234,
      "rewards/rejected": -0.3205750286579132,
      "step": 120
    },
    {
      "epoch": 0.68,
      "grad_norm": 3.6997185994891217,
      "learning_rate": 1.5907889958753132e-07,
      "logits/chosen": -1.9539928436279297,
      "logits/rejected": -2.259643316268921,
      "logps/chosen": -360.8001708984375,
      "logps/rejected": -139.05746459960938,
      "loss": 0.4045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6151716113090515,
      "rewards/margins": 0.8953421115875244,
      "rewards/rejected": -0.2801705002784729,
      "step": 121
    },
    {
      "epoch": 0.68,
      "grad_norm": 3.26180309012713,
      "learning_rate": 1.5832644162101417e-07,
      "logits/chosen": -1.9639915227890015,
      "logits/rejected": -2.3155434131622314,
      "logps/chosen": -341.48138427734375,
      "logps/rejected": -142.3020782470703,
      "loss": 0.4608,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6940339207649231,
      "rewards/margins": 1.0427274703979492,
      "rewards/rejected": -0.34869351983070374,
      "step": 122
    },
    {
      "epoch": 0.69,
      "grad_norm": 3.475146283236811,
      "learning_rate": 1.5756894483617267e-07,
      "logits/chosen": -2.0319931507110596,
      "logits/rejected": -2.0845324993133545,
      "logps/chosen": -108.24813079833984,
      "logps/rejected": -56.015464782714844,
      "loss": 0.4253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.175735205411911,
      "rewards/margins": 0.34109964966773987,
      "rewards/rejected": -0.16536441445350647,
      "step": 123
    },
    {
      "epoch": 0.69,
      "grad_norm": 3.711332096082553,
      "learning_rate": 1.5680647467311557e-07,
      "logits/chosen": -2.0805673599243164,
      "logits/rejected": -2.3789658546447754,
      "logps/chosen": -322.76593017578125,
      "logps/rejected": -113.76570129394531,
      "loss": 0.4062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6527657508850098,
      "rewards/margins": 0.8439643383026123,
      "rewards/rejected": -0.19119860231876373,
      "step": 124
    },
    {
      "epoch": 0.7,
      "grad_norm": 3.565726315114208,
      "learning_rate": 1.560390970016015e-07,
      "logits/chosen": -2.0362613201141357,
      "logits/rejected": -2.2976279258728027,
      "logps/chosen": -359.05126953125,
      "logps/rejected": -207.07476806640625,
      "loss": 0.42,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6665505766868591,
      "rewards/margins": 1.1819754838943481,
      "rewards/rejected": -0.515424907207489,
      "step": 125
    },
    {
      "epoch": 0.7,
      "grad_norm": 3.6197726640789805,
      "learning_rate": 1.5526687811534838e-07,
      "logits/chosen": -1.9589186906814575,
      "logits/rejected": -2.149359703063965,
      "logps/chosen": -383.7507629394531,
      "logps/rejected": -239.794677734375,
      "loss": 0.3565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5040831565856934,
      "rewards/margins": 0.9846323728561401,
      "rewards/rejected": -0.48054924607276917,
      "step": 126
    },
    {
      "epoch": 0.71,
      "grad_norm": 3.4172376150457113,
      "learning_rate": 1.5448988472630654e-07,
      "logits/chosen": -2.003847599029541,
      "logits/rejected": -2.1804280281066895,
      "logps/chosen": -319.24346923828125,
      "logps/rejected": -137.32794189453125,
      "loss": 0.4327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48472583293914795,
      "rewards/margins": 0.8298259973526001,
      "rewards/rejected": -0.34510016441345215,
      "step": 127
    },
    {
      "epoch": 0.72,
      "grad_norm": 3.2166471546285473,
      "learning_rate": 1.5370818395889537e-07,
      "logits/chosen": -2.06440806388855,
      "logits/rejected": -2.2347941398620605,
      "logps/chosen": -379.2797546386719,
      "logps/rejected": -195.081298828125,
      "loss": 0.406,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8205428123474121,
      "rewards/margins": 1.3144938945770264,
      "rewards/rejected": -0.4939510226249695,
      "step": 128
    },
    {
      "epoch": 0.72,
      "grad_norm": 3.142444037300061,
      "learning_rate": 1.5292184334420432e-07,
      "logits/chosen": -2.001006841659546,
      "logits/rejected": -2.2717456817626953,
      "logps/chosen": -307.4756774902344,
      "logps/rejected": -124.45487976074219,
      "loss": 0.4755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.49742940068244934,
      "rewards/margins": 0.9193170070648193,
      "rewards/rejected": -0.4218876361846924,
      "step": 129
    },
    {
      "epoch": 0.73,
      "grad_norm": 3.0019240217136964,
      "learning_rate": 1.5213093081415919e-07,
      "logits/chosen": -2.0484771728515625,
      "logits/rejected": -2.3329827785491943,
      "logps/chosen": -382.6004943847656,
      "logps/rejected": -139.74937438964844,
      "loss": 0.4418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7651157379150391,
      "rewards/margins": 1.1857589483261108,
      "rewards/rejected": -0.4206430912017822,
      "step": 130
    },
    {
      "epoch": 0.73,
      "grad_norm": 3.324382092882249,
      "learning_rate": 1.513355146956531e-07,
      "logits/chosen": -1.9227899312973022,
      "logits/rejected": -2.203601598739624,
      "logps/chosen": -398.6375732421875,
      "logps/rejected": -149.57205200195312,
      "loss": 0.426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5919572114944458,
      "rewards/margins": 1.1777626276016235,
      "rewards/rejected": -0.585805356502533,
      "step": 131
    },
    {
      "epoch": 0.74,
      "grad_norm": 3.665359099834768,
      "learning_rate": 1.5053566370464416e-07,
      "logits/chosen": -2.04799222946167,
      "logits/rejected": -2.237955331802368,
      "logps/chosen": -287.4353942871094,
      "logps/rejected": -117.89936065673828,
      "loss": 0.3617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7061726450920105,
      "rewards/margins": 1.3360240459442139,
      "rewards/rejected": -0.6298512816429138,
      "step": 132
    },
    {
      "epoch": 0.74,
      "grad_norm": 3.0978268391650627,
      "learning_rate": 1.4973144694021873e-07,
      "logits/chosen": -1.8632404804229736,
      "logits/rejected": -2.116689682006836,
      "logps/chosen": -297.6752014160156,
      "logps/rejected": -127.87348937988281,
      "loss": 0.4035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4323517680168152,
      "rewards/margins": 0.7440455555915833,
      "rewards/rejected": -0.31169381737709045,
      "step": 133
    },
    {
      "epoch": 0.75,
      "grad_norm": 3.259874240443474,
      "learning_rate": 1.489229338786222e-07,
      "logits/chosen": -1.9522405862808228,
      "logits/rejected": -2.1952342987060547,
      "logps/chosen": -405.1477355957031,
      "logps/rejected": -266.0748291015625,
      "loss": 0.425,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6451842784881592,
      "rewards/margins": 0.8864087462425232,
      "rewards/rejected": -0.2412244975566864,
      "step": 134
    },
    {
      "epoch": 0.75,
      "grad_norm": 3.19860848209686,
      "learning_rate": 1.4811019436725682e-07,
      "logits/chosen": -2.013823986053467,
      "logits/rejected": -1.9999889135360718,
      "logps/chosen": -173.16363525390625,
      "logps/rejected": -175.79783630371094,
      "loss": 0.4859,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2981584668159485,
      "rewards/margins": 0.20658811926841736,
      "rewards/rejected": 0.09157033264636993,
      "step": 135
    },
    {
      "epoch": 0.76,
      "grad_norm": 3.6728926343933406,
      "learning_rate": 1.4729329861864768e-07,
      "logits/chosen": -2.057213306427002,
      "logits/rejected": -2.1428587436676025,
      "logps/chosen": -313.03656005859375,
      "logps/rejected": -199.88522338867188,
      "loss": 0.3831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5142601728439331,
      "rewards/margins": 1.0320128202438354,
      "rewards/rejected": -0.5177526473999023,
      "step": 136
    },
    {
      "epoch": 0.77,
      "grad_norm": 3.42354969285846,
      "learning_rate": 1.4647231720437685e-07,
      "logits/chosen": -2.0417561531066895,
      "logits/rejected": -2.2340712547302246,
      "logps/chosen": -366.47406005859375,
      "logps/rejected": -236.64149475097656,
      "loss": 0.4089,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4168028235435486,
      "rewards/margins": 0.5594974160194397,
      "rewards/rejected": -0.1426945924758911,
      "step": 137
    },
    {
      "epoch": 0.77,
      "grad_norm": 2.681331408633769,
      "learning_rate": 1.45647321048987e-07,
      "logits/chosen": -2.0262880325317383,
      "logits/rejected": -2.1669650077819824,
      "logps/chosen": -264.099609375,
      "logps/rejected": -138.3554229736328,
      "loss": 0.4216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3780781924724579,
      "rewards/margins": 0.6789387464523315,
      "rewards/rejected": -0.30086055397987366,
      "step": 138
    },
    {
      "epoch": 0.78,
      "grad_norm": 3.1553640769838704,
      "learning_rate": 1.4481838142385402e-07,
      "logits/chosen": -2.100149631500244,
      "logits/rejected": -2.246675729751587,
      "logps/chosen": -292.51287841796875,
      "logps/rejected": -152.18310546875,
      "loss": 0.3896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.460287868976593,
      "rewards/margins": 1.0902036428451538,
      "rewards/rejected": -0.629915714263916,
      "step": 139
    },
    {
      "epoch": 0.78,
      "grad_norm": 3.34078252938261,
      "learning_rate": 1.4398556994102994e-07,
      "logits/chosen": -1.9987834692001343,
      "logits/rejected": -2.1652636528015137,
      "logps/chosen": -336.60321044921875,
      "logps/rejected": -197.3555145263672,
      "loss": 0.4285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7229927778244019,
      "rewards/margins": 1.0515646934509277,
      "rewards/rejected": -0.32857200503349304,
      "step": 140
    },
    {
      "epoch": 0.79,
      "grad_norm": 3.2837068260200324,
      "learning_rate": 1.431489585470564e-07,
      "logits/chosen": -1.9157692193984985,
      "logits/rejected": -2.0339536666870117,
      "logps/chosen": -246.109130859375,
      "logps/rejected": -140.8620147705078,
      "loss": 0.4436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4217866063117981,
      "rewards/margins": 0.8604772686958313,
      "rewards/rejected": -0.4386906027793884,
      "step": 141
    },
    {
      "epoch": 0.79,
      "grad_norm": 2.9978563958102957,
      "learning_rate": 1.4230861951674913e-07,
      "logits/chosen": -1.864048957824707,
      "logits/rejected": -2.2588746547698975,
      "logps/chosen": -398.6397705078125,
      "logps/rejected": -112.80824279785156,
      "loss": 0.4073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7403268814086914,
      "rewards/margins": 1.137719988822937,
      "rewards/rejected": -0.39739301800727844,
      "step": 142
    },
    {
      "epoch": 0.8,
      "grad_norm": 2.692917813565496,
      "learning_rate": 1.4146462544695427e-07,
      "logits/chosen": -1.9859681129455566,
      "logits/rejected": -2.0894880294799805,
      "logps/chosen": -252.89605712890625,
      "logps/rejected": -177.11151123046875,
      "loss": 0.4258,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.41960960626602173,
      "rewards/margins": 0.7052587270736694,
      "rewards/rejected": -0.2856491208076477,
      "step": 143
    },
    {
      "epoch": 0.8,
      "grad_norm": 3.0493282694050374,
      "learning_rate": 1.4061704925027652e-07,
      "logits/chosen": -1.9278736114501953,
      "logits/rejected": -2.2551167011260986,
      "logps/chosen": -409.3521728515625,
      "logps/rejected": -139.8559112548828,
      "loss": 0.4258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7342904210090637,
      "rewards/margins": 1.162542462348938,
      "rewards/rejected": -0.4282519817352295,
      "step": 144
    },
    {
      "epoch": 0.81,
      "grad_norm": 2.8366166452321324,
      "learning_rate": 1.3976596414878042e-07,
      "logits/chosen": -1.9339745044708252,
      "logits/rejected": -2.0851519107818604,
      "logps/chosen": -315.5764465332031,
      "logps/rejected": -161.580810546875,
      "loss": 0.4188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4443025588989258,
      "rewards/margins": 0.8001634478569031,
      "rewards/rejected": -0.3558608591556549,
      "step": 145
    },
    {
      "epoch": 0.82,
      "grad_norm": 3.7752964496805377,
      "learning_rate": 1.3891144366766457e-07,
      "logits/chosen": -1.9236942529678345,
      "logits/rejected": -2.1249706745147705,
      "logps/chosen": -464.9728698730469,
      "logps/rejected": -287.4863586425781,
      "loss": 0.4272,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17092488706111908,
      "rewards/margins": 0.4706559181213379,
      "rewards/rejected": -0.29973104596138,
      "step": 146
    },
    {
      "epoch": 0.82,
      "grad_norm": 3.3368235459802063,
      "learning_rate": 1.380535616289099e-07,
      "logits/chosen": -1.956026554107666,
      "logits/rejected": -2.0740966796875,
      "logps/chosen": -301.49847412109375,
      "logps/rejected": -199.72262573242188,
      "loss": 0.471,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4182983636856079,
      "rewards/margins": 0.6974088549613953,
      "rewards/rejected": -0.27911046147346497,
      "step": 147
    },
    {
      "epoch": 0.83,
      "grad_norm": 2.617033810085924,
      "learning_rate": 1.3719239214490203e-07,
      "logits/chosen": -1.961683750152588,
      "logits/rejected": -2.088846445083618,
      "logps/chosen": -185.6378173828125,
      "logps/rejected": -52.31831741333008,
      "loss": 0.3959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42800790071487427,
      "rewards/margins": 0.7910484075546265,
      "rewards/rejected": -0.36304041743278503,
      "step": 148
    },
    {
      "epoch": 0.83,
      "grad_norm": 2.9547282694046486,
      "learning_rate": 1.3632800961202888e-07,
      "logits/chosen": -2.000007152557373,
      "logits/rejected": -2.2719762325286865,
      "logps/chosen": -264.94610595703125,
      "logps/rejected": -60.602142333984375,
      "loss": 0.4003,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.47297602891921997,
      "rewards/margins": 0.7190853953361511,
      "rewards/rejected": -0.24610941112041473,
      "step": 149
    },
    {
      "epoch": 0.84,
      "grad_norm": 2.672734779611145,
      "learning_rate": 1.3546048870425355e-07,
      "logits/chosen": -1.8334035873413086,
      "logits/rejected": -2.062422752380371,
      "logps/chosen": -295.311767578125,
      "logps/rejected": -144.99703979492188,
      "loss": 0.4231,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.27950870990753174,
      "rewards/margins": 0.37616291642189026,
      "rewards/rejected": -0.09665423631668091,
      "step": 150
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": -1.9373823404312134,
      "eval_logits/rejected": -2.1733014583587646,
      "eval_logps/chosen": -317.19122314453125,
      "eval_logps/rejected": -132.44723510742188,
      "eval_loss": 0.375163197517395,
      "eval_rewards/accuracies": 0.9437500238418579,
      "eval_rewards/chosen": 0.6166489720344543,
      "eval_rewards/margins": 1.1126511096954346,
      "eval_rewards/rejected": -0.4960021376609802,
      "eval_runtime": 184.2667,
      "eval_samples_per_second": 6.908,
      "eval_steps_per_second": 0.868,
      "step": 150
    },
    {
      "epoch": 0.84,
      "grad_norm": 2.8687751990991144,
      "learning_rate": 1.345899043666631e-07,
      "logits/chosen": -2.0048770904541016,
      "logits/rejected": -2.2049386501312256,
      "logps/chosen": -202.36087036132812,
      "logps/rejected": -59.840599060058594,
      "loss": 0.4417,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6141016483306885,
      "rewards/margins": 0.914747953414917,
      "rewards/rejected": -0.3006463646888733,
      "step": 151
    },
    {
      "epoch": 0.85,
      "grad_norm": 2.7728421710924707,
      "learning_rate": 1.3371633180899414e-07,
      "logits/chosen": -1.927375078201294,
      "logits/rejected": -2.266674041748047,
      "logps/chosen": -390.30902099609375,
      "logps/rejected": -173.35003662109375,
      "loss": 0.3938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9191557168960571,
      "rewards/margins": 1.6978180408477783,
      "rewards/rejected": -0.7786622047424316,
      "step": 152
    },
    {
      "epoch": 0.85,
      "grad_norm": 3.0360315000193454,
      "learning_rate": 1.328398464991355e-07,
      "logits/chosen": -1.8318389654159546,
      "logits/rejected": -2.1461503505706787,
      "logps/chosen": -339.9506530761719,
      "logps/rejected": -144.16354370117188,
      "loss": 0.3618,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5340854525566101,
      "rewards/margins": 1.0414400100708008,
      "rewards/rejected": -0.5073546171188354,
      "step": 153
    },
    {
      "epoch": 0.86,
      "grad_norm": 2.7772677805340003,
      "learning_rate": 1.3196052415660855e-07,
      "logits/chosen": -1.9493682384490967,
      "logits/rejected": -2.2572414875030518,
      "logps/chosen": -297.68194580078125,
      "logps/rejected": -81.56572723388672,
      "loss": 0.4573,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.806023359298706,
      "rewards/margins": 1.2982468605041504,
      "rewards/rejected": -0.4922235906124115,
      "step": 154
    },
    {
      "epoch": 0.87,
      "grad_norm": 2.8726649702254616,
      "learning_rate": 1.3107844074602566e-07,
      "logits/chosen": -2.0517020225524902,
      "logits/rejected": -2.3108386993408203,
      "logps/chosen": -284.3307800292969,
      "logps/rejected": -92.55622100830078,
      "loss": 0.3985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7782894372940063,
      "rewards/margins": 1.382967472076416,
      "rewards/rejected": -0.6046780347824097,
      "step": 155
    },
    {
      "epoch": 0.87,
      "grad_norm": 2.915466394554351,
      "learning_rate": 1.301936724705278e-07,
      "logits/chosen": -1.935138463973999,
      "logits/rejected": -2.229339599609375,
      "logps/chosen": -348.0072937011719,
      "logps/rejected": -179.30294799804688,
      "loss": 0.3496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6688775420188904,
      "rewards/margins": 1.195144772529602,
      "rewards/rejected": -0.5262673497200012,
      "step": 156
    },
    {
      "epoch": 0.88,
      "grad_norm": 2.9307421779325877,
      "learning_rate": 1.2930629576520132e-07,
      "logits/chosen": -1.937353491783142,
      "logits/rejected": -2.111067295074463,
      "logps/chosen": -232.2332763671875,
      "logps/rejected": -105.97891998291016,
      "loss": 0.3727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4184137284755707,
      "rewards/margins": 0.6602100729942322,
      "rewards/rejected": -0.2417963743209839,
      "step": 157
    },
    {
      "epoch": 0.88,
      "grad_norm": 2.704607717596039,
      "learning_rate": 1.2841638729047463e-07,
      "logits/chosen": -1.896262764930725,
      "logits/rejected": -2.1101086139678955,
      "logps/chosen": -276.5863037109375,
      "logps/rejected": -109.18448638916016,
      "loss": 0.432,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.822554886341095,
      "rewards/margins": 1.2940058708190918,
      "rewards/rejected": -0.47145119309425354,
      "step": 158
    },
    {
      "epoch": 0.89,
      "grad_norm": 3.030614573668553,
      "learning_rate": 1.2752402392549554e-07,
      "logits/chosen": -1.9761974811553955,
      "logits/rejected": -2.10685658454895,
      "logps/chosen": -211.11436462402344,
      "logps/rejected": -134.52835083007812,
      "loss": 0.4425,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.24754217267036438,
      "rewards/margins": 0.4351383447647095,
      "rewards/rejected": -0.1875961720943451,
      "step": 159
    },
    {
      "epoch": 0.89,
      "grad_norm": 3.298515748860272,
      "learning_rate": 1.2662928276148983e-07,
      "logits/chosen": -2.1360230445861816,
      "logits/rejected": -2.257654905319214,
      "logps/chosen": -249.20669555664062,
      "logps/rejected": -131.0137176513672,
      "loss": 0.4254,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.386399507522583,
      "rewards/margins": 0.8481476306915283,
      "rewards/rejected": -0.4617480933666229,
      "step": 160
    },
    {
      "epoch": 0.9,
      "grad_norm": 3.0649725875120377,
      "learning_rate": 1.257322410951011e-07,
      "logits/chosen": -1.9837696552276611,
      "logits/rejected": -2.2624270915985107,
      "logps/chosen": -298.5034484863281,
      "logps/rejected": -119.38221740722656,
      "loss": 0.3324,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8645215034484863,
      "rewards/margins": 1.5025858879089355,
      "rewards/rejected": -0.6380643248558044,
      "step": 161
    },
    {
      "epoch": 0.91,
      "grad_norm": 3.0712945490963937,
      "learning_rate": 1.248329764217133e-07,
      "logits/chosen": -1.90183424949646,
      "logits/rejected": -2.1589910984039307,
      "logps/chosen": -363.40972900390625,
      "logps/rejected": -109.76152801513672,
      "loss": 0.3242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6098992824554443,
      "rewards/margins": 1.0329221487045288,
      "rewards/rejected": -0.4230228662490845,
      "step": 162
    },
    {
      "epoch": 0.91,
      "grad_norm": 2.9440897282331484,
      "learning_rate": 1.2393156642875578e-07,
      "logits/chosen": -1.9756544828414917,
      "logits/rejected": -2.169515609741211,
      "logps/chosen": -272.5865173339844,
      "logps/rejected": -102.83159637451172,
      "loss": 0.3983,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.38845062255859375,
      "rewards/margins": 0.74920654296875,
      "rewards/rejected": -0.36075589060783386,
      "step": 163
    },
    {
      "epoch": 0.92,
      "grad_norm": 2.7376552759938884,
      "learning_rate": 1.2302808898899197e-07,
      "logits/chosen": -1.858717918395996,
      "logits/rejected": -2.072749137878418,
      "logps/chosen": -332.71893310546875,
      "logps/rejected": -134.6237030029297,
      "loss": 0.3703,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6717262864112854,
      "rewards/margins": 1.630864143371582,
      "rewards/rejected": -0.9591379165649414,
      "step": 164
    },
    {
      "epoch": 0.92,
      "grad_norm": 2.702952272294126,
      "learning_rate": 1.2212262215379195e-07,
      "logits/chosen": -1.915556788444519,
      "logits/rejected": -2.1649136543273926,
      "logps/chosen": -353.37335205078125,
      "logps/rejected": -116.70270538330078,
      "loss": 0.4008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6816975474357605,
      "rewards/margins": 1.3016923666000366,
      "rewards/rejected": -0.6199949979782104,
      "step": 165
    },
    {
      "epoch": 0.93,
      "grad_norm": 2.860968950223889,
      "learning_rate": 1.2121524414638958e-07,
      "logits/chosen": -1.981323003768921,
      "logits/rejected": -2.0863146781921387,
      "logps/chosen": -278.7300109863281,
      "logps/rejected": -162.6415252685547,
      "loss": 0.4229,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5619724988937378,
      "rewards/margins": 0.8598495721817017,
      "rewards/rejected": -0.2978770434856415,
      "step": 166
    },
    {
      "epoch": 0.93,
      "grad_norm": 2.9772601665108738,
      "learning_rate": 1.2030603335512466e-07,
      "logits/chosen": -2.0251593589782715,
      "logits/rejected": -2.337674617767334,
      "logps/chosen": -397.1326904296875,
      "logps/rejected": -146.89852905273438,
      "loss": 0.417,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8214364051818848,
      "rewards/margins": 1.3918532133102417,
      "rewards/rejected": -0.5704168081283569,
      "step": 167
    },
    {
      "epoch": 0.94,
      "grad_norm": 2.6713639375367237,
      "learning_rate": 1.1939506832667128e-07,
      "logits/chosen": -1.9390000104904175,
      "logits/rejected": -2.1995325088500977,
      "logps/chosen": -366.1788330078125,
      "logps/rejected": -156.14825439453125,
      "loss": 0.363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9185819029808044,
      "rewards/margins": 1.6624879837036133,
      "rewards/rejected": -0.7439059615135193,
      "step": 168
    },
    {
      "epoch": 0.94,
      "grad_norm": 2.5368366922940884,
      "learning_rate": 1.1848242775925185e-07,
      "logits/chosen": -1.8598655462265015,
      "logits/rejected": -2.018507719039917,
      "logps/chosen": -140.5003204345703,
      "logps/rejected": -81.32078552246094,
      "loss": 0.3813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22455516457557678,
      "rewards/margins": 0.4118771553039551,
      "rewards/rejected": -0.18732202053070068,
      "step": 169
    },
    {
      "epoch": 0.95,
      "grad_norm": 2.9108161123614926,
      "learning_rate": 1.175681904958386e-07,
      "logits/chosen": -2.0405771732330322,
      "logits/rejected": -2.107203483581543,
      "logps/chosen": -227.36375427246094,
      "logps/rejected": -147.64981079101562,
      "loss": 0.3762,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.35193291306495667,
      "rewards/margins": 0.7042461633682251,
      "rewards/rejected": -0.35231325030326843,
      "step": 170
    },
    {
      "epoch": 0.96,
      "grad_norm": 2.2027237105876867,
      "learning_rate": 1.1665243551734219e-07,
      "logits/chosen": -1.9685847759246826,
      "logits/rejected": -2.2867276668548584,
      "logps/chosen": -288.57080078125,
      "logps/rejected": -110.56059265136719,
      "loss": 0.4365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7529231905937195,
      "rewards/margins": 1.138110876083374,
      "rewards/rejected": -0.38518771529197693,
      "step": 171
    },
    {
      "epoch": 0.96,
      "grad_norm": 2.6386128158434756,
      "learning_rate": 1.157352419357886e-07,
      "logits/chosen": -1.964128851890564,
      "logits/rejected": -2.035123348236084,
      "logps/chosen": -198.30899047851562,
      "logps/rejected": -119.32826232910156,
      "loss": 0.3922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22413687407970428,
      "rewards/margins": 0.5808830261230469,
      "rewards/rejected": -0.3567461669445038,
      "step": 172
    },
    {
      "epoch": 0.97,
      "grad_norm": 2.5439840312023967,
      "learning_rate": 1.1481668898748473e-07,
      "logits/chosen": -1.8627464771270752,
      "logits/rejected": -2.195005178451538,
      "logps/chosen": -508.2653503417969,
      "logps/rejected": -201.6099090576172,
      "loss": 0.4126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7898422479629517,
      "rewards/margins": 1.3333909511566162,
      "rewards/rejected": -0.5435488224029541,
      "step": 173
    },
    {
      "epoch": 0.97,
      "grad_norm": 2.7840535206098735,
      "learning_rate": 1.1389685602617301e-07,
      "logits/chosen": -1.9687414169311523,
      "logits/rejected": -2.079282760620117,
      "logps/chosen": -273.3905944824219,
      "logps/rejected": -167.68350219726562,
      "loss": 0.4538,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.44472670555114746,
      "rewards/margins": 0.6912115216255188,
      "rewards/rejected": -0.2464849352836609,
      "step": 174
    },
    {
      "epoch": 0.98,
      "grad_norm": 3.1528869938353474,
      "learning_rate": 1.1297582251617617e-07,
      "logits/chosen": -2.000757932662964,
      "logits/rejected": -2.3343327045440674,
      "logps/chosen": -323.3289489746094,
      "logps/rejected": -116.5455551147461,
      "loss": 0.4193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5140612125396729,
      "rewards/margins": 0.8541378974914551,
      "rewards/rejected": -0.340076744556427,
      "step": 175
    },
    {
      "epoch": 0.98,
      "grad_norm": 2.5396783882345075,
      "learning_rate": 1.1205366802553229e-07,
      "logits/chosen": -1.9811201095581055,
      "logits/rejected": -2.090970516204834,
      "logps/chosen": -265.3904113769531,
      "logps/rejected": -177.4838409423828,
      "loss": 0.3467,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1779039204120636,
      "rewards/margins": 0.2935892939567566,
      "rewards/rejected": -0.115685373544693,
      "step": 176
    },
    {
      "epoch": 0.99,
      "grad_norm": 2.1452451342239893,
      "learning_rate": 1.1113047221912095e-07,
      "logits/chosen": -1.8996617794036865,
      "logits/rejected": -2.1304678916931152,
      "logps/chosen": -225.79818725585938,
      "logps/rejected": -57.702632904052734,
      "loss": 0.3968,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6985273361206055,
      "rewards/margins": 1.1475094556808472,
      "rewards/rejected": -0.4489821195602417,
      "step": 177
    },
    {
      "epoch": 0.99,
      "grad_norm": 2.9774371938674387,
      "learning_rate": 1.1020631485178082e-07,
      "logits/chosen": -1.9348338842391968,
      "logits/rejected": -2.2529170513153076,
      "logps/chosen": -398.8677978515625,
      "logps/rejected": -131.35577392578125,
      "loss": 0.3689,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9068738222122192,
      "rewards/margins": 1.5566710233688354,
      "rewards/rejected": -0.649797260761261,
      "step": 178
    },
    {
      "epoch": 1.0,
      "grad_norm": 2.5452995220193033,
      "learning_rate": 1.092812757614199e-07,
      "logits/chosen": -1.9580453634262085,
      "logits/rejected": -2.2709877490997314,
      "logps/chosen": -397.8183898925781,
      "logps/rejected": -121.63996124267578,
      "loss": 0.3738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0113794803619385,
      "rewards/margins": 1.6582667827606201,
      "rewards/rejected": -0.6468873023986816,
      "step": 179
    },
    {
      "epoch": 1.01,
      "grad_norm": 2.4951114841562623,
      "learning_rate": 1.0835543486211813e-07,
      "logits/chosen": -1.9591948986053467,
      "logits/rejected": -2.068274974822998,
      "logps/chosen": -218.165771484375,
      "logps/rejected": -147.83116149902344,
      "loss": 0.3839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38550132513046265,
      "rewards/margins": 1.0801736116409302,
      "rewards/rejected": -0.6946722865104675,
      "step": 180
    },
    {
      "epoch": 1.01,
      "grad_norm": 2.739194097290467,
      "learning_rate": 1.0742887213722372e-07,
      "logits/chosen": -1.8857489824295044,
      "logits/rejected": -2.036141872406006,
      "logps/chosen": -347.1433410644531,
      "logps/rejected": -185.33444213867188,
      "loss": 0.3615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.666320264339447,
      "rewards/margins": 1.2669157981872559,
      "rewards/rejected": -0.6005955338478088,
      "step": 181
    },
    {
      "epoch": 1.02,
      "grad_norm": 2.7874477837091036,
      "learning_rate": 1.0650166763244328e-07,
      "logits/chosen": -1.9431777000427246,
      "logits/rejected": -2.0786421298980713,
      "logps/chosen": -217.96388244628906,
      "logps/rejected": -161.750732421875,
      "loss": 0.3242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3562370538711548,
      "rewards/margins": 0.847759485244751,
      "rewards/rejected": -0.4915224611759186,
      "step": 182
    },
    {
      "epoch": 1.02,
      "grad_norm": 2.5576723261926944,
      "learning_rate": 1.0557390144892684e-07,
      "logits/chosen": -1.9544168710708618,
      "logits/rejected": -2.2021868228912354,
      "logps/chosen": -460.05682373046875,
      "logps/rejected": -235.51014709472656,
      "loss": 0.3113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7885012030601501,
      "rewards/margins": 1.7033418416976929,
      "rewards/rejected": -0.914840579032898,
      "step": 183
    },
    {
      "epoch": 1.03,
      "grad_norm": 2.4953298951762664,
      "learning_rate": 1.0464565373634781e-07,
      "logits/chosen": -1.8733316659927368,
      "logits/rejected": -2.055713415145874,
      "logps/chosen": -170.8631591796875,
      "logps/rejected": -65.48750305175781,
      "loss": 0.3993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.37032586336135864,
      "rewards/margins": 0.8034341335296631,
      "rewards/rejected": -0.43310827016830444,
      "step": 184
    },
    {
      "epoch": 1.03,
      "grad_norm": 2.6489494096336954,
      "learning_rate": 1.0371700468597883e-07,
      "logits/chosen": -1.8701441287994385,
      "logits/rejected": -2.035473585128784,
      "logps/chosen": -155.58811950683594,
      "logps/rejected": -72.45659637451172,
      "loss": 0.3746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2507627010345459,
      "rewards/margins": 0.5653121471405029,
      "rewards/rejected": -0.3145494759082794,
      "step": 185
    },
    {
      "epoch": 1.04,
      "grad_norm": 2.8950171966527543,
      "learning_rate": 1.0278803452376414e-07,
      "logits/chosen": -2.0079855918884277,
      "logits/rejected": -2.0556740760803223,
      "logps/chosen": -259.5736999511719,
      "logps/rejected": -169.33746337890625,
      "loss": 0.3624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.393430233001709,
      "rewards/margins": 0.8617305755615234,
      "rewards/rejected": -0.4683002829551697,
      "step": 186
    },
    {
      "epoch": 1.04,
      "grad_norm": 2.7819221061446946,
      "learning_rate": 1.0185882350338879e-07,
      "logits/chosen": -1.8643912076950073,
      "logits/rejected": -2.1565229892730713,
      "logps/chosen": -332.9527587890625,
      "logps/rejected": -144.50399780273438,
      "loss": 0.3721,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.554010272026062,
      "rewards/margins": 1.1745232343673706,
      "rewards/rejected": -0.6205129027366638,
      "step": 187
    },
    {
      "epoch": 1.05,
      "grad_norm": 2.8762918304755365,
      "learning_rate": 1.0092945189934557e-07,
      "logits/chosen": -2.03650164604187,
      "logits/rejected": -2.168185234069824,
      "logps/chosen": -310.5986633300781,
      "logps/rejected": -188.8811492919922,
      "loss": 0.3686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5031630396842957,
      "rewards/margins": 0.8894560933113098,
      "rewards/rejected": -0.38629305362701416,
      "step": 188
    },
    {
      "epoch": 1.06,
      "grad_norm": 2.6024918959221424,
      "learning_rate": 1e-07,
      "logits/chosen": -1.9812936782836914,
      "logits/rejected": -2.0917391777038574,
      "logps/chosen": -264.7093811035156,
      "logps/rejected": -173.5496368408203,
      "loss": 0.429,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.48823249340057373,
      "rewards/margins": 0.9868167638778687,
      "rewards/rejected": -0.49858415126800537,
      "step": 189
    },
    {
      "epoch": 1.06,
      "grad_norm": 2.2471126430767163,
      "learning_rate": 9.907054810065445e-08,
      "logits/chosen": -2.0062572956085205,
      "logits/rejected": -2.277808666229248,
      "logps/chosen": -332.92169189453125,
      "logps/rejected": -126.70708465576172,
      "loss": 0.3629,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8331867456436157,
      "rewards/margins": 1.6676141023635864,
      "rewards/rejected": -0.8344272375106812,
      "step": 190
    },
    {
      "epoch": 1.07,
      "grad_norm": 2.698246167261979,
      "learning_rate": 9.81411764966112e-08,
      "logits/chosen": -1.9375407695770264,
      "logits/rejected": -2.0159647464752197,
      "logps/chosen": -309.874267578125,
      "logps/rejected": -234.45297241210938,
      "loss": 0.4218,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.24357011914253235,
      "rewards/margins": 0.4638202488422394,
      "rewards/rejected": -0.22025014460086823,
      "step": 191
    },
    {
      "epoch": 1.07,
      "grad_norm": 2.2470535964848417,
      "learning_rate": 9.721196547623583e-08,
      "logits/chosen": -1.9590727090835571,
      "logits/rejected": -2.049032211303711,
      "logps/chosen": -219.5438232421875,
      "logps/rejected": -159.00331115722656,
      "loss": 0.3911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3394254446029663,
      "rewards/margins": 0.7338558435440063,
      "rewards/rejected": -0.39443033933639526,
      "step": 192
    },
    {
      "epoch": 1.08,
      "grad_norm": 2.191536937514122,
      "learning_rate": 9.628299531402117e-08,
      "logits/chosen": -2.009471893310547,
      "logits/rejected": -2.1102893352508545,
      "logps/chosen": -288.97894287109375,
      "logps/rejected": -184.86932373046875,
      "loss": 0.4043,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5465821623802185,
      "rewards/margins": 1.0711835622787476,
      "rewards/rejected": -0.5246013402938843,
      "step": 193
    },
    {
      "epoch": 1.08,
      "grad_norm": 2.3562073323406607,
      "learning_rate": 9.535434626365219e-08,
      "logits/chosen": -1.9866628646850586,
      "logits/rejected": -1.9806206226348877,
      "logps/chosen": -145.32054138183594,
      "logps/rejected": -112.49516296386719,
      "loss": 0.399,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06548295170068741,
      "rewards/margins": 0.25554153323173523,
      "rewards/rejected": -0.19005858898162842,
      "step": 194
    },
    {
      "epoch": 1.09,
      "grad_norm": 2.5285073507638876,
      "learning_rate": 9.442609855107316e-08,
      "logits/chosen": -1.9528928995132446,
      "logits/rejected": -2.200953245162964,
      "logps/chosen": -309.15008544921875,
      "logps/rejected": -144.05575561523438,
      "loss": 0.3231,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.46039754152297974,
      "rewards/margins": 0.8913149833679199,
      "rewards/rejected": -0.43091750144958496,
      "step": 195
    },
    {
      "epoch": 1.09,
      "grad_norm": 2.0784906826561236,
      "learning_rate": 9.349833236755673e-08,
      "logits/chosen": -1.8770756721496582,
      "logits/rejected": -2.105363607406616,
      "logps/chosen": -317.50152587890625,
      "logps/rejected": -153.7784881591797,
      "loss": 0.3486,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5030040144920349,
      "rewards/margins": 0.996347963809967,
      "rewards/rejected": -0.4933440089225769,
      "step": 196
    },
    {
      "epoch": 1.1,
      "grad_norm": 2.223176405056625,
      "learning_rate": 9.25711278627763e-08,
      "logits/chosen": -2.0445773601531982,
      "logits/rejected": -2.2006373405456543,
      "logps/chosen": -278.0598449707031,
      "logps/rejected": -142.84823608398438,
      "loss": 0.3397,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6799702048301697,
      "rewards/margins": 1.194618582725525,
      "rewards/rejected": -0.5146484971046448,
      "step": 197
    },
    {
      "epoch": 1.11,
      "grad_norm": 2.406512154679947,
      "learning_rate": 9.164456513788186e-08,
      "logits/chosen": -2.011723518371582,
      "logits/rejected": -1.9958322048187256,
      "logps/chosen": -247.50254821777344,
      "logps/rejected": -222.34512329101562,
      "loss": 0.3644,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3076442778110504,
      "rewards/margins": 0.4636233150959015,
      "rewards/rejected": -0.1559790074825287,
      "step": 198
    },
    {
      "epoch": 1.11,
      "grad_norm": 2.4695974188706433,
      "learning_rate": 9.071872423858009e-08,
      "logits/chosen": -1.9512934684753418,
      "logits/rejected": -2.1975042819976807,
      "logps/chosen": -483.12774658203125,
      "logps/rejected": -263.1443176269531,
      "loss": 0.388,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8154278993606567,
      "rewards/margins": 1.8745701313018799,
      "rewards/rejected": -1.0591423511505127,
      "step": 199
    },
    {
      "epoch": 1.12,
      "grad_norm": 2.9791630871699613,
      "learning_rate": 8.979368514821916e-08,
      "logits/chosen": -2.008679151535034,
      "logits/rejected": -2.0934290885925293,
      "logps/chosen": -258.72503662109375,
      "logps/rejected": -192.31565856933594,
      "loss": 0.3708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44245606660842896,
      "rewards/margins": 0.9912886619567871,
      "rewards/rejected": -0.5488325357437134,
      "step": 200
    },
    {
      "epoch": 1.12,
      "eval_logits/chosen": -1.9327764511108398,
      "eval_logits/rejected": -2.1473922729492188,
      "eval_logps/chosen": -316.40032958984375,
      "eval_logps/rejected": -138.82333374023438,
      "eval_loss": 0.34432658553123474,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": 0.6403750777244568,
      "eval_rewards/margins": 1.3276602029800415,
      "eval_rewards/rejected": -0.6872851848602295,
      "eval_runtime": 190.3561,
      "eval_samples_per_second": 6.687,
      "eval_steps_per_second": 0.841,
      "step": 200
    },
    {
      "epoch": 1.12,
      "grad_norm": 2.259506635951963,
      "learning_rate": 8.886952778087908e-08,
      "logits/chosen": -1.9257737398147583,
      "logits/rejected": -2.1019535064697266,
      "logps/chosen": -170.0314178466797,
      "logps/rejected": -37.75267791748047,
      "loss": 0.4161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4945024251937866,
      "rewards/margins": 0.9052304029464722,
      "rewards/rejected": -0.41072794795036316,
      "step": 201
    },
    {
      "epoch": 1.13,
      "grad_norm": 2.4528744856208076,
      "learning_rate": 8.79463319744677e-08,
      "logits/chosen": -1.9754432439804077,
      "logits/rejected": -2.2768192291259766,
      "logps/chosen": -404.13897705078125,
      "logps/rejected": -176.906005859375,
      "loss": 0.3842,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8032380938529968,
      "rewards/margins": 1.7221509218215942,
      "rewards/rejected": -0.9189128875732422,
      "step": 202
    },
    {
      "epoch": 1.13,
      "grad_norm": 3.28097929328298,
      "learning_rate": 8.702417748382384e-08,
      "logits/chosen": -2.0560543537139893,
      "logits/rejected": -1.9956598281860352,
      "logps/chosen": -257.294677734375,
      "logps/rejected": -254.02484130859375,
      "loss": 0.3235,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1712501496076584,
      "rewards/margins": 0.5382880568504333,
      "rewards/rejected": -0.36703795194625854,
      "step": 203
    },
    {
      "epoch": 1.14,
      "grad_norm": 1.6393878582724704,
      "learning_rate": 8.6103143973827e-08,
      "logits/chosen": -2.0064706802368164,
      "logits/rejected": -2.2147860527038574,
      "logps/chosen": -283.7664794921875,
      "logps/rejected": -125.88124084472656,
      "loss": 0.4117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9503887891769409,
      "rewards/margins": 1.508412480354309,
      "rewards/rejected": -0.5580236911773682,
      "step": 204
    },
    {
      "epoch": 1.15,
      "grad_norm": 2.0503962665532116,
      "learning_rate": 8.518331101251527e-08,
      "logits/chosen": -2.0628817081451416,
      "logits/rejected": -2.214637517929077,
      "logps/chosen": -342.5946350097656,
      "logps/rejected": -173.35684204101562,
      "loss": 0.3738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8031637072563171,
      "rewards/margins": 1.4461843967437744,
      "rewards/rejected": -0.6430206894874573,
      "step": 205
    },
    {
      "epoch": 1.15,
      "grad_norm": 2.3772305238257907,
      "learning_rate": 8.426475806421139e-08,
      "logits/chosen": -1.9149799346923828,
      "logits/rejected": -2.1681957244873047,
      "logps/chosen": -321.2602844238281,
      "logps/rejected": -96.91996002197266,
      "loss": 0.4006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5701212882995605,
      "rewards/margins": 1.2240173816680908,
      "rewards/rejected": -0.6538960337638855,
      "step": 206
    },
    {
      "epoch": 1.16,
      "grad_norm": 2.3700618894565273,
      "learning_rate": 8.33475644826578e-08,
      "logits/chosen": -1.86668860912323,
      "logits/rejected": -2.027719020843506,
      "logps/chosen": -322.3348693847656,
      "logps/rejected": -189.26300048828125,
      "loss": 0.3527,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7509647011756897,
      "rewards/margins": 1.6569186449050903,
      "rewards/rejected": -0.9059540629386902,
      "step": 207
    },
    {
      "epoch": 1.16,
      "grad_norm": 2.3204387917178884,
      "learning_rate": 8.243180950416141e-08,
      "logits/chosen": -1.9971275329589844,
      "logits/rejected": -2.099766492843628,
      "logps/chosen": -222.3831787109375,
      "logps/rejected": -123.86671447753906,
      "loss": 0.3796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40789657831192017,
      "rewards/margins": 0.9203974008560181,
      "rewards/rejected": -0.5125008821487427,
      "step": 208
    },
    {
      "epoch": 1.17,
      "grad_norm": 2.544884720920933,
      "learning_rate": 8.151757224074815e-08,
      "logits/chosen": -1.9076409339904785,
      "logits/rejected": -2.030827522277832,
      "logps/chosen": -233.522216796875,
      "logps/rejected": -173.6394805908203,
      "loss": 0.3593,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.33733195066452026,
      "rewards/margins": 0.6667687296867371,
      "rewards/rejected": -0.329436719417572,
      "step": 209
    },
    {
      "epoch": 1.17,
      "grad_norm": 2.37240337516102,
      "learning_rate": 8.060493167332874e-08,
      "logits/chosen": -2.0240964889526367,
      "logits/rejected": -2.0981948375701904,
      "logps/chosen": -324.484375,
      "logps/rejected": -219.33116149902344,
      "loss": 0.3292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6331191062927246,
      "rewards/margins": 1.715273141860962,
      "rewards/rejected": -1.0821539163589478,
      "step": 210
    },
    {
      "epoch": 1.18,
      "grad_norm": 3.012051922259596,
      "learning_rate": 7.969396664487533e-08,
      "logits/chosen": -1.9107437133789062,
      "logits/rejected": -2.1371631622314453,
      "logps/chosen": -339.2627258300781,
      "logps/rejected": -230.1804962158203,
      "loss": 0.4533,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2044561207294464,
      "rewards/margins": 0.44547808170318604,
      "rewards/rejected": -0.24102194607257843,
      "step": 211
    },
    {
      "epoch": 1.18,
      "grad_norm": 2.2052254247044116,
      "learning_rate": 7.878475585361043e-08,
      "logits/chosen": -1.9317848682403564,
      "logits/rejected": -2.19038724899292,
      "logps/chosen": -323.2309875488281,
      "logps/rejected": -125.88053894042969,
      "loss": 0.4097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6508646011352539,
      "rewards/margins": 1.367360234260559,
      "rewards/rejected": -0.7164957523345947,
      "step": 212
    },
    {
      "epoch": 1.19,
      "grad_norm": 2.235268190842379,
      "learning_rate": 7.787737784620802e-08,
      "logits/chosen": -1.909670352935791,
      "logits/rejected": -2.064894914627075,
      "logps/chosen": -219.2584228515625,
      "logps/rejected": -68.53303527832031,
      "loss": 0.3531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38158148527145386,
      "rewards/margins": 0.9134366512298584,
      "rewards/rejected": -0.5318551659584045,
      "step": 213
    },
    {
      "epoch": 1.2,
      "grad_norm": 2.3948915922057386,
      "learning_rate": 7.6971911011008e-08,
      "logits/chosen": -1.9540221691131592,
      "logits/rejected": -2.157740592956543,
      "logps/chosen": -316.31884765625,
      "logps/rejected": -154.80227661132812,
      "loss": 0.3884,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.45785102248191833,
      "rewards/margins": 1.0194741487503052,
      "rewards/rejected": -0.5616230964660645,
      "step": 214
    },
    {
      "epoch": 1.2,
      "grad_norm": 2.4074374529306155,
      "learning_rate": 7.606843357124425e-08,
      "logits/chosen": -1.9372565746307373,
      "logits/rejected": -2.164517879486084,
      "logps/chosen": -308.13494873046875,
      "logps/rejected": -149.24195861816406,
      "loss": 0.4067,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7418156266212463,
      "rewards/margins": 1.4810779094696045,
      "rewards/rejected": -0.7392622232437134,
      "step": 215
    },
    {
      "epoch": 1.21,
      "grad_norm": 2.130755933970431,
      "learning_rate": 7.516702357828671e-08,
      "logits/chosen": -1.990491509437561,
      "logits/rejected": -2.4479219913482666,
      "logps/chosen": -500.0743103027344,
      "logps/rejected": -156.37615966796875,
      "loss": 0.3141,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9668587446212769,
      "rewards/margins": 1.7484171390533447,
      "rewards/rejected": -0.7815585136413574,
      "step": 216
    },
    {
      "epoch": 1.21,
      "grad_norm": 2.519590433851525,
      "learning_rate": 7.426775890489889e-08,
      "logits/chosen": -1.8623852729797363,
      "logits/rejected": -2.0746591091156006,
      "logps/chosen": -345.7522277832031,
      "logps/rejected": -132.41415405273438,
      "loss": 0.3747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9244540929794312,
      "rewards/margins": 1.5649265050888062,
      "rewards/rejected": -0.6404723525047302,
      "step": 217
    },
    {
      "epoch": 1.22,
      "grad_norm": 2.1874875532732285,
      "learning_rate": 7.337071723851017e-08,
      "logits/chosen": -1.9167838096618652,
      "logits/rejected": -2.1459975242614746,
      "logps/chosen": -303.1180419921875,
      "logps/rejected": -105.34794616699219,
      "loss": 0.2966,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.881919264793396,
      "rewards/margins": 1.6922099590301514,
      "rewards/rejected": -0.8102905750274658,
      "step": 218
    },
    {
      "epoch": 1.22,
      "grad_norm": 2.9186874702232544,
      "learning_rate": 7.247597607450445e-08,
      "logits/chosen": -2.002195119857788,
      "logits/rejected": -2.1577107906341553,
      "logps/chosen": -310.2653503417969,
      "logps/rejected": -124.31275939941406,
      "loss": 0.3963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7073330879211426,
      "rewards/margins": 1.4355955123901367,
      "rewards/rejected": -0.7282623648643494,
      "step": 219
    },
    {
      "epoch": 1.23,
      "grad_norm": 1.995931596526281,
      "learning_rate": 7.15836127095254e-08,
      "logits/chosen": -1.8800973892211914,
      "logits/rejected": -2.0754265785217285,
      "logps/chosen": -272.69476318359375,
      "logps/rejected": -99.58995056152344,
      "loss": 0.361,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5042135119438171,
      "rewards/margins": 1.2575228214263916,
      "rewards/rejected": -0.7533093690872192,
      "step": 220
    },
    {
      "epoch": 1.23,
      "grad_norm": 6.773519914904181,
      "learning_rate": 7.06937042347987e-08,
      "logits/chosen": -1.9714473485946655,
      "logits/rejected": -2.09019136428833,
      "logps/chosen": -364.12158203125,
      "logps/rejected": -252.11019897460938,
      "loss": 0.4276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7859811186790466,
      "rewards/margins": 1.5761654376983643,
      "rewards/rejected": -0.7901844382286072,
      "step": 221
    },
    {
      "epoch": 1.24,
      "grad_norm": 2.4224792412511054,
      "learning_rate": 6.98063275294722e-08,
      "logits/chosen": -2.033931255340576,
      "logits/rejected": -2.032353639602661,
      "logps/chosen": -182.28111267089844,
      "logps/rejected": -170.1015167236328,
      "loss": 0.3984,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20503102242946625,
      "rewards/margins": 0.6778407692909241,
      "rewards/rejected": -0.4728098213672638,
      "step": 222
    },
    {
      "epoch": 1.25,
      "grad_norm": 2.8578473574930796,
      "learning_rate": 6.892155925397436e-08,
      "logits/chosen": -1.863666296005249,
      "logits/rejected": -2.2041678428649902,
      "logps/chosen": -364.0481262207031,
      "logps/rejected": -116.85747528076172,
      "loss": 0.3863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8524085283279419,
      "rewards/margins": 1.405036449432373,
      "rewards/rejected": -0.5526278614997864,
      "step": 223
    },
    {
      "epoch": 1.25,
      "grad_norm": 2.4553850662794003,
      "learning_rate": 6.803947584339147e-08,
      "logits/chosen": -1.9987305402755737,
      "logits/rejected": -2.1810014247894287,
      "logps/chosen": -308.6746826171875,
      "logps/rejected": -156.14413452148438,
      "loss": 0.3519,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.757546067237854,
      "rewards/margins": 1.9111757278442383,
      "rewards/rejected": -1.1536295413970947,
      "step": 224
    },
    {
      "epoch": 1.26,
      "grad_norm": 2.3959548460575517,
      "learning_rate": 6.716015350086448e-08,
      "logits/chosen": -1.9721763134002686,
      "logits/rejected": -2.15269136428833,
      "logps/chosen": -360.6725158691406,
      "logps/rejected": -232.14779663085938,
      "loss": 0.4382,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4904707968235016,
      "rewards/margins": 1.2632602453231812,
      "rewards/rejected": -0.772789478302002,
      "step": 225
    },
    {
      "epoch": 1.26,
      "grad_norm": 1.8852130124655633,
      "learning_rate": 6.628366819100585e-08,
      "logits/chosen": -1.8738517761230469,
      "logits/rejected": -2.2090766429901123,
      "logps/chosen": -344.3710632324219,
      "logps/rejected": -123.95575714111328,
      "loss": 0.3684,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7661735415458679,
      "rewards/margins": 1.6990057229995728,
      "rewards/rejected": -0.9328320026397705,
      "step": 226
    },
    {
      "epoch": 1.27,
      "grad_norm": 2.207509020107613,
      "learning_rate": 6.541009563333689e-08,
      "logits/chosen": -1.970944881439209,
      "logits/rejected": -2.104224681854248,
      "logps/chosen": -401.86761474609375,
      "logps/rejected": -276.5339050292969,
      "loss": 0.3071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6489874720573425,
      "rewards/margins": 1.4881306886672974,
      "rewards/rejected": -0.8391432166099548,
      "step": 227
    },
    {
      "epoch": 1.27,
      "grad_norm": 2.3035753230701586,
      "learning_rate": 6.453951129574643e-08,
      "logits/chosen": -1.8832050561904907,
      "logits/rejected": -2.253293514251709,
      "logps/chosen": -479.3350830078125,
      "logps/rejected": -133.14219665527344,
      "loss": 0.2556,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2150325775146484,
      "rewards/margins": 2.2640023231506348,
      "rewards/rejected": -1.0489698648452759,
      "step": 228
    },
    {
      "epoch": 1.28,
      "grad_norm": 2.563274710615486,
      "learning_rate": 6.367199038797109e-08,
      "logits/chosen": -1.9041603803634644,
      "logits/rejected": -2.0262975692749023,
      "logps/chosen": -199.96775817871094,
      "logps/rejected": -110.95515441894531,
      "loss": 0.3621,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.37201952934265137,
      "rewards/margins": 1.0209636688232422,
      "rewards/rejected": -0.6489440202713013,
      "step": 229
    },
    {
      "epoch": 1.28,
      "grad_norm": 2.2766910704747567,
      "learning_rate": 6.2807607855098e-08,
      "logits/chosen": -1.885288119316101,
      "logits/rejected": -2.1321539878845215,
      "logps/chosen": -268.776611328125,
      "logps/rejected": -114.54924011230469,
      "loss": 0.2867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36375609040260315,
      "rewards/margins": 0.8478094339370728,
      "rewards/rejected": -0.4840533137321472,
      "step": 230
    },
    {
      "epoch": 1.29,
      "grad_norm": 2.2615311670632128,
      "learning_rate": 6.194643837109014e-08,
      "logits/chosen": -1.8483718633651733,
      "logits/rejected": -2.052673816680908,
      "logps/chosen": -231.9535369873047,
      "logps/rejected": -95.31845092773438,
      "loss": 0.3074,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4465009570121765,
      "rewards/margins": 1.0453064441680908,
      "rewards/rejected": -0.5988054275512695,
      "step": 231
    },
    {
      "epoch": 1.3,
      "grad_norm": 1.8323460108873135,
      "learning_rate": 6.108855633233544e-08,
      "logits/chosen": -2.08075213432312,
      "logits/rejected": -2.3139994144439697,
      "logps/chosen": -300.26141357421875,
      "logps/rejected": -123.13629150390625,
      "loss": 0.4024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6193065047264099,
      "rewards/margins": 1.3384159803390503,
      "rewards/rejected": -0.7191093564033508,
      "step": 232
    },
    {
      "epoch": 1.3,
      "grad_norm": 2.308172837492481,
      "learning_rate": 6.02340358512196e-08,
      "logits/chosen": -1.918696641921997,
      "logits/rejected": -2.0569751262664795,
      "logps/chosen": -186.91880798339844,
      "logps/rejected": -65.9375,
      "loss": 0.337,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5072881579399109,
      "rewards/margins": 0.9656892418861389,
      "rewards/rejected": -0.45840105414390564,
      "step": 233
    },
    {
      "epoch": 1.31,
      "grad_norm": 3.2533259381605513,
      "learning_rate": 5.938295074972349e-08,
      "logits/chosen": -1.9810622930526733,
      "logits/rejected": -2.1882359981536865,
      "logps/chosen": -328.361328125,
      "logps/rejected": -165.9792938232422,
      "loss": 0.4462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40560880303382874,
      "rewards/margins": 1.1706748008728027,
      "rewards/rejected": -0.7650659084320068,
      "step": 234
    },
    {
      "epoch": 1.31,
      "grad_norm": 2.477262358757034,
      "learning_rate": 5.8535374553045746e-08,
      "logits/chosen": -1.893616795539856,
      "logits/rejected": -2.1727473735809326,
      "logps/chosen": -326.0169982910156,
      "logps/rejected": -136.06997680664062,
      "loss": 0.366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6259748935699463,
      "rewards/margins": 1.5379406213760376,
      "rewards/rejected": -0.9119656682014465,
      "step": 235
    },
    {
      "epoch": 1.32,
      "grad_norm": 1.9233773900803872,
      "learning_rate": 5.7691380483250864e-08,
      "logits/chosen": -1.9818511009216309,
      "logits/rejected": -2.2494049072265625,
      "logps/chosen": -303.03594970703125,
      "logps/rejected": -107.28809356689453,
      "loss": 0.3623,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6850159168243408,
      "rewards/margins": 1.286649465560913,
      "rewards/rejected": -0.6016334295272827,
      "step": 236
    },
    {
      "epoch": 1.32,
      "grad_norm": 2.4029669436961596,
      "learning_rate": 5.6851041452943636e-08,
      "logits/chosen": -1.9243234395980835,
      "logits/rejected": -2.0990493297576904,
      "logps/chosen": -305.79205322265625,
      "logps/rejected": -156.5312042236328,
      "loss": 0.3443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3460720479488373,
      "rewards/margins": 0.9225568175315857,
      "rewards/rejected": -0.576484739780426,
      "step": 237
    },
    {
      "epoch": 1.33,
      "grad_norm": 2.7172071884731333,
      "learning_rate": 5.601443005897011e-08,
      "logits/chosen": -1.990132212638855,
      "logits/rejected": -2.103468179702759,
      "logps/chosen": -316.22296142578125,
      "logps/rejected": -216.75155639648438,
      "loss": 0.3121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46712929010391235,
      "rewards/margins": 1.2178815603256226,
      "rewards/rejected": -0.7507522106170654,
      "step": 238
    },
    {
      "epoch": 1.34,
      "grad_norm": 2.331069676044177,
      "learning_rate": 5.5181618576145995e-08,
      "logits/chosen": -1.9477646350860596,
      "logits/rejected": -2.16232967376709,
      "logps/chosen": -326.668212890625,
      "logps/rejected": -136.16281127929688,
      "loss": 0.3737,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6249808669090271,
      "rewards/margins": 1.3888401985168457,
      "rewards/rejected": -0.7638593912124634,
      "step": 239
    },
    {
      "epoch": 1.34,
      "grad_norm": 2.4198590554719344,
      "learning_rate": 5.435267895101302e-08,
      "logits/chosen": -2.0229196548461914,
      "logits/rejected": -2.160853624343872,
      "logps/chosen": -235.00137329101562,
      "logps/rejected": -92.64382934570312,
      "loss": 0.4609,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3488693833351135,
      "rewards/margins": 0.6747847199440002,
      "rewards/rejected": -0.3259153962135315,
      "step": 240
    },
    {
      "epoch": 1.35,
      "grad_norm": 2.903826654030489,
      "learning_rate": 5.352768279562314e-08,
      "logits/chosen": -1.9650566577911377,
      "logits/rejected": -2.255340337753296,
      "logps/chosen": -419.90032958984375,
      "logps/rejected": -208.98171997070312,
      "loss": 0.3933,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7806905508041382,
      "rewards/margins": 1.5795037746429443,
      "rewards/rejected": -0.7988132238388062,
      "step": 241
    },
    {
      "epoch": 1.35,
      "grad_norm": 2.104301012306948,
      "learning_rate": 5.2706701381352336e-08,
      "logits/chosen": -1.9476219415664673,
      "logits/rejected": -2.1780595779418945,
      "logps/chosen": -285.74176025390625,
      "logps/rejected": -128.45465087890625,
      "loss": 0.3659,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5625160932540894,
      "rewards/margins": 1.3298622369766235,
      "rewards/rejected": -0.767346203327179,
      "step": 242
    },
    {
      "epoch": 1.36,
      "grad_norm": 2.054448385701696,
      "learning_rate": 5.1889805632743145e-08,
      "logits/chosen": -2.205219030380249,
      "logits/rejected": -2.2034475803375244,
      "logps/chosen": -378.1851501464844,
      "logps/rejected": -274.8927001953125,
      "loss": 0.329,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.49013322591781616,
      "rewards/margins": 0.9879810810089111,
      "rewards/rejected": -0.4978478252887726,
      "step": 243
    },
    {
      "epoch": 1.36,
      "grad_norm": 2.6121103191536537,
      "learning_rate": 5.107706612137775e-08,
      "logits/chosen": -1.9490768909454346,
      "logits/rejected": -2.1126646995544434,
      "logps/chosen": -360.6116943359375,
      "logps/rejected": -191.07272338867188,
      "loss": 0.2892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7919806838035583,
      "rewards/margins": 1.782927393913269,
      "rewards/rejected": -0.9909467101097107,
      "step": 244
    },
    {
      "epoch": 1.37,
      "grad_norm": 2.385541686750622,
      "learning_rate": 5.026855305978128e-08,
      "logits/chosen": -1.9246346950531006,
      "logits/rejected": -2.1882967948913574,
      "logps/chosen": -483.7081604003906,
      "logps/rejected": -254.4589080810547,
      "loss": 0.3677,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9386581182479858,
      "rewards/margins": 1.5556895732879639,
      "rewards/rejected": -0.6170313954353333,
      "step": 245
    },
    {
      "epoch": 1.37,
      "grad_norm": 2.6107252036470188,
      "learning_rate": 4.946433629535585e-08,
      "logits/chosen": -1.9748541116714478,
      "logits/rejected": -1.9948114156723022,
      "logps/chosen": -188.89405822753906,
      "logps/rejected": -140.08143615722656,
      "loss": 0.4692,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.32503047585487366,
      "rewards/margins": 1.246155858039856,
      "rewards/rejected": -0.9211252927780151,
      "step": 246
    },
    {
      "epoch": 1.38,
      "grad_norm": 2.406035236060261,
      "learning_rate": 4.8664485304346916e-08,
      "logits/chosen": -1.9609439373016357,
      "logits/rejected": -2.1323890686035156,
      "logps/chosen": -370.1985778808594,
      "logps/rejected": -151.9228515625,
      "loss": 0.3818,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8187414407730103,
      "rewards/margins": 1.6757946014404297,
      "rewards/rejected": -0.8570532202720642,
      "step": 247
    },
    {
      "epoch": 1.39,
      "grad_norm": 2.3506751324584236,
      "learning_rate": 4.7869069185840824e-08,
      "logits/chosen": -1.9399272203445435,
      "logits/rejected": -2.1106154918670654,
      "logps/chosen": -385.4277038574219,
      "logps/rejected": -187.8704071044922,
      "loss": 0.2971,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8000746965408325,
      "rewards/margins": 1.7866300344467163,
      "rewards/rejected": -0.9865550994873047,
      "step": 248
    },
    {
      "epoch": 1.39,
      "grad_norm": 2.037166364259834,
      "learning_rate": 4.707815665579569e-08,
      "logits/chosen": -1.9395835399627686,
      "logits/rejected": -2.105896234512329,
      "logps/chosen": -218.52117919921875,
      "logps/rejected": -102.7484359741211,
      "loss": 0.4023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4448435306549072,
      "rewards/margins": 1.021249771118164,
      "rewards/rejected": -0.5764063000679016,
      "step": 249
    },
    {
      "epoch": 1.4,
      "grad_norm": 2.0548700051829827,
      "learning_rate": 4.629181604110464e-08,
      "logits/chosen": -1.8626420497894287,
      "logits/rejected": -2.0713019371032715,
      "logps/chosen": -264.1625061035156,
      "logps/rejected": -78.62328338623047,
      "loss": 0.4062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46505922079086304,
      "rewards/margins": 1.1402562856674194,
      "rewards/rejected": -0.6751970052719116,
      "step": 250
    },
    {
      "epoch": 1.4,
      "eval_logits/chosen": -1.9314517974853516,
      "eval_logits/rejected": -2.156139850616455,
      "eval_logps/chosen": -316.2469787597656,
      "eval_logps/rejected": -142.72573852539062,
      "eval_loss": 0.32974618673324585,
      "eval_rewards/accuracies": 0.9312499761581421,
      "eval_rewards/chosen": 0.644976019859314,
      "eval_rewards/margins": 1.4493337869644165,
      "eval_rewards/rejected": -0.8043577075004578,
      "eval_runtime": 186.1915,
      "eval_samples_per_second": 6.837,
      "eval_steps_per_second": 0.859,
      "step": 250
    },
    {
      "epoch": 1.4,
      "grad_norm": 2.1864725160708818,
      "learning_rate": 4.5510115273693473e-08,
      "logits/chosen": -1.94536554813385,
      "logits/rejected": -2.1171536445617676,
      "logps/chosen": -295.99072265625,
      "logps/rejected": -195.84324645996094,
      "loss": 0.3722,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.40991413593292236,
      "rewards/margins": 0.8959781527519226,
      "rewards/rejected": -0.48606404662132263,
      "step": 251
    },
    {
      "epoch": 1.41,
      "grad_norm": 2.1951187853623693,
      "learning_rate": 4.473312188465166e-08,
      "logits/chosen": -1.919504165649414,
      "logits/rejected": -2.0694282054901123,
      "logps/chosen": -303.9930419921875,
      "logps/rejected": -130.7193603515625,
      "loss": 0.3174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7677474021911621,
      "rewards/margins": 1.868253231048584,
      "rewards/rejected": -1.1005058288574219,
      "step": 252
    },
    {
      "epoch": 1.41,
      "grad_norm": 3.1913113320998026,
      "learning_rate": 4.3960902998398517e-08,
      "logits/chosen": -1.9696305990219116,
      "logits/rejected": -2.1716620922088623,
      "logps/chosen": -374.0699462890625,
      "logps/rejected": -171.0696563720703,
      "loss": 0.4265,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3457423746585846,
      "rewards/margins": 1.1523842811584473,
      "rewards/rejected": -0.8066419363021851,
      "step": 253
    },
    {
      "epoch": 1.42,
      "grad_norm": 2.1165374906752796,
      "learning_rate": 4.319352532688443e-08,
      "logits/chosen": -1.9709705114364624,
      "logits/rejected": -2.1842715740203857,
      "logps/chosen": -341.5719299316406,
      "logps/rejected": -153.75521850585938,
      "loss": 0.3503,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8018146753311157,
      "rewards/margins": 1.9906085729599,
      "rewards/rejected": -1.1887940168380737,
      "step": 254
    },
    {
      "epoch": 1.42,
      "grad_norm": 2.713035319428413,
      "learning_rate": 4.2431055163827315e-08,
      "logits/chosen": -1.9593266248703003,
      "logits/rejected": -2.101644515991211,
      "logps/chosen": -299.6078796386719,
      "logps/rejected": -176.44241333007812,
      "loss": 0.2919,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7900698781013489,
      "rewards/margins": 1.7910141944885254,
      "rewards/rejected": -1.0009442567825317,
      "step": 255
    },
    {
      "epoch": 1.43,
      "grad_norm": 2.1241918465749783,
      "learning_rate": 4.167355837898584e-08,
      "logits/chosen": -1.9116507768630981,
      "logits/rejected": -2.0239267349243164,
      "logps/chosen": -224.43470764160156,
      "logps/rejected": -121.06668090820312,
      "loss": 0.3013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3974186182022095,
      "rewards/margins": 1.1547796726226807,
      "rewards/rejected": -0.7573609352111816,
      "step": 256
    },
    {
      "epoch": 1.44,
      "grad_norm": 2.7982635154400732,
      "learning_rate": 4.092110041246865e-08,
      "logits/chosen": -1.8939900398254395,
      "logits/rejected": -2.0005412101745605,
      "logps/chosen": -175.42764282226562,
      "logps/rejected": -72.14698791503906,
      "loss": 0.4166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4599054157733917,
      "rewards/margins": 0.9669243097305298,
      "rewards/rejected": -0.5070188641548157,
      "step": 257
    },
    {
      "epoch": 1.44,
      "grad_norm": 2.3714338867608507,
      "learning_rate": 4.017374626908124e-08,
      "logits/chosen": -1.9074149131774902,
      "logits/rejected": -2.0512611865997314,
      "logps/chosen": -270.6667175292969,
      "logps/rejected": -171.60247802734375,
      "loss": 0.3572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6291319727897644,
      "rewards/margins": 1.2309937477111816,
      "rewards/rejected": -0.601861834526062,
      "step": 258
    },
    {
      "epoch": 1.45,
      "grad_norm": 2.125630878152154,
      "learning_rate": 3.9431560512710026e-08,
      "logits/chosen": -1.9675098657608032,
      "logits/rejected": -2.1794068813323975,
      "logps/chosen": -383.7911682128906,
      "logps/rejected": -206.51065063476562,
      "loss": 0.3502,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5929781794548035,
      "rewards/margins": 1.4138081073760986,
      "rewards/rejected": -0.8208298683166504,
      "step": 259
    },
    {
      "epoch": 1.45,
      "grad_norm": 2.3708331370378977,
      "learning_rate": 3.869460726074474e-08,
      "logits/chosen": -1.9422714710235596,
      "logits/rejected": -2.1994147300720215,
      "logps/chosen": -375.3470458984375,
      "logps/rejected": -194.87181091308594,
      "loss": 0.3401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7767963409423828,
      "rewards/margins": 1.8286038637161255,
      "rewards/rejected": -1.0518076419830322,
      "step": 260
    },
    {
      "epoch": 1.46,
      "grad_norm": 2.151865974418779,
      "learning_rate": 3.7962950178539277e-08,
      "logits/chosen": -1.8709148168563843,
      "logits/rejected": -2.118030548095703,
      "logps/chosen": -486.78729248046875,
      "logps/rejected": -245.8949737548828,
      "loss": 0.3139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.190994381904602,
      "rewards/margins": 2.3700695037841797,
      "rewards/rejected": -1.1790751218795776,
      "step": 261
    },
    {
      "epoch": 1.46,
      "grad_norm": 2.073175030368598,
      "learning_rate": 3.723665247391181e-08,
      "logits/chosen": -1.998550295829773,
      "logits/rejected": -2.0971145629882812,
      "logps/chosen": -159.14913940429688,
      "logps/rejected": -89.764404296875,
      "loss": 0.3692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.37837278842926025,
      "rewards/margins": 0.7685126066207886,
      "rewards/rejected": -0.3901398181915283,
      "step": 262
    },
    {
      "epoch": 1.47,
      "grad_norm": 2.6211615886977406,
      "learning_rate": 3.6515776891684046e-08,
      "logits/chosen": -1.947357177734375,
      "logits/rejected": -2.213313341140747,
      "logps/chosen": -407.5662536621094,
      "logps/rejected": -170.8851318359375,
      "loss": 0.3545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8491164445877075,
      "rewards/margins": 1.8658862113952637,
      "rewards/rejected": -1.0167697668075562,
      "step": 263
    },
    {
      "epoch": 1.47,
      "grad_norm": 2.181847753329417,
      "learning_rate": 3.580038570826093e-08,
      "logits/chosen": -1.954185962677002,
      "logits/rejected": -1.9828972816467285,
      "logps/chosen": -225.95716857910156,
      "logps/rejected": -138.2814178466797,
      "loss": 0.3124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5291526317596436,
      "rewards/margins": 1.2315007448196411,
      "rewards/rejected": -0.7023481130599976,
      "step": 264
    },
    {
      "epoch": 1.48,
      "grad_norm": 2.48244810599722,
      "learning_rate": 3.509054072625031e-08,
      "logits/chosen": -2.0149214267730713,
      "logits/rejected": -2.144228219985962,
      "logps/chosen": -233.55215454101562,
      "logps/rejected": -158.93942260742188,
      "loss": 0.357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21644878387451172,
      "rewards/margins": 0.5699070692062378,
      "rewards/rejected": -0.3534582853317261,
      "step": 265
    },
    {
      "epoch": 1.49,
      "grad_norm": 2.5289077187729605,
      "learning_rate": 3.438630326912414e-08,
      "logits/chosen": -1.9141268730163574,
      "logits/rejected": -2.064079761505127,
      "logps/chosen": -210.72866821289062,
      "logps/rejected": -109.2860336303711,
      "loss": 0.3904,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.49795305728912354,
      "rewards/margins": 0.9178673028945923,
      "rewards/rejected": -0.41991421580314636,
      "step": 266
    },
    {
      "epoch": 1.49,
      "grad_norm": 2.571622602426196,
      "learning_rate": 3.36877341759205e-08,
      "logits/chosen": -1.9544380903244019,
      "logits/rejected": -2.116626262664795,
      "logps/chosen": -357.0731201171875,
      "logps/rejected": -189.14581298828125,
      "loss": 0.3862,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5316759943962097,
      "rewards/margins": 1.2794767618179321,
      "rewards/rejected": -0.7478006482124329,
      "step": 267
    },
    {
      "epoch": 1.5,
      "grad_norm": 1.949899789832065,
      "learning_rate": 3.2994893795987765e-08,
      "logits/chosen": -1.91293466091156,
      "logits/rejected": -2.20516300201416,
      "logps/chosen": -343.4618835449219,
      "logps/rejected": -165.45309448242188,
      "loss": 0.3214,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6179537773132324,
      "rewards/margins": 1.422529697418213,
      "rewards/rejected": -0.8045759201049805,
      "step": 268
    },
    {
      "epoch": 1.5,
      "grad_norm": 2.0690940688924604,
      "learning_rate": 3.2307841983771176e-08,
      "logits/chosen": -1.8847911357879639,
      "logits/rejected": -2.0202267169952393,
      "logps/chosen": -200.80783081054688,
      "logps/rejected": -104.58744049072266,
      "loss": 0.3988,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2336064577102661,
      "rewards/margins": 0.8035344481468201,
      "rewards/rejected": -0.569927990436554,
      "step": 269
    },
    {
      "epoch": 1.51,
      "grad_norm": 3.361447315187423,
      "learning_rate": 3.1626638093641775e-08,
      "logits/chosen": -2.018540382385254,
      "logits/rejected": -2.115633726119995,
      "logps/chosen": -364.541259765625,
      "logps/rejected": -258.31201171875,
      "loss": 0.4431,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7286382913589478,
      "rewards/margins": 1.7735204696655273,
      "rewards/rejected": -1.0448822975158691,
      "step": 270
    },
    {
      "epoch": 1.51,
      "grad_norm": 2.2699381072918072,
      "learning_rate": 3.0951340974769036e-08,
      "logits/chosen": -2.064911365509033,
      "logits/rejected": -2.240605354309082,
      "logps/chosen": -236.48304748535156,
      "logps/rejected": -107.43959045410156,
      "loss": 0.3952,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.588148832321167,
      "rewards/margins": 1.1847357749938965,
      "rewards/rejected": -0.5965870022773743,
      "step": 271
    },
    {
      "epoch": 1.52,
      "grad_norm": 1.667750743463304,
      "learning_rate": 3.0282008966036645e-08,
      "logits/chosen": -1.9100630283355713,
      "logits/rejected": -2.129060983657837,
      "logps/chosen": -336.9149169921875,
      "logps/rejected": -199.61476135253906,
      "loss": 0.3072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.721794068813324,
      "rewards/margins": 1.6085577011108398,
      "rewards/rejected": -0.8867635726928711,
      "step": 272
    },
    {
      "epoch": 1.53,
      "grad_norm": 1.8755306754711736,
      "learning_rate": 2.961869989100284e-08,
      "logits/chosen": -1.973421573638916,
      "logits/rejected": -2.1563172340393066,
      "logps/chosen": -216.48602294921875,
      "logps/rejected": -124.95690155029297,
      "loss": 0.3827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5886920094490051,
      "rewards/margins": 1.2941111326217651,
      "rewards/rejected": -0.7054190635681152,
      "step": 273
    },
    {
      "epoch": 1.53,
      "grad_norm": 5.227217314587311,
      "learning_rate": 2.8961471052904852e-08,
      "logits/chosen": -2.024624824523926,
      "logits/rejected": -2.251070499420166,
      "logps/chosen": -281.59869384765625,
      "logps/rejected": -135.55624389648438,
      "loss": 0.4336,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7197825312614441,
      "rewards/margins": 1.5994011163711548,
      "rewards/rejected": -0.8796184659004211,
      "step": 274
    },
    {
      "epoch": 1.54,
      "grad_norm": 2.009469131089514,
      "learning_rate": 2.8310379229708548e-08,
      "logits/chosen": -1.9827313423156738,
      "logits/rejected": -2.371861457824707,
      "logps/chosen": -489.7880859375,
      "logps/rejected": -177.23590087890625,
      "loss": 0.3523,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9191684126853943,
      "rewards/margins": 1.5952149629592896,
      "rewards/rejected": -0.6760467886924744,
      "step": 275
    },
    {
      "epoch": 1.54,
      "grad_norm": 2.1048586567415457,
      "learning_rate": 2.7665480669203377e-08,
      "logits/chosen": -1.934632420539856,
      "logits/rejected": -2.1390960216522217,
      "logps/chosen": -299.7361145019531,
      "logps/rejected": -145.28750610351562,
      "loss": 0.2493,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7144760489463806,
      "rewards/margins": 1.7577918767929077,
      "rewards/rejected": -1.0433157682418823,
      "step": 276
    },
    {
      "epoch": 1.55,
      "grad_norm": 1.9269914350384307,
      "learning_rate": 2.7026831084143254e-08,
      "logits/chosen": -1.9214329719543457,
      "logits/rejected": -2.1084671020507812,
      "logps/chosen": -256.8978271484375,
      "logps/rejected": -152.1795196533203,
      "loss": 0.2297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6236655116081238,
      "rewards/margins": 1.38833487033844,
      "rewards/rejected": -0.7646692991256714,
      "step": 277
    },
    {
      "epoch": 1.55,
      "grad_norm": 2.0675257149998636,
      "learning_rate": 2.6394485647433273e-08,
      "logits/chosen": -1.9709903001785278,
      "logits/rejected": -2.265573501586914,
      "logps/chosen": -394.1219482421875,
      "logps/rejected": -185.23574829101562,
      "loss": 0.2808,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8027446269989014,
      "rewards/margins": 1.6347885131835938,
      "rewards/rejected": -0.8320440053939819,
      "step": 278
    },
    {
      "epoch": 1.56,
      "grad_norm": 2.788876731328824,
      "learning_rate": 2.5768498987363595e-08,
      "logits/chosen": -1.9366790056228638,
      "logits/rejected": -2.2282567024230957,
      "logps/chosen": -319.8380126953125,
      "logps/rejected": -102.86990356445312,
      "loss": 0.3694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8548070192337036,
      "rewards/margins": 1.4455407857894897,
      "rewards/rejected": -0.5907337665557861,
      "step": 279
    },
    {
      "epoch": 1.56,
      "grad_norm": 1.9890175505345056,
      "learning_rate": 2.514892518288988e-08,
      "logits/chosen": -1.9826221466064453,
      "logits/rejected": -2.222606658935547,
      "logps/chosen": -382.8952331542969,
      "logps/rejected": -154.84893798828125,
      "loss": 0.2993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.884526252746582,
      "rewards/margins": 1.8476271629333496,
      "rewards/rejected": -0.9631009101867676,
      "step": 280
    },
    {
      "epoch": 1.57,
      "grad_norm": 1.9892914243896007,
      "learning_rate": 2.453581775896164e-08,
      "logits/chosen": -1.8972669839859009,
      "logits/rejected": -2.1901392936706543,
      "logps/chosen": -385.07183837890625,
      "logps/rejected": -140.5347137451172,
      "loss": 0.2683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8599714636802673,
      "rewards/margins": 1.7295414209365845,
      "rewards/rejected": -0.8695698380470276,
      "step": 281
    },
    {
      "epoch": 1.58,
      "grad_norm": 3.1227195996097796,
      "learning_rate": 2.3929229681898e-08,
      "logits/chosen": -1.9234641790390015,
      "logits/rejected": -2.137057304382324,
      "logps/chosen": -378.1383361816406,
      "logps/rejected": -225.235107421875,
      "loss": 0.3887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6592192053794861,
      "rewards/margins": 1.4562088251113892,
      "rewards/rejected": -0.7969896793365479,
      "step": 282
    },
    {
      "epoch": 1.58,
      "grad_norm": 2.259095062423167,
      "learning_rate": 2.3329213354812048e-08,
      "logits/chosen": -1.8679221868515015,
      "logits/rejected": -2.045198678970337,
      "logps/chosen": -308.4653625488281,
      "logps/rejected": -166.0535430908203,
      "loss": 0.3296,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.636101245880127,
      "rewards/margins": 1.3980377912521362,
      "rewards/rejected": -0.7619364857673645,
      "step": 283
    },
    {
      "epoch": 1.59,
      "grad_norm": 2.048149379545429,
      "learning_rate": 2.2735820613083835e-08,
      "logits/chosen": -1.9634499549865723,
      "logits/rejected": -2.1904802322387695,
      "logps/chosen": -245.1457061767578,
      "logps/rejected": -68.16486358642578,
      "loss": 0.348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4485977292060852,
      "rewards/margins": 0.9627138376235962,
      "rewards/rejected": -0.514116108417511,
      "step": 284
    },
    {
      "epoch": 1.59,
      "grad_norm": 1.887396473086629,
      "learning_rate": 2.214910271988204e-08,
      "logits/chosen": -1.9823853969573975,
      "logits/rejected": -2.187169075012207,
      "logps/chosen": -291.22772216796875,
      "logps/rejected": -149.49351501464844,
      "loss": 0.3743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3778786063194275,
      "rewards/margins": 1.0260906219482422,
      "rewards/rejected": -0.6482120156288147,
      "step": 285
    },
    {
      "epoch": 1.6,
      "grad_norm": 2.3234230452821976,
      "learning_rate": 2.1569110361735675e-08,
      "logits/chosen": -2.0167317390441895,
      "logits/rejected": -2.172858238220215,
      "logps/chosen": -277.48248291015625,
      "logps/rejected": -144.8683319091797,
      "loss": 0.3099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5940869450569153,
      "rewards/margins": 1.403023362159729,
      "rewards/rejected": -0.8089364171028137,
      "step": 286
    },
    {
      "epoch": 1.6,
      "grad_norm": 2.5446773204041664,
      "learning_rate": 2.0995893644155004e-08,
      "logits/chosen": -1.9866153001785278,
      "logits/rejected": -2.0847387313842773,
      "logps/chosen": -278.55133056640625,
      "logps/rejected": -204.3761444091797,
      "loss": 0.377,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11578173190355301,
      "rewards/margins": 0.541024923324585,
      "rewards/rejected": -0.42524319887161255,
      "step": 287
    },
    {
      "epoch": 1.61,
      "grad_norm": 1.8357875496740423,
      "learning_rate": 2.0429502087303165e-08,
      "logits/chosen": -1.9452314376831055,
      "logits/rejected": -2.0776212215423584,
      "logps/chosen": -377.2801208496094,
      "logps/rejected": -231.69754028320312,
      "loss": 0.3189,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7734673023223877,
      "rewards/margins": 1.5090703964233398,
      "rewards/rejected": -0.7356030941009521,
      "step": 288
    },
    {
      "epoch": 1.61,
      "grad_norm": 2.592831499188782,
      "learning_rate": 1.9869984621717882e-08,
      "logits/chosen": -1.886064887046814,
      "logits/rejected": -2.187819719314575,
      "logps/chosen": -343.24664306640625,
      "logps/rejected": -130.65252685546875,
      "loss": 0.4297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6091063618659973,
      "rewards/margins": 1.680466651916504,
      "rewards/rejected": -1.0713602304458618,
      "step": 289
    },
    {
      "epoch": 1.62,
      "grad_norm": 1.8632324855903686,
      "learning_rate": 1.9317389584084564e-08,
      "logits/chosen": -2.046363592147827,
      "logits/rejected": -2.3104305267333984,
      "logps/chosen": -275.2044372558594,
      "logps/rejected": -81.00444030761719,
      "loss": 0.3841,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.362895131111145,
      "rewards/margins": 2.0078790187835693,
      "rewards/rejected": -0.6449840068817139,
      "step": 290
    },
    {
      "epoch": 1.63,
      "grad_norm": 2.155918100213792,
      "learning_rate": 1.8771764713060355e-08,
      "logits/chosen": -1.9281926155090332,
      "logits/rejected": -2.106454610824585,
      "logps/chosen": -272.47381591796875,
      "logps/rejected": -113.74870300292969,
      "loss": 0.2838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8106430768966675,
      "rewards/margins": 1.7507152557373047,
      "rewards/rejected": -0.9400720596313477,
      "step": 291
    },
    {
      "epoch": 1.63,
      "grad_norm": 2.1800107546257763,
      "learning_rate": 1.823315714515018e-08,
      "logits/chosen": -1.9668960571289062,
      "logits/rejected": -2.028254508972168,
      "logps/chosen": -374.9870300292969,
      "logps/rejected": -222.8441162109375,
      "loss": 0.3292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5992096662521362,
      "rewards/margins": 1.4854466915130615,
      "rewards/rejected": -0.8862370848655701,
      "step": 292
    },
    {
      "epoch": 1.64,
      "grad_norm": 2.0467928150919388,
      "learning_rate": 1.7701613410634364e-08,
      "logits/chosen": -2.026965856552124,
      "logits/rejected": -2.1441550254821777,
      "logps/chosen": -362.87579345703125,
      "logps/rejected": -164.18157958984375,
      "loss": 0.3324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6154202818870544,
      "rewards/margins": 1.6078412532806396,
      "rewards/rejected": -0.9924209117889404,
      "step": 293
    },
    {
      "epoch": 1.64,
      "grad_norm": 1.771060411498593,
      "learning_rate": 1.7177179429549138e-08,
      "logits/chosen": -1.9229191541671753,
      "logits/rejected": -2.0828027725219727,
      "logps/chosen": -307.8000793457031,
      "logps/rejected": -152.2085723876953,
      "loss": 0.3442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5766472220420837,
      "rewards/margins": 1.5805346965789795,
      "rewards/rejected": -1.0038875341415405,
      "step": 294
    },
    {
      "epoch": 1.65,
      "grad_norm": 2.1477749219966182,
      "learning_rate": 1.6659900507719403e-08,
      "logits/chosen": -1.956030011177063,
      "logits/rejected": -2.126971960067749,
      "logps/chosen": -277.1318664550781,
      "logps/rejected": -143.57424926757812,
      "loss": 0.4088,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5368402600288391,
      "rewards/margins": 1.3292208909988403,
      "rewards/rejected": -0.7923805713653564,
      "step": 295
    },
    {
      "epoch": 1.65,
      "grad_norm": 2.0607954307134566,
      "learning_rate": 1.614982133284495e-08,
      "logits/chosen": -1.9652516841888428,
      "logits/rejected": -2.0515949726104736,
      "logps/chosen": -107.14926147460938,
      "logps/rejected": -52.71076583862305,
      "loss": 0.3249,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14517386257648468,
      "rewards/margins": 0.4607216417789459,
      "rewards/rejected": -0.31554779410362244,
      "step": 296
    },
    {
      "epoch": 1.66,
      "grad_norm": 2.1416211825003013,
      "learning_rate": 1.5646985970639714e-08,
      "logits/chosen": -1.8579660654067993,
      "logits/rejected": -2.1249003410339355,
      "logps/chosen": -384.6595153808594,
      "logps/rejected": -188.4464111328125,
      "loss": 0.3381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4016474485397339,
      "rewards/margins": 1.1661272048950195,
      "rewards/rejected": -0.7644796967506409,
      "step": 297
    },
    {
      "epoch": 1.66,
      "grad_norm": 2.34908271482458,
      "learning_rate": 1.515143786102503e-08,
      "logits/chosen": -1.8695746660232544,
      "logits/rejected": -2.0640392303466797,
      "logps/chosen": -237.24574279785156,
      "logps/rejected": -136.90809631347656,
      "loss": 0.375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2771724462509155,
      "rewards/margins": 0.8384150862693787,
      "rewards/rejected": -0.5612426400184631,
      "step": 298
    },
    {
      "epoch": 1.67,
      "grad_norm": 2.4924316107175666,
      "learning_rate": 1.4663219814376938e-08,
      "logits/chosen": -2.012023687362671,
      "logits/rejected": -2.0818071365356445,
      "logps/chosen": -273.9322814941406,
      "logps/rejected": -240.0913848876953,
      "loss": 0.3164,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.328775554895401,
      "rewards/margins": 0.7493981122970581,
      "rewards/rejected": -0.4206225872039795,
      "step": 299
    },
    {
      "epoch": 1.68,
      "grad_norm": 2.804749956277777,
      "learning_rate": 1.4182374007827603e-08,
      "logits/chosen": -1.9487468004226685,
      "logits/rejected": -2.167121410369873,
      "logps/chosen": -270.26507568359375,
      "logps/rejected": -154.2471923828125,
      "loss": 0.4296,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2056507021188736,
      "rewards/margins": 0.027986496686935425,
      "rewards/rejected": -0.2336372286081314,
      "step": 300
    },
    {
      "epoch": 1.68,
      "eval_logits/chosen": -1.9295151233673096,
      "eval_logits/rejected": -2.141464948654175,
      "eval_logps/chosen": -316.00396728515625,
      "eval_logps/rejected": -144.11239624023438,
      "eval_loss": 0.32466286420822144,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": 0.6522671580314636,
      "eval_rewards/margins": 1.4982246160507202,
      "eval_rewards/rejected": -0.8459575772285461,
      "eval_runtime": 186.691,
      "eval_samples_per_second": 6.819,
      "eval_steps_per_second": 0.857,
      "step": 300
    },
    {
      "epoch": 1.68,
      "grad_norm": 1.9411042495172686,
      "learning_rate": 1.3708941981621814e-08,
      "logits/chosen": -1.9763463735580444,
      "logits/rejected": -2.107821464538574,
      "logps/chosen": -155.54547119140625,
      "logps/rejected": -65.61723327636719,
      "loss": 0.3496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.37429141998291016,
      "rewards/margins": 0.8814553022384644,
      "rewards/rejected": -0.507163941860199,
      "step": 301
    },
    {
      "epoch": 1.69,
      "grad_norm": 2.349563136392854,
      "learning_rate": 1.3242964635528208e-08,
      "logits/chosen": -1.9118015766143799,
      "logits/rejected": -2.0467817783355713,
      "logps/chosen": -196.6197052001953,
      "logps/rejected": -72.90583038330078,
      "loss": 0.4717,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.455363929271698,
      "rewards/margins": 0.8942562341690063,
      "rewards/rejected": -0.43889227509498596,
      "step": 302
    },
    {
      "epoch": 1.69,
      "grad_norm": 2.0443074078432297,
      "learning_rate": 1.278448222530606e-08,
      "logits/chosen": -1.822318196296692,
      "logits/rejected": -2.1045279502868652,
      "logps/chosen": -255.2604522705078,
      "logps/rejected": -105.29451751708984,
      "loss": 0.3565,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7090134620666504,
      "rewards/margins": 1.4576520919799805,
      "rewards/rejected": -0.7486386895179749,
      "step": 303
    },
    {
      "epoch": 1.7,
      "grad_norm": 2.329468016744532,
      "learning_rate": 1.2333534359227383e-08,
      "logits/chosen": -1.9810900688171387,
      "logits/rejected": -1.9879897832870483,
      "logps/chosen": -187.4176025390625,
      "logps/rejected": -137.21522521972656,
      "loss": 0.3768,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21685890853405,
      "rewards/margins": 0.7572988867759705,
      "rewards/rejected": -0.5404399037361145,
      "step": 304
    },
    {
      "epoch": 1.7,
      "grad_norm": 2.4935403946193335,
      "learning_rate": 1.1890159994655425e-08,
      "logits/chosen": -1.9184632301330566,
      "logits/rejected": -2.1819629669189453,
      "logps/chosen": -317.76031494140625,
      "logps/rejected": -118.99726104736328,
      "loss": 0.3979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6526443958282471,
      "rewards/margins": 1.706324815750122,
      "rewards/rejected": -1.053680419921875,
      "step": 305
    },
    {
      "epoch": 1.71,
      "grad_norm": 2.4833278286779588,
      "learning_rate": 1.145439743467902e-08,
      "logits/chosen": -1.944310188293457,
      "logits/rejected": -2.1891067028045654,
      "logps/chosen": -392.849853515625,
      "logps/rejected": -190.91070556640625,
      "loss": 0.3401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5746330618858337,
      "rewards/margins": 1.7914217710494995,
      "rewards/rejected": -1.2167887687683105,
      "step": 306
    },
    {
      "epoch": 1.72,
      "grad_norm": 1.8943188797272705,
      "learning_rate": 1.1026284324803492e-08,
      "logits/chosen": -1.9464324712753296,
      "logits/rejected": -2.071516990661621,
      "logps/chosen": -250.5147247314453,
      "logps/rejected": -113.3388442993164,
      "loss": 0.3642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5535628199577332,
      "rewards/margins": 1.0718276500701904,
      "rewards/rejected": -0.5182647109031677,
      "step": 307
    },
    {
      "epoch": 1.72,
      "grad_norm": 2.8544492338017693,
      "learning_rate": 1.0605857649698669e-08,
      "logits/chosen": -1.902798056602478,
      "logits/rejected": -2.297213077545166,
      "logps/chosen": -473.9614562988281,
      "logps/rejected": -205.9657745361328,
      "loss": 0.347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6481414437294006,
      "rewards/margins": 1.6791346073150635,
      "rewards/rejected": -1.030993103981018,
      "step": 308
    },
    {
      "epoch": 1.73,
      "grad_norm": 2.0036521120268076,
      "learning_rate": 1.0193153730003601e-08,
      "logits/chosen": -1.9088249206542969,
      "logits/rejected": -2.0862650871276855,
      "logps/chosen": -252.01986694335938,
      "logps/rejected": -133.6540985107422,
      "loss": 0.3304,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.459441214799881,
      "rewards/margins": 0.782798171043396,
      "rewards/rejected": -0.3233569860458374,
      "step": 309
    },
    {
      "epoch": 1.73,
      "grad_norm": 2.7913507857395947,
      "learning_rate": 9.78820821918893e-09,
      "logits/chosen": -1.8910510540008545,
      "logits/rejected": -2.1956124305725098,
      "logps/chosen": -254.30776977539062,
      "logps/rejected": -76.68388366699219,
      "loss": 0.4119,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5045368671417236,
      "rewards/margins": 1.1815844774246216,
      "rewards/rejected": -0.677047610282898,
      "step": 310
    },
    {
      "epoch": 1.74,
      "grad_norm": 2.565223836267683,
      "learning_rate": 9.391056100476735e-09,
      "logits/chosen": -1.951928973197937,
      "logits/rejected": -2.1905078887939453,
      "logps/chosen": -359.3843994140625,
      "logps/rejected": -189.37869262695312,
      "loss": 0.3387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8327935934066772,
      "rewards/margins": 1.7248027324676514,
      "rewards/rejected": -0.8920091390609741,
      "step": 311
    },
    {
      "epoch": 1.74,
      "grad_norm": 2.519209117959205,
      "learning_rate": 9.001731683818336e-09,
      "logits/chosen": -1.8875353336334229,
      "logits/rejected": -2.0544509887695312,
      "logps/chosen": -331.9053955078125,
      "logps/rejected": -170.03109741210938,
      "loss": 0.3695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8446786403656006,
      "rewards/margins": 1.7915393114089966,
      "rewards/rejected": -0.946860671043396,
      "step": 312
    },
    {
      "epoch": 1.75,
      "grad_norm": 1.7585924805588735,
      "learning_rate": 8.620268602930269e-09,
      "logits/chosen": -1.9674144983291626,
      "logits/rejected": -1.948687195777893,
      "logps/chosen": -274.6681823730469,
      "logps/rejected": -228.2263641357422,
      "loss": 0.2954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7268482446670532,
      "rewards/margins": 1.5142990350723267,
      "rewards/rejected": -0.7874506711959839,
      "step": 313
    },
    {
      "epoch": 1.75,
      "grad_norm": 2.314971302336536,
      "learning_rate": 8.246699812388713e-09,
      "logits/chosen": -1.9341325759887695,
      "logits/rejected": -2.092617988586426,
      "logps/chosen": -338.190673828125,
      "logps/rejected": -150.9620819091797,
      "loss": 0.3171,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7337810397148132,
      "rewards/margins": 1.5115951299667358,
      "rewards/rejected": -0.7778141498565674,
      "step": 314
    },
    {
      "epoch": 1.76,
      "grad_norm": 2.0178597083087175,
      "learning_rate": 7.881057584782447e-09,
      "logits/chosen": -1.9406661987304688,
      "logits/rejected": -2.1710622310638428,
      "logps/chosen": -285.05120849609375,
      "logps/rejected": -94.08568572998047,
      "loss": 0.385,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5303260684013367,
      "rewards/margins": 1.2865056991577148,
      "rewards/rejected": -0.7561795711517334,
      "step": 315
    },
    {
      "epoch": 1.77,
      "grad_norm": 2.2008626151145534,
      "learning_rate": 7.523373507924947e-09,
      "logits/chosen": -1.8313828706741333,
      "logits/rejected": -2.026157855987549,
      "logps/chosen": -331.306884765625,
      "logps/rejected": -151.93821716308594,
      "loss": 0.3016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.559520959854126,
      "rewards/margins": 1.3658560514450073,
      "rewards/rejected": -0.806334912776947,
      "step": 316
    },
    {
      "epoch": 1.77,
      "grad_norm": 2.3365923631145225,
      "learning_rate": 7.173678482125389e-09,
      "logits/chosen": -2.0174169540405273,
      "logits/rejected": -2.2585856914520264,
      "logps/chosen": -267.46875,
      "logps/rejected": -115.35077667236328,
      "loss": 0.4006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4813220202922821,
      "rewards/margins": 1.1780027151107788,
      "rewards/rejected": -0.6966806650161743,
      "step": 317
    },
    {
      "epoch": 1.78,
      "grad_norm": 2.481179658279427,
      "learning_rate": 6.83200271751927e-09,
      "logits/chosen": -1.8875877857208252,
      "logits/rejected": -2.227430820465088,
      "logps/chosen": -371.4129638671875,
      "logps/rejected": -164.30038452148438,
      "loss": 0.36,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8697217106819153,
      "rewards/margins": 1.7959797382354736,
      "rewards/rejected": -0.9262580275535583,
      "step": 318
    },
    {
      "epoch": 1.78,
      "grad_norm": 1.8970176783719166,
      "learning_rate": 6.498375731458528e-09,
      "logits/chosen": -1.9906672239303589,
      "logits/rejected": -2.092620611190796,
      "logps/chosen": -159.76637268066406,
      "logps/rejected": -73.02427673339844,
      "loss": 0.3725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.289163202047348,
      "rewards/margins": 0.7067059874534607,
      "rewards/rejected": -0.41754278540611267,
      "step": 319
    },
    {
      "epoch": 1.79,
      "grad_norm": 2.0293931798898868,
      "learning_rate": 6.172826345961479e-09,
      "logits/chosen": -1.8877040147781372,
      "logits/rejected": -2.1244630813598633,
      "logps/chosen": -468.999755859375,
      "logps/rejected": -173.8732147216797,
      "loss": 0.3367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5770574808120728,
      "rewards/margins": 1.7707054615020752,
      "rewards/rejected": -1.193647861480713,
      "step": 320
    },
    {
      "epoch": 1.79,
      "grad_norm": 2.505226667316074,
      "learning_rate": 5.8553826852230095e-09,
      "logits/chosen": -1.9780012369155884,
      "logits/rejected": -2.1993906497955322,
      "logps/chosen": -371.938720703125,
      "logps/rejected": -178.89736938476562,
      "loss": 0.3714,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.41208526492118835,
      "rewards/margins": 1.3043644428253174,
      "rewards/rejected": -0.8922792077064514,
      "step": 321
    },
    {
      "epoch": 1.8,
      "grad_norm": 1.6481034388278855,
      "learning_rate": 5.54607217318479e-09,
      "logits/chosen": -1.971070408821106,
      "logits/rejected": -2.187108278274536,
      "logps/chosen": -375.25604248046875,
      "logps/rejected": -157.79771423339844,
      "loss": 0.299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4872535765171051,
      "rewards/margins": 1.6334389448165894,
      "rewards/rejected": -1.146185278892517,
      "step": 322
    },
    {
      "epoch": 1.8,
      "grad_norm": 2.4584799350769444,
      "learning_rate": 5.244921531166246e-09,
      "logits/chosen": -1.9654029607772827,
      "logits/rejected": -2.007652997970581,
      "logps/chosen": -234.24276733398438,
      "logps/rejected": -158.03627014160156,
      "loss": 0.3454,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5839539766311646,
      "rewards/margins": 1.3285213708877563,
      "rewards/rejected": -0.744567334651947,
      "step": 323
    },
    {
      "epoch": 1.81,
      "grad_norm": 2.5817479172502,
      "learning_rate": 4.951956775555999e-09,
      "logits/chosen": -2.014432430267334,
      "logits/rejected": -2.1201930046081543,
      "logps/chosen": -320.6275634765625,
      "logps/rejected": -213.67913818359375,
      "loss": 0.3529,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1901564747095108,
      "rewards/margins": 0.7247769236564636,
      "rewards/rejected": -0.534620463848114,
      "step": 324
    },
    {
      "epoch": 1.82,
      "grad_norm": 2.0825760729013036,
      "learning_rate": 4.6672032155644305e-09,
      "logits/chosen": -1.882224202156067,
      "logits/rejected": -1.9761323928833008,
      "logps/chosen": -116.42622375488281,
      "logps/rejected": -46.48798370361328,
      "loss": 0.329,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17195764183998108,
      "rewards/margins": 0.4231622815132141,
      "rewards/rejected": -0.2512046992778778,
      "step": 325
    },
    {
      "epoch": 1.82,
      "grad_norm": 2.080471104569788,
      "learning_rate": 4.390685451037024e-09,
      "logits/chosen": -1.9777361154556274,
      "logits/rejected": -2.40352201461792,
      "logps/chosen": -445.56231689453125,
      "logps/rejected": -186.0106201171875,
      "loss": 0.3198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8660315275192261,
      "rewards/margins": 2.2276761531829834,
      "rewards/rejected": -1.3616446256637573,
      "step": 326
    },
    {
      "epoch": 1.83,
      "grad_norm": 1.6682834833589366,
      "learning_rate": 4.122427370329451e-09,
      "logits/chosen": -1.9932342767715454,
      "logits/rejected": -2.1058011054992676,
      "logps/chosen": -290.7546691894531,
      "logps/rejected": -160.9483642578125,
      "loss": 0.257,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7041120529174805,
      "rewards/margins": 1.5494294166564941,
      "rewards/rejected": -0.8453173637390137,
      "step": 327
    },
    {
      "epoch": 1.83,
      "grad_norm": 2.4402456764277964,
      "learning_rate": 3.862452148243622e-09,
      "logits/chosen": -1.8214092254638672,
      "logits/rejected": -2.0429835319519043,
      "logps/chosen": -294.1298828125,
      "logps/rejected": -129.96456909179688,
      "loss": 0.3295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3714965283870697,
      "rewards/margins": 1.1992201805114746,
      "rewards/rejected": -0.8277237415313721,
      "step": 328
    },
    {
      "epoch": 1.84,
      "grad_norm": 2.34982397767895,
      "learning_rate": 3.6107822440257675e-09,
      "logits/chosen": -2.0181312561035156,
      "logits/rejected": -2.2349228858947754,
      "logps/chosen": -374.7834167480469,
      "logps/rejected": -183.97332763671875,
      "loss": 0.3781,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7463111281394958,
      "rewards/margins": 1.571717381477356,
      "rewards/rejected": -0.8254062533378601,
      "step": 329
    },
    {
      "epoch": 1.84,
      "grad_norm": 3.505631357350379,
      "learning_rate": 3.3674393994260863e-09,
      "logits/chosen": -1.9698312282562256,
      "logits/rejected": -2.2149014472961426,
      "logps/chosen": -351.15777587890625,
      "logps/rejected": -143.33502197265625,
      "loss": 0.4017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6873552799224854,
      "rewards/margins": 1.7417612075805664,
      "rewards/rejected": -1.0544058084487915,
      "step": 330
    },
    {
      "epoch": 1.85,
      "grad_norm": 2.45112122987896,
      "learning_rate": 3.132444636820575e-09,
      "logits/chosen": -1.9869520664215088,
      "logits/rejected": -2.1476073265075684,
      "logps/chosen": -330.85552978515625,
      "logps/rejected": -184.3342742919922,
      "loss": 0.3518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6776991486549377,
      "rewards/margins": 1.1076922416687012,
      "rewards/rejected": -0.4299930930137634,
      "step": 331
    },
    {
      "epoch": 1.85,
      "grad_norm": 2.18016599869388,
      "learning_rate": 2.9058182573947987e-09,
      "logits/chosen": -1.9990299940109253,
      "logits/rejected": -2.189465284347534,
      "logps/chosen": -263.61297607421875,
      "logps/rejected": -127.04591369628906,
      "loss": 0.3608,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4782324433326721,
      "rewards/margins": 1.0925434827804565,
      "rewards/rejected": -0.6143109798431396,
      "step": 332
    },
    {
      "epoch": 1.86,
      "grad_norm": 5.079415618213185,
      "learning_rate": 2.687579839390153e-09,
      "logits/chosen": -2.04642915725708,
      "logits/rejected": -2.0504298210144043,
      "logps/chosen": -368.04278564453125,
      "logps/rejected": -330.5069580078125,
      "loss": 0.4208,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3292311429977417,
      "rewards/margins": 0.7461775541305542,
      "rewards/rejected": -0.4169463515281677,
      "step": 333
    },
    {
      "epoch": 1.87,
      "grad_norm": 2.4400731985711492,
      "learning_rate": 2.477748236412469e-09,
      "logits/chosen": -1.8668960332870483,
      "logits/rejected": -2.124622106552124,
      "logps/chosen": -311.70550537109375,
      "logps/rejected": -141.21267700195312,
      "loss": 0.3444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6067173480987549,
      "rewards/margins": 1.432044267654419,
      "rewards/rejected": -0.8253269195556641,
      "step": 334
    },
    {
      "epoch": 1.87,
      "grad_norm": 2.1709665403473495,
      "learning_rate": 2.2763415758032314e-09,
      "logits/chosen": -2.0512101650238037,
      "logits/rejected": -2.173598051071167,
      "logps/chosen": -420.999267578125,
      "logps/rejected": -267.1089172363281,
      "loss": 0.373,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9176816940307617,
      "rewards/margins": 1.998265266418457,
      "rewards/rejected": -1.0805835723876953,
      "step": 335
    },
    {
      "epoch": 1.88,
      "grad_norm": 2.434479209137729,
      "learning_rate": 2.0833772570736373e-09,
      "logits/chosen": -1.9979689121246338,
      "logits/rejected": -2.113584518432617,
      "logps/chosen": -347.56341552734375,
      "logps/rejected": -191.31202697753906,
      "loss": 0.381,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.769294023513794,
      "rewards/margins": 1.7911299467086792,
      "rewards/rejected": -1.0218359231948853,
      "step": 336
    },
    {
      "epoch": 1.88,
      "grad_norm": 2.3107985454840714,
      "learning_rate": 1.898871950401337e-09,
      "logits/chosen": -1.9912981986999512,
      "logits/rejected": -2.03279709815979,
      "logps/chosen": -330.6841735839844,
      "logps/rejected": -269.5788879394531,
      "loss": 0.3441,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5533915162086487,
      "rewards/margins": 1.3993961811065674,
      "rewards/rejected": -0.8460047245025635,
      "step": 337
    },
    {
      "epoch": 1.89,
      "grad_norm": 2.1169045580497508,
      "learning_rate": 1.7228415951904163e-09,
      "logits/chosen": -1.9560718536376953,
      "logits/rejected": -2.2123360633850098,
      "logps/chosen": -327.4252014160156,
      "logps/rejected": -157.6983184814453,
      "loss": 0.3534,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8153419494628906,
      "rewards/margins": 1.6094058752059937,
      "rewards/rejected": -0.7940639853477478,
      "step": 338
    },
    {
      "epoch": 1.89,
      "grad_norm": 2.0334452969637873,
      "learning_rate": 1.5553013986942642e-09,
      "logits/chosen": -2.0031869411468506,
      "logits/rejected": -2.055514335632324,
      "logps/chosen": -245.74737548828125,
      "logps/rejected": -185.9472198486328,
      "loss": 0.3302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3879874646663666,
      "rewards/margins": 0.9499048590660095,
      "rewards/rejected": -0.5619173645973206,
      "step": 339
    },
    {
      "epoch": 1.9,
      "grad_norm": 2.361728211596182,
      "learning_rate": 1.3962658347019819e-09,
      "logits/chosen": -2.008934497833252,
      "logits/rejected": -2.1810312271118164,
      "logps/chosen": -275.261962890625,
      "logps/rejected": -139.23110961914062,
      "loss": 0.3062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5827020406723022,
      "rewards/margins": 1.514220952987671,
      "rewards/rejected": -0.9315187335014343,
      "step": 340
    },
    {
      "epoch": 1.91,
      "grad_norm": 2.4538771688499867,
      "learning_rate": 1.2457486422878138e-09,
      "logits/chosen": -1.9547114372253418,
      "logits/rejected": -2.1434385776519775,
      "logps/chosen": -314.6387023925781,
      "logps/rejected": -157.8639678955078,
      "loss": 0.3439,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2895660400390625,
      "rewards/margins": 0.775239109992981,
      "rewards/rejected": -0.48567306995391846,
      "step": 341
    },
    {
      "epoch": 1.91,
      "grad_norm": 1.783407147587096,
      "learning_rate": 1.1037628246243769e-09,
      "logits/chosen": -1.960133671760559,
      "logits/rejected": -2.021034002304077,
      "logps/chosen": -156.89002990722656,
      "logps/rejected": -101.70440673828125,
      "loss": 0.3248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2527593970298767,
      "rewards/margins": 0.6134127378463745,
      "rewards/rejected": -0.3606533706188202,
      "step": 342
    },
    {
      "epoch": 1.92,
      "grad_norm": 1.9545553326476837,
      "learning_rate": 9.70320647859213e-10,
      "logits/chosen": -1.9362705945968628,
      "logits/rejected": -2.137017250061035,
      "logps/chosen": -236.40830993652344,
      "logps/rejected": -113.88504791259766,
      "loss": 0.2863,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.44171327352523804,
      "rewards/margins": 1.3618862628936768,
      "rewards/rejected": -0.9201730489730835,
      "step": 343
    },
    {
      "epoch": 1.92,
      "grad_norm": 2.611315971432463,
      "learning_rate": 8.454336400552154e-10,
      "logits/chosen": -1.94658625125885,
      "logits/rejected": -2.1128108501434326,
      "logps/chosen": -318.1528625488281,
      "logps/rejected": -129.91864013671875,
      "loss": 0.3379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.557726263999939,
      "rewards/margins": 1.4944050312042236,
      "rewards/rejected": -0.9366787672042847,
      "step": 344
    },
    {
      "epoch": 1.93,
      "grad_norm": 1.982035369334317,
      "learning_rate": 7.291125901946027e-10,
      "logits/chosen": -1.9644968509674072,
      "logits/rejected": -2.26253604888916,
      "logps/chosen": -368.5287780761719,
      "logps/rejected": -166.108154296875,
      "loss": 0.281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8076134324073792,
      "rewards/margins": 1.7294673919677734,
      "rewards/rejected": -0.9218538999557495,
      "step": 345
    },
    {
      "epoch": 1.93,
      "grad_norm": 2.52608570768576,
      "learning_rate": 6.21367547246976e-10,
      "logits/chosen": -1.9160689115524292,
      "logits/rejected": -2.1490960121154785,
      "logps/chosen": -239.294189453125,
      "logps/rejected": -85.73963165283203,
      "loss": 0.3805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7477155923843384,
      "rewards/margins": 1.5674340724945068,
      "rewards/rejected": -0.819718599319458,
      "step": 346
    },
    {
      "epoch": 1.94,
      "grad_norm": 2.319706584271849,
      "learning_rate": 5.222078193011125e-10,
      "logits/chosen": -1.9619083404541016,
      "logits/rejected": -2.0621178150177,
      "logps/chosen": -207.6097869873047,
      "logps/rejected": -122.38450622558594,
      "loss": 0.4051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3802430033683777,
      "rewards/margins": 0.986553966999054,
      "rewards/rejected": -0.6063109040260315,
      "step": 347
    },
    {
      "epoch": 1.94,
      "grad_norm": 2.3736151183896417,
      "learning_rate": 4.3164197276084336e-10,
      "logits/chosen": -1.9763154983520508,
      "logits/rejected": -2.198726177215576,
      "logps/chosen": -359.36981201171875,
      "logps/rejected": -161.1787109375,
      "loss": 0.3248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6110528707504272,
      "rewards/margins": 1.4726297855377197,
      "rewards/rejected": -0.861577033996582,
      "step": 348
    },
    {
      "epoch": 1.95,
      "grad_norm": 1.828908050044205,
      "learning_rate": 3.4967783160507747e-10,
      "logits/chosen": -1.923030972480774,
      "logits/rejected": -2.0622808933258057,
      "logps/chosen": -292.14422607421875,
      "logps/rejected": -191.8591766357422,
      "loss": 0.3958,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6675036549568176,
      "rewards/margins": 1.3652247190475464,
      "rewards/rejected": -0.6977209448814392,
      "step": 349
    },
    {
      "epoch": 1.96,
      "grad_norm": 2.6272657192149906,
      "learning_rate": 2.7632247671177665e-10,
      "logits/chosen": -1.8597277402877808,
      "logits/rejected": -2.125450849533081,
      "logps/chosen": -281.3786926269531,
      "logps/rejected": -157.38714599609375,
      "loss": 0.452,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2373468577861786,
      "rewards/margins": 0.5166945457458496,
      "rewards/rejected": -0.27934765815734863,
      "step": 350
    },
    {
      "epoch": 1.96,
      "eval_logits/chosen": -1.929107666015625,
      "eval_logits/rejected": -2.14176344871521,
      "eval_logps/chosen": -316.05657958984375,
      "eval_logps/rejected": -144.36167907714844,
      "eval_loss": 0.3238605260848999,
      "eval_rewards/accuracies": 0.9437500238418579,
      "eval_rewards/chosen": 0.6506881713867188,
      "eval_rewards/margins": 1.5041240453720093,
      "eval_rewards/rejected": -0.8534358739852905,
      "eval_runtime": 187.279,
      "eval_samples_per_second": 6.797,
      "eval_steps_per_second": 0.854,
      "step": 350
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 358,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "total_flos": 33870165049344.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}