{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.9950825430277486,
  "eval_steps": 10000,
  "global_step": 710,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.03,
      "learning_rate": 1.4084507042253522e-07,
      "logits/chosen": -0.024068962782621384,
      "logits/rejected": 0.04506533965468407,
      "logps/chosen": -317.1720275878906,
      "logps/rejected": -207.3963623046875,
      "loss": 0.6933,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.0009543737396597862,
      "rewards/margins": -0.0014725492801517248,
      "rewards/rejected": 0.0005181756569072604,
      "step": 10
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.8169014084507043e-07,
      "logits/chosen": -0.04674551263451576,
      "logits/rejected": -0.010908829048275948,
      "logps/chosen": -294.0361328125,
      "logps/rejected": -201.79061889648438,
      "loss": 0.6914,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.0003093578852713108,
      "rewards/margins": 0.004586204886436462,
      "rewards/rejected": -0.004276847001165152,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.225352112676056e-07,
      "logits/chosen": -0.04512999951839447,
      "logits/rejected": 0.007197662256658077,
      "logps/chosen": -361.146240234375,
      "logps/rejected": -249.73080444335938,
      "loss": 0.6841,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.001304158358834684,
      "rewards/margins": 0.015663906931877136,
      "rewards/rejected": -0.01696806587278843,
      "step": 30
    },
    {
      "epoch": 0.11,
      "learning_rate": 5.633802816901409e-07,
      "logits/chosen": -0.04949279874563217,
      "logits/rejected": 0.00024662315263412893,
      "logps/chosen": -291.42913818359375,
      "logps/rejected": -196.7120361328125,
      "loss": 0.6759,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.015403245575726032,
      "rewards/margins": 0.05249834060668945,
      "rewards/rejected": -0.06790158897638321,
      "step": 40
    },
    {
      "epoch": 0.14,
      "learning_rate": 7.04225352112676e-07,
      "logits/chosen": 0.02841530181467533,
      "logits/rejected": 0.0736595019698143,
      "logps/chosen": -373.4976806640625,
      "logps/rejected": -227.70068359375,
      "loss": 0.665,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.05390547588467598,
      "rewards/margins": 0.11358483880758286,
      "rewards/rejected": -0.16749031841754913,
      "step": 50
    },
    {
      "epoch": 0.17,
      "learning_rate": 8.450704225352112e-07,
      "logits/chosen": -0.03540544956922531,
      "logits/rejected": 0.025740886107087135,
      "logps/chosen": -323.0583801269531,
      "logps/rejected": -230.3257293701172,
      "loss": 0.6567,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.03229106217622757,
      "rewards/margins": 0.1074444055557251,
      "rewards/rejected": -0.13973546028137207,
      "step": 60
    },
    {
      "epoch": 0.2,
      "learning_rate": 9.859154929577465e-07,
      "logits/chosen": -0.06292358785867691,
      "logits/rejected": -0.009262708015739918,
      "logps/chosen": -354.81341552734375,
      "logps/rejected": -255.65560913085938,
      "loss": 0.6477,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.03358080983161926,
      "rewards/margins": 0.17669352889060974,
      "rewards/rejected": -0.210274338722229,
      "step": 70
    },
    {
      "epoch": 0.22,
      "learning_rate": 9.995106132599868e-07,
      "logits/chosen": 0.03767090290784836,
      "logits/rejected": 0.09468533098697662,
      "logps/chosen": -319.8956298828125,
      "logps/rejected": -235.5646514892578,
      "loss": 0.667,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07292340695858002,
      "rewards/margins": 0.1298426389694214,
      "rewards/rejected": -0.2027660608291626,
      "step": 80
    },
    {
      "epoch": 0.25,
      "learning_rate": 9.978201358980644e-07,
      "logits/chosen": -0.016046693548560143,
      "logits/rejected": 0.04123395308852196,
      "logps/chosen": -361.1316223144531,
      "logps/rejected": -279.39520263671875,
      "loss": 0.6491,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.030158694833517075,
      "rewards/margins": 0.1393767148256302,
      "rewards/rejected": -0.10921802371740341,
      "step": 90
    },
    {
      "epoch": 0.28,
      "learning_rate": 9.949266103908894e-07,
      "logits/chosen": -0.01002789568156004,
      "logits/rejected": 0.055082082748413086,
      "logps/chosen": -275.36773681640625,
      "logps/rejected": -211.86721801757812,
      "loss": 0.6472,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.04291190952062607,
      "rewards/margins": 0.11093775182962418,
      "rewards/rejected": -0.06802584230899811,
      "step": 100
    },
    {
      "epoch": 0.31,
      "learning_rate": 9.908370293252287e-07,
      "logits/chosen": -0.008449924178421497,
      "logits/rejected": 0.018720177933573723,
      "logps/chosen": -323.53802490234375,
      "logps/rejected": -244.490966796875,
      "loss": 0.6407,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.07049892842769623,
      "rewards/margins": 0.1456976681947708,
      "rewards/rejected": -0.07519873231649399,
      "step": 110
    },
    {
      "epoch": 0.34,
      "learning_rate": 9.855612757141654e-07,
      "logits/chosen": -0.07508296519517899,
      "logits/rejected": -0.012953217141330242,
      "logps/chosen": -336.1876525878906,
      "logps/rejected": -208.52011108398438,
      "loss": 0.6333,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.03278186544775963,
      "rewards/margins": 0.26016712188720703,
      "rewards/rejected": -0.2273852527141571,
      "step": 120
    },
    {
      "epoch": 0.37,
      "learning_rate": 9.791120991134902e-07,
      "logits/chosen": -0.10753818601369858,
      "logits/rejected": -0.01082658488303423,
      "logps/chosen": -376.611083984375,
      "logps/rejected": -251.7242889404297,
      "loss": 0.6491,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.026872599497437477,
      "rewards/margins": 0.2888794541358948,
      "rewards/rejected": -0.3157520890235901,
      "step": 130
    },
    {
      "epoch": 0.39,
      "learning_rate": 9.715050848107168e-07,
      "logits/chosen": -0.17280681431293488,
      "logits/rejected": -0.0874427855014801,
      "logps/chosen": -342.14593505859375,
      "logps/rejected": -229.78652954101562,
      "loss": 0.6346,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.014674955978989601,
      "rewards/margins": 0.2581936717033386,
      "rewards/rejected": -0.2728686034679413,
      "step": 140
    },
    {
      "epoch": 0.42,
      "learning_rate": 9.627586161611731e-07,
      "logits/chosen": -0.09239596873521805,
      "logits/rejected": -0.035311758518218994,
      "logps/chosen": -282.2192077636719,
      "logps/rejected": -214.80810546875,
      "loss": 0.6501,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03641432523727417,
      "rewards/margins": 0.19125035405158997,
      "rewards/rejected": -0.22766467928886414,
      "step": 150
    },
    {
      "epoch": 0.45,
      "learning_rate": 9.528938301621955e-07,
      "logits/chosen": -0.168453648686409,
      "logits/rejected": -0.1073572039604187,
      "logps/chosen": -341.64599609375,
      "logps/rejected": -223.13101196289062,
      "loss": 0.6342,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.01668923906981945,
      "rewards/margins": 0.19806070625782013,
      "rewards/rejected": -0.18137145042419434,
      "step": 160
    },
    {
      "epoch": 0.48,
      "learning_rate": 9.419345663727804e-07,
      "logits/chosen": -0.13118991255760193,
      "logits/rejected": -0.0702921599149704,
      "logps/chosen": -344.593505859375,
      "logps/rejected": -231.71804809570312,
      "loss": 0.6221,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.004311258438974619,
      "rewards/margins": 0.2375575602054596,
      "rewards/rejected": -0.23324629664421082,
      "step": 170
    },
    {
      "epoch": 0.51,
      "learning_rate": 9.299073093021404e-07,
      "logits/chosen": -0.05861334875226021,
      "logits/rejected": 0.004674489144235849,
      "logps/chosen": -355.00872802734375,
      "logps/rejected": -255.2248992919922,
      "loss": 0.6285,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13683992624282837,
      "rewards/margins": 0.26262304186820984,
      "rewards/rejected": -0.3994629979133606,
      "step": 180
    },
    {
      "epoch": 0.53,
      "learning_rate": 9.168411244063861e-07,
      "logits/chosen": -0.15201464295387268,
      "logits/rejected": -0.08699367940425873,
      "logps/chosen": -369.53936767578125,
      "logps/rejected": -261.6515197753906,
      "loss": 0.6253,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.18763788044452667,
      "rewards/margins": 0.2461237609386444,
      "rewards/rejected": -0.4337615966796875,
      "step": 190
    },
    {
      "epoch": 0.56,
      "learning_rate": 9.02767587848013e-07,
      "logits/chosen": -0.1659875363111496,
      "logits/rejected": -0.11425626277923584,
      "logps/chosen": -322.26971435546875,
      "logps/rejected": -269.41571044921875,
      "loss": 0.6377,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1545395851135254,
      "rewards/margins": 0.16578510403633118,
      "rewards/rejected": -0.32032471895217896,
      "step": 200
    },
    {
      "epoch": 0.59,
      "learning_rate": 8.877207101879301e-07,
      "logits/chosen": -0.12081719934940338,
      "logits/rejected": -0.055270951241254807,
      "logps/chosen": -303.1000061035156,
      "logps/rejected": -226.3379364013672,
      "loss": 0.6265,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.1504133641719818,
      "rewards/margins": 0.2184509038925171,
      "rewards/rejected": -0.3688642680644989,
      "step": 210
    },
    {
      "epoch": 0.62,
      "learning_rate": 8.717368541944452e-07,
      "logits/chosen": -0.14912042021751404,
      "logits/rejected": -0.11508848518133163,
      "logps/chosen": -313.95745849609375,
      "logps/rejected": -232.803955078125,
      "loss": 0.6245,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.17989172041416168,
      "rewards/margins": 0.21268931031227112,
      "rewards/rejected": -0.3925810158252716,
      "step": 220
    },
    {
      "epoch": 0.65,
      "learning_rate": 8.54854646967831e-07,
      "logits/chosen": -0.20850150287151337,
      "logits/rejected": -0.14839352667331696,
      "logps/chosen": -351.56768798828125,
      "logps/rejected": -286.0954284667969,
      "loss": 0.6232,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.20482465624809265,
      "rewards/margins": 0.2566435933113098,
      "rewards/rejected": -0.46146830916404724,
      "step": 230
    },
    {
      "epoch": 0.67,
      "learning_rate": 8.371148865928318e-07,
      "logits/chosen": -0.13414549827575684,
      "logits/rejected": -0.09452676773071289,
      "logps/chosen": -334.4315490722656,
      "logps/rejected": -271.9152526855469,
      "loss": 0.6104,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.18954138457775116,
      "rewards/margins": 0.24784322082996368,
      "rewards/rejected": -0.43738460540771484,
      "step": 240
    },
    {
      "epoch": 0.7,
      "learning_rate": 8.185604435447001e-07,
      "logits/chosen": -0.08637334406375885,
      "logits/rejected": -0.03546728193759918,
      "logps/chosen": -349.38787841796875,
      "logps/rejected": -278.61761474609375,
      "loss": 0.6238,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.2417462319135666,
      "rewards/margins": 0.2916719615459442,
      "rewards/rejected": -0.5334181785583496,
      "step": 250
    },
    {
      "epoch": 0.73,
      "learning_rate": 7.992361570870287e-07,
      "logits/chosen": -0.15081752836704254,
      "logits/rejected": -0.11529238522052765,
      "logps/chosen": -311.26507568359375,
      "logps/rejected": -253.03659057617188,
      "loss": 0.625,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.34321683645248413,
      "rewards/margins": 0.19419686496257782,
      "rewards/rejected": -0.5374137163162231,
      "step": 260
    },
    {
      "epoch": 0.76,
      "learning_rate": 7.791887269117441e-07,
      "logits/chosen": -0.08736265450716019,
      "logits/rejected": 0.010209694504737854,
      "logps/chosen": -350.77197265625,
      "logps/rejected": -249.4473114013672,
      "loss": 0.6198,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.33235999941825867,
      "rewards/margins": 0.3289110064506531,
      "rewards/rejected": -0.6612709760665894,
      "step": 270
    },
    {
      "epoch": 0.79,
      "learning_rate": 7.584666002831294e-07,
      "logits/chosen": -0.17468181252479553,
      "logits/rejected": -0.10544611513614655,
      "logps/chosen": -393.7838439941406,
      "logps/rejected": -275.46636962890625,
      "loss": 0.6284,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.20970232784748077,
      "rewards/margins": 0.31893718242645264,
      "rewards/rejected": -0.5286394357681274,
      "step": 280
    },
    {
      "epoch": 0.81,
      "learning_rate": 7.37119854958609e-07,
      "logits/chosen": -0.18520912528038025,
      "logits/rejected": -0.12915001809597015,
      "logps/chosen": -372.53961181640625,
      "logps/rejected": -301.4205017089844,
      "loss": 0.6426,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3479081690311432,
      "rewards/margins": 0.25431200861930847,
      "rewards/rejected": -0.6022201776504517,
      "step": 290
    },
    {
      "epoch": 0.84,
      "learning_rate": 7.152000781692285e-07,
      "logits/chosen": -0.20307299494743347,
      "logits/rejected": -0.1757517158985138,
      "logps/chosen": -368.33392333984375,
      "logps/rejected": -302.0033874511719,
      "loss": 0.6219,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2545304298400879,
      "rewards/margins": 0.2229403257369995,
      "rewards/rejected": -0.4774707853794098,
      "step": 300
    },
    {
      "epoch": 0.87,
      "learning_rate": 6.927602419522946e-07,
      "logits/chosen": -0.15039893984794617,
      "logits/rejected": -0.1383979469537735,
      "logps/chosen": -319.64349365234375,
      "logps/rejected": -265.85198974609375,
      "loss": 0.624,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.3667163848876953,
      "rewards/margins": 0.18039202690124512,
      "rewards/rejected": -0.5471083521842957,
      "step": 310
    },
    {
      "epoch": 0.9,
      "learning_rate": 6.698545751374463e-07,
      "logits/chosen": -0.25044673681259155,
      "logits/rejected": -0.12538060545921326,
      "logps/chosen": -413.25946044921875,
      "logps/rejected": -303.199951171875,
      "loss": 0.6216,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.3125981390476227,
      "rewards/margins": 0.38619184494018555,
      "rewards/rejected": -0.6987899541854858,
      "step": 320
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.465384322955224e-07,
      "logits/chosen": -0.1757752150297165,
      "logits/rejected": -0.11078636348247528,
      "logps/chosen": -362.4681091308594,
      "logps/rejected": -267.5201416015625,
      "loss": 0.6074,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.33040475845336914,
      "rewards/margins": 0.37607377767562866,
      "rewards/rejected": -0.7064785957336426,
      "step": 330
    },
    {
      "epoch": 0.96,
      "learning_rate": 6.228681599669248e-07,
      "logits/chosen": -0.19059506058692932,
      "logits/rejected": -0.09184812009334564,
      "logps/chosen": -430.0191345214844,
      "logps/rejected": -296.02557373046875,
      "loss": 0.617,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3559810519218445,
      "rewards/margins": 0.3992285132408142,
      "rewards/rejected": -0.7552096247673035,
      "step": 340
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.989009604927586e-07,
      "logits/chosen": -0.1337263584136963,
      "logits/rejected": -0.01602357253432274,
      "logps/chosen": -391.5013122558594,
      "logps/rejected": -306.56982421875,
      "loss": 0.5985,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.3254554867744446,
      "rewards/margins": 0.4386405944824219,
      "rewards/rejected": -0.7640960812568665,
      "step": 350
    },
    {
      "epoch": 1.01,
      "learning_rate": 5.74694753777815e-07,
      "logits/chosen": -0.13974535465240479,
      "logits/rejected": -0.11217441409826279,
      "logps/chosen": -327.7983703613281,
      "logps/rejected": -279.513916015625,
      "loss": 0.545,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.42400461435317993,
      "rewards/margins": 0.4070449769496918,
      "rewards/rejected": -0.8310495615005493,
      "step": 360
    },
    {
      "epoch": 1.04,
      "learning_rate": 5.503080373194666e-07,
      "logits/chosen": -0.2235272228717804,
      "logits/rejected": -0.12439509481191635,
      "logps/chosen": -374.24261474609375,
      "logps/rejected": -333.127685546875,
      "loss": 0.4827,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.4394102096557617,
      "rewards/margins": 0.6707636713981628,
      "rewards/rejected": -1.1101738214492798,
      "step": 370
    },
    {
      "epoch": 1.07,
      "learning_rate": 5.257997448407366e-07,
      "logits/chosen": -0.12490369379520416,
      "logits/rejected": 0.020254041999578476,
      "logps/chosen": -408.45367431640625,
      "logps/rejected": -308.180908203125,
      "loss": 0.4897,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.4755324423313141,
      "rewards/margins": 0.6596079468727112,
      "rewards/rejected": -1.1351404190063477,
      "step": 380
    },
    {
      "epoch": 1.1,
      "learning_rate": 5.012291038691665e-07,
      "logits/chosen": -0.15194548666477203,
      "logits/rejected": -0.03352439031004906,
      "logps/chosen": -419.17730712890625,
      "logps/rejected": -391.656982421875,
      "loss": 0.4746,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.6772257089614868,
      "rewards/margins": 0.8299336433410645,
      "rewards/rejected": -1.5071594715118408,
      "step": 390
    },
    {
      "epoch": 1.12,
      "learning_rate": 4.7665549260567063e-07,
      "logits/chosen": -0.20493462681770325,
      "logits/rejected": -0.13634856045246124,
      "logps/chosen": -411.919189453125,
      "logps/rejected": -344.78759765625,
      "loss": 0.4713,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.7057473063468933,
      "rewards/margins": 0.824458122253418,
      "rewards/rejected": -1.530205249786377,
      "step": 400
    },
    {
      "epoch": 1.15,
      "learning_rate": 4.521382964292663e-07,
      "logits/chosen": -0.14551517367362976,
      "logits/rejected": -0.07351706176996231,
      "logps/chosen": -394.6050109863281,
      "logps/rejected": -345.28082275390625,
      "loss": 0.4741,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.6415594816207886,
      "rewards/margins": 0.7878659963607788,
      "rewards/rejected": -1.4294254779815674,
      "step": 410
    },
    {
      "epoch": 1.18,
      "learning_rate": 4.277367643844574e-07,
      "logits/chosen": -0.1973930448293686,
      "logits/rejected": -0.10258449614048004,
      "logps/chosen": -446.4418029785156,
      "logps/rejected": -366.457763671875,
      "loss": 0.4605,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8334900736808777,
      "rewards/margins": 0.7750234603881836,
      "rewards/rejected": -1.608513593673706,
      "step": 420
    },
    {
      "epoch": 1.21,
      "learning_rate": 4.035098659980891e-07,
      "logits/chosen": -0.1986023634672165,
      "logits/rejected": -0.06475992500782013,
      "logps/chosen": -419.17010498046875,
      "logps/rejected": -395.53509521484375,
      "loss": 0.465,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.0268628597259521,
      "rewards/margins": 0.7243725657463074,
      "rewards/rejected": -1.7512352466583252,
      "step": 430
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.795161487716928e-07,
      "logits/chosen": -0.20193979144096375,
      "logits/rejected": -0.09314943850040436,
      "logps/chosen": -491.21405029296875,
      "logps/rejected": -423.76654052734375,
      "loss": 0.4524,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.0318418741226196,
      "rewards/margins": 0.9399174451828003,
      "rewards/rejected": -1.9717592000961304,
      "step": 440
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.5581359669371223e-07,
      "logits/chosen": -0.11288833618164062,
      "logits/rejected": -0.1004786491394043,
      "logps/chosen": -417.3036193847656,
      "logps/rejected": -372.6753845214844,
      "loss": 0.4561,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.969109058380127,
      "rewards/margins": 0.6991706490516663,
      "rewards/rejected": -1.6682794094085693,
      "step": 450
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.324594901135326e-07,
      "logits/chosen": -0.17446324229240417,
      "logits/rejected": -0.0842059999704361,
      "logps/chosen": -438.217041015625,
      "logps/rejected": -390.52874755859375,
      "loss": 0.459,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.0395996570587158,
      "rewards/margins": 0.7797072529792786,
      "rewards/rejected": -1.8193069696426392,
      "step": 460
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.095102673159463e-07,
      "logits/chosen": -0.13965001702308655,
      "logits/rejected": -0.09614251554012299,
      "logps/chosen": -477.21337890625,
      "logps/rejected": -420.8619079589844,
      "loss": 0.4336,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.1968367099761963,
      "rewards/margins": 0.8374635577201843,
      "rewards/rejected": -2.0343000888824463,
      "step": 470
    },
    {
      "epoch": 1.35,
      "learning_rate": 2.870213881305802e-07,
      "logits/chosen": -0.06153956800699234,
      "logits/rejected": 0.04337477311491966,
      "logps/chosen": -457.060302734375,
      "logps/rejected": -406.51300048828125,
      "loss": 0.4411,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.1784467697143555,
      "rewards/margins": 0.8234208226203918,
      "rewards/rejected": -2.0018675327301025,
      "step": 480
    },
    {
      "epoch": 1.38,
      "learning_rate": 2.6504719990588745e-07,
      "logits/chosen": -0.05881907790899277,
      "logits/rejected": 0.013253748416900635,
      "logps/chosen": -455.1761169433594,
      "logps/rejected": -401.30133056640625,
      "loss": 0.4396,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.2112154960632324,
      "rewards/margins": 0.8642939329147339,
      "rewards/rejected": -2.0755093097686768,
      "step": 490
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.436408061715988e-07,
      "logits/chosen": 0.04037608206272125,
      "logits/rejected": 0.047458432614803314,
      "logps/chosen": -382.39141845703125,
      "logps/rejected": -422.6522521972656,
      "loss": 0.4409,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.066866159439087,
      "rewards/margins": 0.8686450719833374,
      "rewards/rejected": -1.9355109930038452,
      "step": 500
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.22853938307025e-07,
      "logits/chosen": 0.09550214558839798,
      "logits/rejected": 0.1434255838394165,
      "logps/chosen": -393.2830810546875,
      "logps/rejected": -355.2184753417969,
      "loss": 0.4469,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.153294324874878,
      "rewards/margins": 0.6956819891929626,
      "rewards/rejected": -1.8489763736724854,
      "step": 510
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.0273683052534173e-07,
      "logits/chosen": 0.14489802718162537,
      "logits/rejected": 0.22391514480113983,
      "logps/chosen": -465.29595947265625,
      "logps/rejected": -435.1458435058594,
      "loss": 0.4414,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2600550651550293,
      "rewards/margins": 0.9257229566574097,
      "rewards/rejected": -2.1857781410217285,
      "step": 520
    },
    {
      "epoch": 1.49,
      "learning_rate": 1.833380984759764e-07,
      "logits/chosen": 0.07251317799091339,
      "logits/rejected": 0.17206279933452606,
      "logps/chosen": -410.1485900878906,
      "logps/rejected": -438.0814514160156,
      "loss": 0.4487,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.2980937957763672,
      "rewards/margins": 0.8264884948730469,
      "rewards/rejected": -2.124582290649414,
      "step": 530
    },
    {
      "epoch": 1.52,
      "learning_rate": 1.6470462175846606e-07,
      "logits/chosen": 0.09354039281606674,
      "logits/rejected": 0.13454324007034302,
      "logps/chosen": -464.46990966796875,
      "logps/rejected": -444.4037170410156,
      "loss": 0.4383,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.0875951051712036,
      "rewards/margins": 0.913548469543457,
      "rewards/rejected": -2.001143455505371,
      "step": 540
    },
    {
      "epoch": 1.55,
      "learning_rate": 1.468814306317092e-07,
      "logits/chosen": 0.12879224121570587,
      "logits/rejected": 0.1979069709777832,
      "logps/chosen": -426.343505859375,
      "logps/rejected": -380.24493408203125,
      "loss": 0.4398,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.0752108097076416,
      "rewards/margins": 0.9150044322013855,
      "rewards/rejected": -1.9902150630950928,
      "step": 550
    },
    {
      "epoch": 1.57,
      "learning_rate": 1.299115971923958e-07,
      "logits/chosen": 0.12415747344493866,
      "logits/rejected": 0.20912082493305206,
      "logps/chosen": -448.64031982421875,
      "logps/rejected": -442.22576904296875,
      "loss": 0.4317,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.2348341941833496,
      "rewards/margins": 0.9574505090713501,
      "rewards/rejected": -2.1922848224639893,
      "step": 560
    },
    {
      "epoch": 1.6,
      "learning_rate": 1.1383613128559305e-07,
      "logits/chosen": 0.024107109755277634,
      "logits/rejected": 0.1119920164346695,
      "logps/chosen": -505.82928466796875,
      "logps/rejected": -505.4091796875,
      "loss": 0.426,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.2975566387176514,
      "rewards/margins": 1.085707187652588,
      "rewards/rejected": -2.3832640647888184,
      "step": 570
    },
    {
      "epoch": 1.63,
      "learning_rate": 9.869388139903495e-08,
      "logits/chosen": 0.012987576425075531,
      "logits/rejected": 0.06353282183408737,
      "logps/chosen": -541.9244995117188,
      "logps/rejected": -499.44110107421875,
      "loss": 0.4104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3797237873077393,
      "rewards/margins": 0.9557130932807922,
      "rewards/rejected": -2.335437059402466,
      "step": 580
    },
    {
      "epoch": 1.66,
      "learning_rate": 8.452144078061818e-08,
      "logits/chosen": 0.14748625457286835,
      "logits/rejected": 0.21033573150634766,
      "logps/chosen": -428.7500915527344,
      "logps/rejected": -418.6878356933594,
      "loss": 0.4248,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.361243724822998,
      "rewards/margins": 0.893297553062439,
      "rewards/rejected": -2.2545411586761475,
      "step": 590
    },
    {
      "epoch": 1.69,
      "learning_rate": 7.135305900598321e-08,
      "logits/chosen": 0.012422094121575356,
      "logits/rejected": 0.08027663081884384,
      "logps/chosen": -506.3257751464844,
      "logps/rejected": -460.52777099609375,
      "loss": 0.4245,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.5179449319839478,
      "rewards/margins": 1.0011281967163086,
      "rewards/rejected": -2.5190727710723877,
      "step": 600
    },
    {
      "epoch": 1.71,
      "learning_rate": 5.9220559209888166e-08,
      "logits/chosen": 0.12760373950004578,
      "logits/rejected": 0.20078882575035095,
      "logps/chosen": -452.77734375,
      "logps/rejected": -487.36016845703125,
      "loss": 0.4266,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.444874882698059,
      "rewards/margins": 1.0283567905426025,
      "rewards/rejected": -2.473231792449951,
      "step": 610
    },
    {
      "epoch": 1.74,
      "learning_rate": 4.815326118139812e-08,
      "logits/chosen": 0.14059333503246307,
      "logits/rejected": 0.2606312930583954,
      "logps/chosen": -415.8949279785156,
      "logps/rejected": -418.4481506347656,
      "loss": 0.4339,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.5133248567581177,
      "rewards/margins": 0.8190656900405884,
      "rewards/rejected": -2.332390546798706,
      "step": 620
    },
    {
      "epoch": 1.77,
      "learning_rate": 3.81779105087407e-08,
      "logits/chosen": 0.08177526295185089,
      "logits/rejected": 0.21824178099632263,
      "logps/chosen": -476.4537658691406,
      "logps/rejected": -467.0603942871094,
      "loss": 0.4436,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.559093713760376,
      "rewards/margins": 0.9111778140068054,
      "rewards/rejected": -2.470271587371826,
      "step": 630
    },
    {
      "epoch": 1.8,
      "learning_rate": 2.9318613945057637e-08,
      "logits/chosen": 0.12043829262256622,
      "logits/rejected": 0.19589121639728546,
      "logps/chosen": -504.67608642578125,
      "logps/rejected": -489.88330078125,
      "loss": 0.4181,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.4812028408050537,
      "rewards/margins": 1.078580617904663,
      "rewards/rejected": -2.559783458709717,
      "step": 640
    },
    {
      "epoch": 1.83,
      "learning_rate": 2.1596781151249523e-08,
      "logits/chosen": 0.15832188725471497,
      "logits/rejected": 0.2765315771102905,
      "logps/chosen": -461.1602478027344,
      "logps/rejected": -456.65826416015625,
      "loss": 0.4321,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.4598538875579834,
      "rewards/margins": 0.8676374554634094,
      "rewards/rejected": -2.327491283416748,
      "step": 650
    },
    {
      "epoch": 1.85,
      "learning_rate": 1.5031072956701695e-08,
      "logits/chosen": 0.13437607884407043,
      "logits/rejected": 0.19320687651634216,
      "logps/chosen": -503.2486267089844,
      "logps/rejected": -498.3321228027344,
      "loss": 0.4196,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.5257575511932373,
      "rewards/margins": 1.106710433959961,
      "rewards/rejected": -2.632467746734619,
      "step": 660
    },
    {
      "epoch": 1.88,
      "learning_rate": 9.637356262923723e-09,
      "logits/chosen": 0.24336127936840057,
      "logits/rejected": 0.2697839140892029,
      "logps/chosen": -468.03289794921875,
      "logps/rejected": -444.1060485839844,
      "loss": 0.4387,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.5293312072753906,
      "rewards/margins": 0.842644214630127,
      "rewards/rejected": -2.3719754219055176,
      "step": 670
    },
    {
      "epoch": 1.91,
      "learning_rate": 5.428665699084789e-09,
      "logits/chosen": 0.10800081491470337,
      "logits/rejected": 0.1402808427810669,
      "logps/chosen": -492.41448974609375,
      "logps/rejected": -494.878173828125,
      "loss": 0.4088,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.4665706157684326,
      "rewards/margins": 1.0468090772628784,
      "rewards/rejected": -2.5133795738220215,
      "step": 680
    },
    {
      "epoch": 1.94,
      "learning_rate": 2.415172122110343e-09,
      "logits/chosen": 0.13986334204673767,
      "logits/rejected": 0.26917481422424316,
      "logps/chosen": -483.06024169921875,
      "logps/rejected": -482.5399475097656,
      "loss": 0.4382,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.4039775133132935,
      "rewards/margins": 1.0037094354629517,
      "rewards/rejected": -2.407686948776245,
      "step": 690
    },
    {
      "epoch": 1.97,
      "learning_rate": 6.041580374618327e-10,
      "logits/chosen": 0.105182945728302,
      "logits/rejected": 0.17750731110572815,
      "logps/chosen": -485.490478515625,
      "logps/rejected": -456.46063232421875,
      "loss": 0.4411,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.5071966648101807,
      "rewards/margins": 0.8808605074882507,
      "rewards/rejected": -2.388056993484497,
      "step": 700
    },
    {
      "epoch": 2.0,
      "learning_rate": 0.0,
      "logits/chosen": 0.1599053144454956,
      "logits/rejected": 0.2173675298690796,
      "logps/chosen": -413.2229919433594,
      "logps/rejected": -438.8427734375,
      "loss": 0.4277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4848761558532715,
      "rewards/margins": 0.8370813131332397,
      "rewards/rejected": -2.3219573497772217,
      "step": 710
    },
    {
      "epoch": 2.0,
      "step": 710,
      "total_flos": 0.0,
      "train_loss": 0.5407548575334146,
      "train_runtime": 10382.7821,
      "train_samples_per_second": 8.774,
      "train_steps_per_second": 0.068
    }
  ],
  "logging_steps": 10,
  "max_steps": 710,
  "num_train_epochs": 2,
  "save_steps": 10000,
  "total_flos": 0.0,
  "trial_name": null,
  "trial_params": null
}