{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9998423955870764,
  "eval_steps": 500,
  "global_step": 1586,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 3.1446540880503146e-08,
      "logits/chosen": 0.20935016870498657,
      "logits/rejected": 0.28666022419929504,
      "logps/chosen": -338.6566467285156,
      "logps/rejected": -308.9756164550781,
      "loss": 0.1265,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.144654088050315e-07,
      "logits/chosen": 0.21327850222587585,
      "logits/rejected": 0.2737487554550171,
      "logps/chosen": -325.64306640625,
      "logps/rejected": -319.2493896484375,
      "loss": 0.1202,
      "rewards/accuracies": 0.4027777910232544,
      "rewards/chosen": 4.140691817156039e-05,
      "rewards/margins": -2.3560824047308415e-05,
      "rewards/rejected": 6.496775313280523e-05,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.28930817610063e-07,
      "logits/chosen": 0.260469526052475,
      "logits/rejected": 0.3009797930717468,
      "logps/chosen": -294.43804931640625,
      "logps/rejected": -289.858154296875,
      "loss": 0.1078,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 3.346197263454087e-05,
      "rewards/margins": 5.3948046115692705e-05,
      "rewards/rejected": -2.0486073481151834e-05,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.433962264150944e-07,
      "logits/chosen": 0.19516317546367645,
      "logits/rejected": 0.22890889644622803,
      "logps/chosen": -330.41143798828125,
      "logps/rejected": -330.6087341308594,
      "loss": 0.1203,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -5.2339473768370226e-05,
      "rewards/margins": 0.000176291519892402,
      "rewards/rejected": -0.0002286309900227934,
      "step": 30
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.257861635220126e-06,
      "logits/chosen": 0.22363004088401794,
      "logits/rejected": 0.2845616936683655,
      "logps/chosen": -351.8126525878906,
      "logps/rejected": -298.5321044921875,
      "loss": 0.1223,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.0003051063104066998,
      "rewards/margins": 0.0003100560570601374,
      "rewards/rejected": -0.0006151623092591763,
      "step": 40
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5723270440251573e-06,
      "logits/chosen": 0.2019948959350586,
      "logits/rejected": 0.25790125131607056,
      "logps/chosen": -321.6785888671875,
      "logps/rejected": -317.89813232421875,
      "loss": 0.1163,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0004650927730835974,
      "rewards/margins": 0.0002983523881994188,
      "rewards/rejected": -0.0007634450448676944,
      "step": 50
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8867924528301889e-06,
      "logits/chosen": 0.23230671882629395,
      "logits/rejected": 0.24606795608997345,
      "logps/chosen": -320.841552734375,
      "logps/rejected": -297.08306884765625,
      "loss": 0.1075,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0008995781536214054,
      "rewards/margins": 0.0004379089514259249,
      "rewards/rejected": -0.0013374872505664825,
      "step": 60
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.2012578616352204e-06,
      "logits/chosen": 0.18308493494987488,
      "logits/rejected": 0.2341008186340332,
      "logps/chosen": -288.20172119140625,
      "logps/rejected": -298.20050048828125,
      "loss": 0.1169,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0013604220002889633,
      "rewards/margins": 0.0006924796034581959,
      "rewards/rejected": -0.0020529015455394983,
      "step": 70
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.515723270440252e-06,
      "logits/chosen": 0.24107146263122559,
      "logits/rejected": 0.2765248417854309,
      "logps/chosen": -327.1878356933594,
      "logps/rejected": -324.80706787109375,
      "loss": 0.1138,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.0024016976822167635,
      "rewards/margins": 0.0004348896036390215,
      "rewards/rejected": -0.0028365873731672764,
      "step": 80
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.830188679245283e-06,
      "logits/chosen": 0.23600873351097107,
      "logits/rejected": 0.2604605257511139,
      "logps/chosen": -322.81610107421875,
      "logps/rejected": -312.8374328613281,
      "loss": 0.1158,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0036561130546033382,
      "rewards/margins": 0.0011059448588639498,
      "rewards/rejected": -0.004762057680636644,
      "step": 90
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1446540880503146e-06,
      "logits/chosen": 0.17113199830055237,
      "logits/rejected": 0.19848307967185974,
      "logps/chosen": -339.1964416503906,
      "logps/rejected": -318.428955078125,
      "loss": 0.1188,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0050820643082261086,
      "rewards/margins": 0.002656476106494665,
      "rewards/rejected": -0.007738540414720774,
      "step": 100
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.4591194968553458e-06,
      "logits/chosen": 0.16253027319908142,
      "logits/rejected": 0.20139364898204803,
      "logps/chosen": -339.23040771484375,
      "logps/rejected": -336.61444091796875,
      "loss": 0.1205,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.008081575855612755,
      "rewards/margins": 0.0035987249575555325,
      "rewards/rejected": -0.011680299416184425,
      "step": 110
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.7735849056603777e-06,
      "logits/chosen": 0.17907488346099854,
      "logits/rejected": 0.1907232403755188,
      "logps/chosen": -347.43646240234375,
      "logps/rejected": -350.59881591796875,
      "loss": 0.1156,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.012615310959517956,
      "rewards/margins": 0.0076774852350354195,
      "rewards/rejected": -0.020292794331908226,
      "step": 120
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.08805031446541e-06,
      "logits/chosen": 0.14514799416065216,
      "logits/rejected": 0.1459697037935257,
      "logps/chosen": -318.0201416015625,
      "logps/rejected": -335.25506591796875,
      "loss": 0.1111,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.01840413734316826,
      "rewards/margins": 0.015484845265746117,
      "rewards/rejected": -0.033888980746269226,
      "step": 130
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.402515723270441e-06,
      "logits/chosen": 0.10525090992450714,
      "logits/rejected": 0.12337298691272736,
      "logps/chosen": -353.93353271484375,
      "logps/rejected": -395.66522216796875,
      "loss": 0.117,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.026070892810821533,
      "rewards/margins": 0.020400088280439377,
      "rewards/rejected": -0.04647097736597061,
      "step": 140
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.716981132075472e-06,
      "logits/chosen": 0.104182168841362,
      "logits/rejected": 0.09100546687841415,
      "logps/chosen": -375.31732177734375,
      "logps/rejected": -394.6401062011719,
      "loss": 0.1104,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.04359036684036255,
      "rewards/margins": 0.019685441628098488,
      "rewards/rejected": -0.06327580660581589,
      "step": 150
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999993941548484e-06,
      "logits/chosen": 0.06796596199274063,
      "logits/rejected": 0.10237328708171844,
      "logps/chosen": -380.8462829589844,
      "logps/rejected": -407.0494079589844,
      "loss": 0.1092,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.05505872890353203,
      "rewards/margins": 0.028139878064393997,
      "rewards/rejected": -0.08319860696792603,
      "step": 160
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999266962896092e-06,
      "logits/chosen": 0.08151620626449585,
      "logits/rejected": 0.07926828414201736,
      "logps/chosen": -364.17974853515625,
      "logps/rejected": -370.903564453125,
      "loss": 0.1007,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.053988486528396606,
      "rewards/margins": 0.021424416452646255,
      "rewards/rejected": -0.07541291415691376,
      "step": 170
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997328697661372e-06,
      "logits/chosen": 0.0645218938589096,
      "logits/rejected": 0.09341205656528473,
      "logps/chosen": -343.18609619140625,
      "logps/rejected": -353.36810302734375,
      "loss": 0.1208,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.049203019589185715,
      "rewards/margins": 0.016046565026044846,
      "rewards/rejected": -0.06524958461523056,
      "step": 180
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.994180085237635e-06,
      "logits/chosen": 0.13387538492679596,
      "logits/rejected": 0.13471734523773193,
      "logps/chosen": -333.0649108886719,
      "logps/rejected": -387.0137939453125,
      "loss": 0.1179,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.05196870118379593,
      "rewards/margins": 0.02883673831820488,
      "rewards/rejected": -0.08080543577671051,
      "step": 190
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.98982265162112e-06,
      "logits/chosen": 0.09480293095111847,
      "logits/rejected": 0.08700738847255707,
      "logps/chosen": -373.3707580566406,
      "logps/rejected": -419.16131591796875,
      "loss": 0.0986,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.058679305016994476,
      "rewards/margins": 0.025583749637007713,
      "rewards/rejected": -0.08426305651664734,
      "step": 200
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.984258508671404e-06,
      "logits/chosen": 0.12381438910961151,
      "logits/rejected": 0.11012091487646103,
      "logps/chosen": -403.3990478515625,
      "logps/rejected": -435.538818359375,
      "loss": 0.0994,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.051610808819532394,
      "rewards/margins": 0.041461896151304245,
      "rewards/rejected": -0.09307270497083664,
      "step": 210
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.977490353087878e-06,
      "logits/chosen": 0.13996745645999908,
      "logits/rejected": 0.15085197985172272,
      "logps/chosen": -365.3772277832031,
      "logps/rejected": -394.3701477050781,
      "loss": 0.0994,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.046448659151792526,
      "rewards/margins": 0.0269100870937109,
      "rewards/rejected": -0.07335875183343887,
      "step": 220
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.969521465102775e-06,
      "logits/chosen": 0.10541512072086334,
      "logits/rejected": 0.13170066475868225,
      "logps/chosen": -372.6682434082031,
      "logps/rejected": -432.30670166015625,
      "loss": 0.1162,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.055505841970443726,
      "rewards/margins": 0.03007945977151394,
      "rewards/rejected": -0.08558530360460281,
      "step": 230
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.960355706891375e-06,
      "logits/chosen": 0.08097681403160095,
      "logits/rejected": 0.08701089024543762,
      "logps/chosen": -377.55023193359375,
      "logps/rejected": -453.13702392578125,
      "loss": 0.1107,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.061971623450517654,
      "rewards/margins": 0.04781196638941765,
      "rewards/rejected": -0.1097835898399353,
      "step": 240
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.94999752070019e-06,
      "logits/chosen": 0.10144323110580444,
      "logits/rejected": 0.13508056104183197,
      "logps/chosen": -362.32012939453125,
      "logps/rejected": -365.26947021484375,
      "loss": 0.1084,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.06062735244631767,
      "rewards/margins": 0.024900587275624275,
      "rewards/rejected": -0.0855279415845871,
      "step": 250
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.938451926693985e-06,
      "logits/chosen": 0.11075524240732193,
      "logits/rejected": 0.1454174965620041,
      "logps/chosen": -375.95538330078125,
      "logps/rejected": -407.412109375,
      "loss": 0.1061,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07085081189870834,
      "rewards/margins": 0.03195417672395706,
      "rewards/rejected": -0.10280498117208481,
      "step": 260
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9257245205227296e-06,
      "logits/chosen": 0.14314480125904083,
      "logits/rejected": 0.122921422123909,
      "logps/chosen": -381.03472900390625,
      "logps/rejected": -460.407470703125,
      "loss": 0.0954,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06635353714227676,
      "rewards/margins": 0.05833283066749573,
      "rewards/rejected": -0.12468637526035309,
      "step": 270
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.911821470609627e-06,
      "logits/chosen": 0.11854016780853271,
      "logits/rejected": 0.0927438884973526,
      "logps/chosen": -393.5327453613281,
      "logps/rejected": -386.33795166015625,
      "loss": 0.1045,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08982465416193008,
      "rewards/margins": 0.011397195979952812,
      "rewards/rejected": -0.10122185945510864,
      "step": 280
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.896749515161544e-06,
      "logits/chosen": 0.16842451691627502,
      "logits/rejected": 0.13145343959331512,
      "logps/chosen": -371.6466064453125,
      "logps/rejected": -430.08587646484375,
      "loss": 0.0924,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06959854066371918,
      "rewards/margins": 0.037486035376787186,
      "rewards/rejected": -0.10708458721637726,
      "step": 290
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.880515958903296e-06,
      "logits/chosen": 0.14044585824012756,
      "logits/rejected": 0.14415714144706726,
      "logps/chosen": -387.0291442871094,
      "logps/rejected": -432.3477478027344,
      "loss": 0.0995,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07424234598875046,
      "rewards/margins": 0.029624024406075478,
      "rewards/rejected": -0.10386637598276138,
      "step": 300
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8631286695373626e-06,
      "logits/chosen": 0.12493617832660675,
      "logits/rejected": 0.16041526198387146,
      "logps/chosen": -432.5501403808594,
      "logps/rejected": -449.15167236328125,
      "loss": 0.0916,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.08585993945598602,
      "rewards/margins": 0.024735430255532265,
      "rewards/rejected": -0.11059536784887314,
      "step": 310
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.844596073930749e-06,
      "logits/chosen": 0.11006021499633789,
      "logits/rejected": 0.15848229825496674,
      "logps/chosen": -401.59051513671875,
      "logps/rejected": -428.4137268066406,
      "loss": 0.1124,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08783275634050369,
      "rewards/margins": 0.04196416586637497,
      "rewards/rejected": -0.12979692220687866,
      "step": 320
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.824927154030856e-06,
      "logits/chosen": 0.14780564606189728,
      "logits/rejected": 0.1997770518064499,
      "logps/chosen": -412.12664794921875,
      "logps/rejected": -425.85711669921875,
      "loss": 0.1016,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08760993927717209,
      "rewards/margins": 0.03202151134610176,
      "rewards/rejected": -0.11963143199682236,
      "step": 330
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.804131442512309e-06,
      "logits/chosen": 0.15109729766845703,
      "logits/rejected": 0.17160046100616455,
      "logps/chosen": -411.62109375,
      "logps/rejected": -422.8251953125,
      "loss": 0.1035,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08082294464111328,
      "rewards/margins": 0.03992414474487305,
      "rewards/rejected": -0.12074708938598633,
      "step": 340
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.78221901815689e-06,
      "logits/chosen": 0.11030396074056625,
      "logits/rejected": 0.10951008647680283,
      "logps/chosen": -354.53839111328125,
      "logps/rejected": -379.4768371582031,
      "loss": 0.0944,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07093557715415955,
      "rewards/margins": 0.02872433327138424,
      "rewards/rejected": -0.09965990483760834,
      "step": 350
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7592005009687836e-06,
      "logits/chosen": 0.12368909269571304,
      "logits/rejected": 0.1551579087972641,
      "logps/chosen": -422.1844787597656,
      "logps/rejected": -442.5235290527344,
      "loss": 0.0942,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.09620015323162079,
      "rewards/margins": 0.03423624113202095,
      "rewards/rejected": -0.13043639063835144,
      "step": 360
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.735087047027525e-06,
      "logits/chosen": 0.09877191483974457,
      "logits/rejected": 0.1075328141450882,
      "logps/chosen": -345.7369689941406,
      "logps/rejected": -428.72320556640625,
      "loss": 0.1058,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08192293345928192,
      "rewards/margins": 0.041438352316617966,
      "rewards/rejected": -0.12336127460002899,
      "step": 370
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.709890343081127e-06,
      "logits/chosen": 0.11233127117156982,
      "logits/rejected": 0.09415660798549652,
      "logps/chosen": -375.85552978515625,
      "logps/rejected": -400.69561767578125,
      "loss": 0.0912,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07030915468931198,
      "rewards/margins": 0.028951376676559448,
      "rewards/rejected": -0.09926052391529083,
      "step": 380
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6836226008820215e-06,
      "logits/chosen": 0.16574355959892273,
      "logits/rejected": 0.1816074252128601,
      "logps/chosen": -416.6439514160156,
      "logps/rejected": -431.9356994628906,
      "loss": 0.0968,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06910283863544464,
      "rewards/margins": 0.021879449486732483,
      "rewards/rejected": -0.09098228812217712,
      "step": 390
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.656296551268547e-06,
      "logits/chosen": 0.15201489627361298,
      "logits/rejected": 0.18494170904159546,
      "logps/chosen": -382.65350341796875,
      "logps/rejected": -439.33502197265625,
      "loss": 0.0938,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.0796704962849617,
      "rewards/margins": 0.04547972232103348,
      "rewards/rejected": -0.12515023350715637,
      "step": 400
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.627925437994868e-06,
      "logits/chosen": 0.13324889540672302,
      "logits/rejected": 0.15600983798503876,
      "logps/chosen": -342.90924072265625,
      "logps/rejected": -418.2884216308594,
      "loss": 0.0905,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08169208467006683,
      "rewards/margins": 0.04994327947497368,
      "rewards/rejected": -0.1316353678703308,
      "step": 410
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.598523011312296e-06,
      "logits/chosen": 0.15302890539169312,
      "logits/rejected": 0.15263435244560242,
      "logps/chosen": -399.0650939941406,
      "logps/rejected": -442.39080810546875,
      "loss": 0.0898,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08143939077854156,
      "rewards/margins": 0.03956509754061699,
      "rewards/rejected": -0.12100447714328766,
      "step": 420
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.568103521305137e-06,
      "logits/chosen": 0.18960033357143402,
      "logits/rejected": 0.15051600337028503,
      "logps/chosen": -382.1358337402344,
      "logps/rejected": -444.8484802246094,
      "loss": 0.0959,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.07425881922245026,
      "rewards/margins": 0.04583179950714111,
      "rewards/rejected": -0.12009060382843018,
      "step": 430
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.536681710984296e-06,
      "logits/chosen": 0.14668092131614685,
      "logits/rejected": 0.20314042270183563,
      "logps/chosen": -405.0791015625,
      "logps/rejected": -454.7623596191406,
      "loss": 0.0927,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07047592103481293,
      "rewards/margins": 0.05656527355313301,
      "rewards/rejected": -0.12704120576381683,
      "step": 440
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.5042728091419764e-06,
      "logits/chosen": 0.14518819749355316,
      "logits/rejected": 0.15771886706352234,
      "logps/chosen": -351.4585266113281,
      "logps/rejected": -418.5862731933594,
      "loss": 0.0903,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06795553117990494,
      "rewards/margins": 0.03801413252949715,
      "rewards/rejected": -0.10596966743469238,
      "step": 450
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.470892522970937e-06,
      "logits/chosen": 0.12991555035114288,
      "logits/rejected": 0.13929405808448792,
      "logps/chosen": -369.97943115234375,
      "logps/rejected": -418.1014099121094,
      "loss": 0.0993,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06941680610179901,
      "rewards/margins": 0.03608622029423714,
      "rewards/rejected": -0.10550303757190704,
      "step": 460
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.436557030451907e-06,
      "logits/chosen": 0.13183408975601196,
      "logits/rejected": 0.14138123393058777,
      "logps/chosen": -386.870849609375,
      "logps/rejected": -483.94854736328125,
      "loss": 0.1037,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.08128295838832855,
      "rewards/margins": 0.05732026696205139,
      "rewards/rejected": -0.13860321044921875,
      "step": 470
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.401282972512805e-06,
      "logits/chosen": 0.09972018748521805,
      "logits/rejected": 0.10249191522598267,
      "logps/chosen": -423.345703125,
      "logps/rejected": -447.601318359375,
      "loss": 0.1098,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.09919498860836029,
      "rewards/margins": 0.02907257154583931,
      "rewards/rejected": -0.1282675564289093,
      "step": 480
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.365087444963611e-06,
      "logits/chosen": 0.08806703239679337,
      "logits/rejected": 0.1713884323835373,
      "logps/chosen": -467.56182861328125,
      "logps/rejected": -477.89581298828125,
      "loss": 0.0904,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.09871919453144073,
      "rewards/margins": 0.04472241550683975,
      "rewards/rejected": -0.14344163239002228,
      "step": 490
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.327987990210763e-06,
      "logits/chosen": 0.126537024974823,
      "logits/rejected": 0.15067018568515778,
      "logps/chosen": -361.6537170410156,
      "logps/rejected": -433.5013122558594,
      "loss": 0.0984,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.08296778053045273,
      "rewards/margins": 0.053381673991680145,
      "rewards/rejected": -0.13634945452213287,
      "step": 500
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.290002588755112e-06,
      "logits/chosen": 0.14435867965221405,
      "logits/rejected": 0.18025091290473938,
      "logps/chosen": -383.94110107421875,
      "logps/rejected": -440.88446044921875,
      "loss": 0.0917,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07438959181308746,
      "rewards/margins": 0.035205405205488205,
      "rewards/rejected": -0.10959500074386597,
      "step": 510
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.251149650477553e-06,
      "logits/chosen": 0.11688809096813202,
      "logits/rejected": 0.1297851949930191,
      "logps/chosen": -420.3448181152344,
      "logps/rejected": -450.2854919433594,
      "loss": 0.1182,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.10054901987314224,
      "rewards/margins": 0.025883013382554054,
      "rewards/rejected": -0.12643203139305115,
      "step": 520
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.211448005716547e-06,
      "logits/chosen": 0.140035018324852,
      "logits/rejected": 0.1768953651189804,
      "logps/chosen": -340.6114196777344,
      "logps/rejected": -433.100341796875,
      "loss": 0.1061,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06748305261135101,
      "rewards/margins": 0.05944541096687317,
      "rewards/rejected": -0.12692847847938538,
      "step": 530
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.170916896141877e-06,
      "logits/chosen": 0.11086130142211914,
      "logits/rejected": 0.11766747385263443,
      "logps/chosen": -409.5714111328125,
      "logps/rejected": -471.7931213378906,
      "loss": 0.0909,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08654967695474625,
      "rewards/margins": 0.052199166268110275,
      "rewards/rejected": -0.13874883949756622,
      "step": 540
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.129575965429036e-06,
      "logits/chosen": 0.1578402817249298,
      "logits/rejected": 0.12391265481710434,
      "logps/chosen": -406.35784912109375,
      "logps/rejected": -423.5259704589844,
      "loss": 0.0988,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07460321485996246,
      "rewards/margins": 0.04027136042714119,
      "rewards/rejected": -0.11487456411123276,
      "step": 550
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.087445249738795e-06,
      "logits/chosen": 0.13769268989562988,
      "logits/rejected": 0.14535634219646454,
      "logps/chosen": -402.810791015625,
      "logps/rejected": -468.11578369140625,
      "loss": 0.0885,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07266028970479965,
      "rewards/margins": 0.04542839899659157,
      "rewards/rejected": -0.11808867752552032,
      "step": 560
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.044545168006535e-06,
      "logits/chosen": 0.135701984167099,
      "logits/rejected": 0.15048106014728546,
      "logps/chosen": -367.40716552734375,
      "logps/rejected": -425.750732421875,
      "loss": 0.0933,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06163717433810234,
      "rewards/margins": 0.0523550882935524,
      "rewards/rejected": -0.11399225890636444,
      "step": 570
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.000896512046079e-06,
      "logits/chosen": 0.06643296778202057,
      "logits/rejected": 0.11304454505443573,
      "logps/chosen": -382.0234069824219,
      "logps/rejected": -408.7083435058594,
      "loss": 0.0964,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.08725063502788544,
      "rewards/margins": 0.030442586168646812,
      "rewards/rejected": -0.117693230509758,
      "step": 580
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.956520436472794e-06,
      "logits/chosen": 0.09026265144348145,
      "logits/rejected": 0.14468029141426086,
      "logps/chosen": -425.26214599609375,
      "logps/rejected": -457.64990234375,
      "loss": 0.0923,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09045656025409698,
      "rewards/margins": 0.050621867179870605,
      "rewards/rejected": -0.1410784274339676,
      "step": 590
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.911438448450867e-06,
      "logits/chosen": 0.09659644216299057,
      "logits/rejected": 0.10519146919250488,
      "logps/chosen": -409.36737060546875,
      "logps/rejected": -426.3362731933594,
      "loss": 0.0965,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09249825030565262,
      "rewards/margins": 0.03923904523253441,
      "rewards/rejected": -0.13173729181289673,
      "step": 600
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.865672397269712e-06,
      "logits/chosen": 0.16088059544563293,
      "logits/rejected": 0.15578779578208923,
      "logps/chosen": -421.38134765625,
      "logps/rejected": -467.1226501464844,
      "loss": 0.103,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.09401367604732513,
      "rewards/margins": 0.037443626672029495,
      "rewards/rejected": -0.13145729899406433,
      "step": 610
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.819244463754566e-06,
      "logits/chosen": 0.1309213638305664,
      "logits/rejected": 0.183641254901886,
      "logps/chosen": -372.6539611816406,
      "logps/rejected": -422.17333984375,
      "loss": 0.1079,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07998184859752655,
      "rewards/margins": 0.03546987846493721,
      "rewards/rejected": -0.11545171588659286,
      "step": 620
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7721771495164e-06,
      "logits/chosen": 0.18562278151512146,
      "logits/rejected": 0.17855581641197205,
      "logps/chosen": -368.06634521484375,
      "logps/rejected": -410.43157958984375,
      "loss": 0.0984,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06547744572162628,
      "rewards/margins": 0.037668634206056595,
      "rewards/rejected": -0.10314607620239258,
      "step": 630
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7244932660463628e-06,
      "logits/chosen": 0.10567553341388702,
      "logits/rejected": 0.09214256703853607,
      "logps/chosen": -387.8129577636719,
      "logps/rejected": -428.2767028808594,
      "loss": 0.1038,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08541779965162277,
      "rewards/margins": 0.04381578415632248,
      "rewards/rejected": -0.12923358380794525,
      "step": 640
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.676215923660037e-06,
      "logits/chosen": 0.1147516518831253,
      "logits/rejected": 0.13854558765888214,
      "logps/chosen": -388.88397216796875,
      "logps/rejected": -402.66162109375,
      "loss": 0.0979,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08885400742292404,
      "rewards/margins": 0.03468873351812363,
      "rewards/rejected": -0.12354274094104767,
      "step": 650
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6273685202968727e-06,
      "logits/chosen": 0.14652499556541443,
      "logits/rejected": 0.23127985000610352,
      "logps/chosen": -427.195556640625,
      "logps/rejected": -474.3085021972656,
      "loss": 0.1012,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.10611389577388763,
      "rewards/margins": 0.047405023127794266,
      "rewards/rejected": -0.1535189151763916,
      "step": 660
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.577974730180212e-06,
      "logits/chosen": 0.1295718103647232,
      "logits/rejected": 0.18317390978336334,
      "logps/chosen": -454.80938720703125,
      "logps/rejected": -480.46490478515625,
      "loss": 0.1008,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.09846655279397964,
      "rewards/margins": 0.038999587297439575,
      "rewards/rejected": -0.13746614754199982,
      "step": 670
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.528058492343423e-06,
      "logits/chosen": 0.11106806993484497,
      "logits/rejected": 0.1717728078365326,
      "logps/chosen": -415.18414306640625,
      "logps/rejected": -438.2942810058594,
      "loss": 0.1083,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1040610671043396,
      "rewards/margins": 0.030150944367051125,
      "rewards/rejected": -0.13421201705932617,
      "step": 680
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.477643999027683e-06,
      "logits/chosen": 0.16643357276916504,
      "logits/rejected": 0.18508140742778778,
      "logps/chosen": -382.0606384277344,
      "logps/rejected": -410.14691162109375,
      "loss": 0.0969,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08891114592552185,
      "rewards/margins": 0.03431970998644829,
      "rewards/rejected": -0.12323085963726044,
      "step": 690
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4267556839570467e-06,
      "logits/chosen": 0.11085295677185059,
      "logits/rejected": 0.11592169851064682,
      "logps/chosen": -380.14892578125,
      "logps/rejected": -436.01422119140625,
      "loss": 0.0928,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08309037983417511,
      "rewards/margins": 0.050611067563295364,
      "rewards/rejected": -0.13370142877101898,
      "step": 700
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.375418210496473e-06,
      "logits/chosen": 0.14460065960884094,
      "logits/rejected": 0.17223772406578064,
      "logps/chosen": -410.51422119140625,
      "logps/rejected": -431.4485778808594,
      "loss": 0.1039,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08633999526500702,
      "rewards/margins": 0.03471770137548447,
      "rewards/rejected": -0.12105770409107208,
      "step": 710
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3236564596985677e-06,
      "logits/chosen": 0.11466125398874283,
      "logits/rejected": 0.14213159680366516,
      "logps/chosen": -429.1424865722656,
      "logps/rejected": -489.776123046875,
      "loss": 0.083,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08991511166095734,
      "rewards/margins": 0.04996410012245178,
      "rewards/rejected": -0.13987921178340912,
      "step": 720
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2714955182448044e-06,
      "logits/chosen": 0.06632956117391586,
      "logits/rejected": 0.10460522025823593,
      "logps/chosen": -355.56988525390625,
      "logps/rejected": -453.8470764160156,
      "loss": 0.1004,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07422816753387451,
      "rewards/margins": 0.059829723089933395,
      "rewards/rejected": -0.1340578943490982,
      "step": 730
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2189606662871017e-06,
      "logits/chosen": 0.1409912258386612,
      "logits/rejected": 0.14539703726768494,
      "logps/chosen": -390.64251708984375,
      "logps/rejected": -435.5365295410156,
      "loss": 0.0906,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07141707837581635,
      "rewards/margins": 0.03945548087358475,
      "rewards/rejected": -0.1108725517988205,
      "step": 740
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1660773651956223e-06,
      "logits/chosen": 0.12456997483968735,
      "logits/rejected": 0.1372366100549698,
      "logps/chosen": -355.9635314941406,
      "logps/rejected": -383.43853759765625,
      "loss": 0.1004,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07638130336999893,
      "rewards/margins": 0.037354812026023865,
      "rewards/rejected": -0.1137361153960228,
      "step": 750
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1128712452187535e-06,
      "logits/chosen": 0.09267710149288177,
      "logits/rejected": 0.07987795025110245,
      "logps/chosen": -402.697509765625,
      "logps/rejected": -450.74786376953125,
      "loss": 0.0967,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08393266797065735,
      "rewards/margins": 0.04311661794781685,
      "rewards/rejected": -0.1270492970943451,
      "step": 760
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0593680930612295e-06,
      "logits/chosen": 0.0781303271651268,
      "logits/rejected": 0.16469840705394745,
      "logps/chosen": -437.64013671875,
      "logps/rejected": -474.09661865234375,
      "loss": 0.1021,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.10180965811014175,
      "rewards/margins": 0.03583382070064545,
      "rewards/rejected": -0.1376434713602066,
      "step": 770
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.005593839386434e-06,
      "logits/chosen": 0.10138174146413803,
      "logits/rejected": 0.1595858335494995,
      "logps/chosen": -466.75811767578125,
      "logps/rejected": -494.52435302734375,
      "loss": 0.0913,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.09192174673080444,
      "rewards/margins": 0.048992156982421875,
      "rewards/rejected": -0.14091390371322632,
      "step": 780
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.951574546248928e-06,
      "logits/chosen": 0.09503944218158722,
      "logits/rejected": 0.1364067792892456,
      "logps/chosen": -458.38775634765625,
      "logps/rejected": -459.333251953125,
      "loss": 0.0985,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.09930691868066788,
      "rewards/margins": 0.029138978570699692,
      "rewards/rejected": -0.12844590842723846,
      "step": 790
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.897336394463301e-06,
      "logits/chosen": 0.10846900939941406,
      "logits/rejected": 0.09937886893749237,
      "logps/chosen": -424.815673828125,
      "logps/rejected": -459.7933044433594,
      "loss": 0.0941,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.08887843042612076,
      "rewards/margins": 0.05371922254562378,
      "rewards/rejected": -0.14259764552116394,
      "step": 800
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.842905670915462e-06,
      "logits/chosen": 0.1058458462357521,
      "logits/rejected": 0.0782022550702095,
      "logps/chosen": -434.9790954589844,
      "logps/rejected": -469.34454345703125,
      "loss": 0.0967,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.09677759557962418,
      "rewards/margins": 0.04304363578557968,
      "rewards/rejected": -0.13982121646404266,
      "step": 810
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.788308755822522e-06,
      "logits/chosen": 0.0967346802353859,
      "logits/rejected": 0.10881376266479492,
      "logps/chosen": -369.01995849609375,
      "logps/rejected": -449.07623291015625,
      "loss": 0.0926,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.09960204362869263,
      "rewards/margins": 0.04991160333156586,
      "rewards/rejected": -0.1495136320590973,
      "step": 820
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.733572109947444e-06,
      "logits/chosen": 0.10651399195194244,
      "logits/rejected": 0.128664031624794,
      "logps/chosen": -466.0299377441406,
      "logps/rejected": -499.25762939453125,
      "loss": 0.0972,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1191006675362587,
      "rewards/margins": 0.044090814888477325,
      "rewards/rejected": -0.16319148242473602,
      "step": 830
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6787222617746583e-06,
      "logits/chosen": 0.12026312202215195,
      "logits/rejected": 0.1642097681760788,
      "logps/chosen": -380.1086120605469,
      "logps/rejected": -439.4449157714844,
      "loss": 0.0884,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09910561889410019,
      "rewards/margins": 0.04509486258029938,
      "rewards/rejected": -0.14420048892498016,
      "step": 840
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.62378579465285e-06,
      "logits/chosen": 0.12080010026693344,
      "logits/rejected": 0.15950220823287964,
      "logps/chosen": -441.6851501464844,
      "logps/rejected": -498.75152587890625,
      "loss": 0.0919,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.09932798892259598,
      "rewards/margins": 0.05052763968706131,
      "rewards/rejected": -0.14985564351081848,
      "step": 850
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.568789333911165e-06,
      "logits/chosen": 0.1327330321073532,
      "logits/rejected": 0.14310786128044128,
      "logps/chosen": -375.2086486816406,
      "logps/rejected": -488.7005310058594,
      "loss": 0.0957,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.08790336549282074,
      "rewards/margins": 0.0683864951133728,
      "rewards/rejected": -0.15628986060619354,
      "step": 860
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5137595339550564e-06,
      "logits/chosen": 0.10466363281011581,
      "logits/rejected": 0.12011264264583588,
      "logps/chosen": -457.970458984375,
      "logps/rejected": -444.8792419433594,
      "loss": 0.0943,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.10931634902954102,
      "rewards/margins": 0.027462299913167953,
      "rewards/rejected": -0.13677865266799927,
      "step": 870
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4587230653480577e-06,
      "logits/chosen": 0.07321979105472565,
      "logits/rejected": 0.09171778708696365,
      "logps/chosen": -435.8045349121094,
      "logps/rejected": -464.2123107910156,
      "loss": 0.0896,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.11067759990692139,
      "rewards/margins": 0.045581571757793427,
      "rewards/rejected": -0.15625916421413422,
      "step": 880
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.403706601885705e-06,
      "logits/chosen": 0.12501958012580872,
      "logits/rejected": 0.10027003288269043,
      "logps/chosen": -439.4681701660156,
      "logps/rejected": -450.8436584472656,
      "loss": 0.0988,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.09043081104755402,
      "rewards/margins": 0.03802182897925377,
      "rewards/rejected": -0.12845264375209808,
      "step": 890
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3487368076679095e-06,
      "logits/chosen": 0.03233950585126877,
      "logits/rejected": 0.10005395114421844,
      "logps/chosen": -423.36163330078125,
      "logps/rejected": -499.0625,
      "loss": 0.0823,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.10427310317754745,
      "rewards/margins": 0.07294373214244843,
      "rewards/rejected": -0.17721685767173767,
      "step": 900
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.2938403241760153e-06,
      "logits/chosen": 0.046649765223264694,
      "logits/rejected": 0.06434310227632523,
      "logps/chosen": -385.28753662109375,
      "logps/rejected": -428.0790100097656,
      "loss": 0.0921,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.097981296479702,
      "rewards/margins": 0.046177938580513,
      "rewards/rejected": -0.1441592425107956,
      "step": 910
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2390437573608313e-06,
      "logits/chosen": 0.04348721355199814,
      "logits/rejected": 0.09973670542240143,
      "logps/chosen": -419.55841064453125,
      "logps/rejected": -455.224853515625,
      "loss": 0.0924,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.11863915622234344,
      "rewards/margins": 0.03215927258133888,
      "rewards/rejected": -0.15079843997955322,
      "step": 920
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1843736647478733e-06,
      "logits/chosen": 0.12401044368743896,
      "logits/rejected": 0.11555816233158112,
      "logps/chosen": -461.777099609375,
      "logps/rejected": -446.89154052734375,
      "loss": 0.0958,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.10517005622386932,
      "rewards/margins": 0.03241740167140961,
      "rewards/rejected": -0.13758745789527893,
      "step": 930
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1298565425660842e-06,
      "logits/chosen": 0.12125201523303986,
      "logits/rejected": 0.13117629289627075,
      "logps/chosen": -443.516845703125,
      "logps/rejected": -483.62030029296875,
      "loss": 0.086,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.09896551072597504,
      "rewards/margins": 0.045016057789325714,
      "rewards/rejected": -0.14398157596588135,
      "step": 940
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0755188129062566e-06,
      "logits/chosen": 0.1026390790939331,
      "logits/rejected": 0.09158008545637131,
      "logps/chosen": -444.39862060546875,
      "logps/rejected": -479.50390625,
      "loss": 0.0983,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.10992302745580673,
      "rewards/margins": 0.03600381314754486,
      "rewards/rejected": -0.145926833152771,
      "step": 950
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.0213868109153843e-06,
      "logits/chosen": 0.0887732207775116,
      "logits/rejected": 0.13315047323703766,
      "logps/chosen": -451.1080627441406,
      "logps/rejected": -477.95477294921875,
      "loss": 0.0883,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.10277727991342545,
      "rewards/margins": 0.045780349522829056,
      "rewards/rejected": -0.1485576331615448,
      "step": 960
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.967486772033164e-06,
      "logits/chosen": 0.1042795404791832,
      "logits/rejected": 0.1479608118534088,
      "logps/chosen": -433.328125,
      "logps/rejected": -472.20904541015625,
      "loss": 0.0802,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09643669426441193,
      "rewards/margins": 0.04762507602572441,
      "rewards/rejected": -0.14406177401542664,
      "step": 970
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9138448192768072e-06,
      "logits/chosen": 0.08118772506713867,
      "logits/rejected": 0.09627708047628403,
      "logps/chosen": -424.64898681640625,
      "logps/rejected": -493.44476318359375,
      "loss": 0.0919,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08744023740291595,
      "rewards/margins": 0.0591917559504509,
      "rewards/rejected": -0.14663198590278625,
      "step": 980
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8604869505803407e-06,
      "logits/chosen": 0.1322592794895172,
      "logits/rejected": 0.13863101601600647,
      "logps/chosen": -423.8280334472656,
      "logps/rejected": -469.0419921875,
      "loss": 0.0978,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.09548251330852509,
      "rewards/margins": 0.05783646181225777,
      "rewards/rejected": -0.15331897139549255,
      "step": 990
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8074390261945418e-06,
      "logits/chosen": 0.10184427350759506,
      "logits/rejected": 0.12423571199178696,
      "logps/chosen": -377.3212585449219,
      "logps/rejected": -427.81317138671875,
      "loss": 0.0902,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.0946311205625534,
      "rewards/margins": 0.046779900789260864,
      "rewards/rejected": -0.14141102135181427,
      "step": 1000
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7547267561535836e-06,
      "logits/chosen": 0.06255786120891571,
      "logits/rejected": 0.12503619492053986,
      "logps/chosen": -418.79144287109375,
      "logps/rejected": -441.3435974121094,
      "loss": 0.0864,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.10577164590358734,
      "rewards/margins": 0.05561785027384758,
      "rewards/rejected": -0.16138949990272522,
      "step": 1010
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.702375687814501e-06,
      "logits/chosen": 0.10921992361545563,
      "logits/rejected": 0.0866740271449089,
      "logps/chosen": -404.1749267578125,
      "logps/rejected": -476.277099609375,
      "loss": 0.0791,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.10695375502109528,
      "rewards/margins": 0.05870165675878525,
      "rewards/rejected": -0.16565541923046112,
      "step": 1020
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.650411193475489e-06,
      "logits/chosen": 0.074369415640831,
      "logits/rejected": 0.12412852048873901,
      "logps/chosen": -420.69793701171875,
      "logps/rejected": -484.34161376953125,
      "loss": 0.0868,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.11432509124279022,
      "rewards/margins": 0.051989488303661346,
      "rewards/rejected": -0.16631458699703217,
      "step": 1030
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5988584580790452e-06,
      "logits/chosen": 0.09442147612571716,
      "logits/rejected": 0.10659205913543701,
      "logps/chosen": -458.6270446777344,
      "logps/rejected": -523.6939697265625,
      "loss": 0.0705,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10721229016780853,
      "rewards/margins": 0.05447040870785713,
      "rewards/rejected": -0.16168268024921417,
      "step": 1040
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5477424670059227e-06,
      "logits/chosen": 0.08504728972911835,
      "logits/rejected": 0.10464467108249664,
      "logps/chosen": -423.983154296875,
      "logps/rejected": -449.3802185058594,
      "loss": 0.0851,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.11461503803730011,
      "rewards/margins": 0.03833803907036781,
      "rewards/rejected": -0.15295308828353882,
      "step": 1050
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4970879939657929e-06,
      "logits/chosen": 0.08304135501384735,
      "logits/rejected": 0.11797134578227997,
      "logps/chosen": -407.0821838378906,
      "logps/rejected": -456.46533203125,
      "loss": 0.0796,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.08808410167694092,
      "rewards/margins": 0.05510552600026131,
      "rewards/rejected": -0.14318962395191193,
      "step": 1060
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4469195889904983e-06,
      "logits/chosen": 0.044432397931814194,
      "logits/rejected": 0.10810061544179916,
      "logps/chosen": -429.70794677734375,
      "logps/rejected": -452.116943359375,
      "loss": 0.0926,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10013709962368011,
      "rewards/margins": 0.03565113991498947,
      "rewards/rejected": -0.13578826189041138,
      "step": 1070
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3972615665357228e-06,
      "logits/chosen": 0.08212409913539886,
      "logits/rejected": 0.08114130049943924,
      "logps/chosen": -425.5025939941406,
      "logps/rejected": -518.18115234375,
      "loss": 0.0884,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09838120639324188,
      "rewards/margins": 0.061208415776491165,
      "rewards/rejected": -0.15958961844444275,
      "step": 1080
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3481379936968154e-06,
      "logits/chosen": 0.06021580845117569,
      "logits/rejected": 0.1053982749581337,
      "logps/chosen": -471.2576599121094,
      "logps/rejected": -499.8750915527344,
      "loss": 0.1021,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.10752852261066437,
      "rewards/margins": 0.04095206782221794,
      "rewards/rejected": -0.1484805792570114,
      "step": 1090
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.2995726785445187e-06,
      "logits/chosen": 0.08110491931438446,
      "logits/rejected": 0.13657256960868835,
      "logps/chosen": -457.90234375,
      "logps/rejected": -445.5087890625,
      "loss": 0.0918,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.11181896924972534,
      "rewards/margins": 0.03636185824871063,
      "rewards/rejected": -0.14818084239959717,
      "step": 1100
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2515891585862248e-06,
      "logits/chosen": 0.06536366045475006,
      "logits/rejected": 0.10223551094532013,
      "logps/chosen": -424.5037536621094,
      "logps/rejected": -465.29425048828125,
      "loss": 0.112,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10022538900375366,
      "rewards/margins": 0.03513466566801071,
      "rewards/rejected": -0.13536006212234497,
      "step": 1110
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.204210689358359e-06,
      "logits/chosen": 0.15191073715686798,
      "logits/rejected": 0.12366287410259247,
      "logps/chosen": -436.87823486328125,
      "logps/rejected": -426.62762451171875,
      "loss": 0.0811,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.09861338883638382,
      "rewards/margins": 0.033628519624471664,
      "rewards/rejected": -0.13224190473556519,
      "step": 1120
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1574602331554352e-06,
      "logits/chosen": 0.05633612722158432,
      "logits/rejected": 0.04986396059393883,
      "logps/chosen": -424.77984619140625,
      "logps/rejected": -440.7940979003906,
      "loss": 0.1042,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1184263676404953,
      "rewards/margins": 0.029354307800531387,
      "rewards/rejected": -0.1477806717157364,
      "step": 1130
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1113604479012202e-06,
      "logits/chosen": 0.04834729805588722,
      "logits/rejected": 0.079189732670784,
      "logps/chosen": -402.3204040527344,
      "logps/rejected": -482.1253967285156,
      "loss": 0.1046,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.10266365110874176,
      "rewards/margins": 0.04556725174188614,
      "rewards/rejected": -0.1482308954000473,
      "step": 1140
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0659336761674163e-06,
      "logits/chosen": 0.07221159338951111,
      "logits/rejected": 0.06988708674907684,
      "logps/chosen": -378.05255126953125,
      "logps/rejected": -401.8747863769531,
      "loss": 0.0872,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.09310795366764069,
      "rewards/margins": 0.04454265162348747,
      "rewards/rejected": -0.13765059411525726,
      "step": 1150
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0212019343451889e-06,
      "logits/chosen": 0.051744282245635986,
      "logits/rejected": 0.05535260960459709,
      "logps/chosen": -367.1768798828125,
      "logps/rejected": -443.02960205078125,
      "loss": 0.0823,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.08505652099847794,
      "rewards/margins": 0.06588119268417358,
      "rewards/rejected": -0.15093770623207092,
      "step": 1160
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.771869019747708e-07,
      "logits/chosen": 0.0807739868760109,
      "logits/rejected": 0.09035824239253998,
      "logps/chosen": -437.5768127441406,
      "logps/rejected": -521.4659423828125,
      "loss": 0.0918,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.11788342148065567,
      "rewards/margins": 0.07230274379253387,
      "rewards/rejected": -0.19018617272377014,
      "step": 1170
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.339099112383263e-07,
      "logits/chosen": 0.08397279679775238,
      "logits/rejected": 0.12051109224557877,
      "logps/chosen": -429.1914978027344,
      "logps/rejected": -477.98968505859375,
      "loss": 0.0898,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.1165337786078453,
      "rewards/margins": 0.0657678097486496,
      "rewards/rejected": -0.1823015958070755,
      "step": 1180
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.913919366211679e-07,
      "logits/chosen": 0.01225881278514862,
      "logits/rejected": 0.08070291578769684,
      "logps/chosen": -453.7749938964844,
      "logps/rejected": -491.718017578125,
      "loss": 0.0784,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.12794212996959686,
      "rewards/margins": 0.046219903975725174,
      "rewards/rejected": -0.17416204512119293,
      "step": 1190
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.496535847463264e-07,
      "logits/chosen": 0.05466824769973755,
      "logits/rejected": 0.06951786577701569,
      "logps/chosen": -409.43731689453125,
      "logps/rejected": -478.67730712890625,
      "loss": 0.0878,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.0958373174071312,
      "rewards/margins": 0.06815023720264435,
      "rewards/rejected": -0.16398754715919495,
      "step": 1200
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.08715084387417e-07,
      "logits/chosen": 0.0821441262960434,
      "logits/rejected": 0.09297174215316772,
      "logps/chosen": -443.9449768066406,
      "logps/rejected": -495.43304443359375,
      "loss": 0.0746,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.09165932238101959,
      "rewards/margins": 0.06980036199092865,
      "rewards/rejected": -0.16145966947078705,
      "step": 1210
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.685962766646251e-07,
      "logits/chosen": 0.0996304377913475,
      "logits/rejected": 0.1416376680135727,
      "logps/chosen": -421.4239807128906,
      "logps/rejected": -448.3440856933594,
      "loss": 0.0854,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1046370267868042,
      "rewards/margins": 0.04357679933309555,
      "rewards/rejected": -0.14821383357048035,
      "step": 1220
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.29316605428573e-07,
      "logits/chosen": 0.050212837755680084,
      "logits/rejected": 0.08997879177331924,
      "logps/chosen": -405.666748046875,
      "logps/rejected": -481.518798828125,
      "loss": 0.0874,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.10379258543252945,
      "rewards/margins": 0.06835626810789108,
      "rewards/rejected": -0.17214885354042053,
      "step": 1230
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.908951078367284e-07,
      "logits/chosen": 0.06391054391860962,
      "logits/rejected": 0.11078351736068726,
      "logps/chosen": -472.7290954589844,
      "logps/rejected": -525.4988403320312,
      "loss": 0.0804,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.11707256734371185,
      "rewards/margins": 0.06153646111488342,
      "rewards/rejected": -0.17860902845859528,
      "step": 1240
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.533504051269185e-07,
      "logits/chosen": 0.09103193879127502,
      "logits/rejected": 0.1026821881532669,
      "logps/chosen": -356.58807373046875,
      "logps/rejected": -444.40869140625,
      "loss": 0.0813,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.08095113933086395,
      "rewards/margins": 0.059402190148830414,
      "rewards/rejected": -0.14035332202911377,
      "step": 1250
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.167006935924164e-07,
      "logits/chosen": 0.07466550171375275,
      "logits/rejected": 0.08906082808971405,
      "logps/chosen": -419.82733154296875,
      "logps/rejected": -447.23419189453125,
      "loss": 0.0858,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.09368356317281723,
      "rewards/margins": 0.049560584127902985,
      "rewards/rejected": -0.14324414730072021,
      "step": 1260
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.809637357629916e-07,
      "logits/chosen": 0.09017828106880188,
      "logits/rejected": 0.07519755512475967,
      "logps/chosen": -386.6206970214844,
      "logps/rejected": -444.78900146484375,
      "loss": 0.0884,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.09151438623666763,
      "rewards/margins": 0.0505540557205677,
      "rewards/rejected": -0.14206844568252563,
      "step": 1270
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.461568517961763e-07,
      "logits/chosen": 0.022731300443410873,
      "logits/rejected": 0.03403092175722122,
      "logps/chosen": -406.1392822265625,
      "logps/rejected": -505.94366455078125,
      "loss": 0.0879,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11663617938756943,
      "rewards/margins": 0.07096937298774719,
      "rewards/rejected": -0.18760555982589722,
      "step": 1280
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.122969110829346e-07,
      "logits/chosen": 0.03181394562125206,
      "logits/rejected": 0.045416783541440964,
      "logps/chosen": -403.97100830078125,
      "logps/rejected": -524.0074462890625,
      "loss": 0.0895,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.11092919111251831,
      "rewards/margins": 0.07563652843236923,
      "rewards/rejected": -0.18656572699546814,
      "step": 1290
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.794003240718004e-07,
      "logits/chosen": 0.09289757162332535,
      "logits/rejected": 0.09983552247285843,
      "logps/chosen": -467.2046813964844,
      "logps/rejected": -487.23760986328125,
      "loss": 0.095,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10705260187387466,
      "rewards/margins": 0.050529349595308304,
      "rewards/rejected": -0.15758195519447327,
      "step": 1300
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.4748303431543907e-07,
      "logits/chosen": 0.07837380468845367,
      "logits/rejected": 0.10405907779932022,
      "logps/chosen": -479.838134765625,
      "logps/rejected": -476.2897033691406,
      "loss": 0.0852,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.10937539488077164,
      "rewards/margins": 0.04146861657500267,
      "rewards/rejected": -0.150844007730484,
      "step": 1310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1656051074350113e-07,
      "logits/chosen": 0.075148805975914,
      "logits/rejected": 0.08904428780078888,
      "logps/chosen": -405.589599609375,
      "logps/rejected": -448.927734375,
      "loss": 0.0901,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09542171657085419,
      "rewards/margins": 0.03852013498544693,
      "rewards/rejected": -0.1339418590068817,
      "step": 1320
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.866477401655e-07,
      "logits/chosen": 0.07279117405414581,
      "logits/rejected": 0.08455020189285278,
      "logps/chosen": -385.3067626953125,
      "logps/rejected": -440.7181701660156,
      "loss": 0.0822,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.09135736525058746,
      "rewards/margins": 0.04145896062254906,
      "rewards/rejected": -0.13281632959842682,
      "step": 1330
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.577592200073518e-07,
      "logits/chosen": 0.05959012359380722,
      "logits/rejected": 0.04779018834233284,
      "logps/chosen": -411.390625,
      "logps/rejected": -462.2435607910156,
      "loss": 0.0844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09003390371799469,
      "rewards/margins": 0.03744681924581528,
      "rewards/rejected": -0.12748071551322937,
      "step": 1340
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.2990895128510364e-07,
      "logits/chosen": 0.05087254196405411,
      "logits/rejected": 0.07081006467342377,
      "logps/chosen": -413.0889587402344,
      "logps/rejected": -446.05242919921875,
      "loss": 0.0881,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.09529262781143188,
      "rewards/margins": 0.045550812035799026,
      "rewards/rejected": -0.1408434361219406,
      "step": 1350
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.0311043181924237e-07,
      "logits/chosen": 0.028466597199440002,
      "logits/rejected": 0.06887885928153992,
      "logps/chosen": -412.6106872558594,
      "logps/rejected": -454.3301696777344,
      "loss": 0.0969,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.10608142614364624,
      "rewards/margins": 0.02706090547144413,
      "rewards/rejected": -0.13314232230186462,
      "step": 1360
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.773766496928898e-07,
      "logits/chosen": 0.06292058527469635,
      "logits/rejected": 0.08972954005002975,
      "logps/chosen": -411.333251953125,
      "logps/rejected": -474.076416015625,
      "loss": 0.0967,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0901075154542923,
      "rewards/margins": 0.06678055226802826,
      "rewards/rejected": -0.15688805282115936,
      "step": 1370
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.527200769570376e-07,
      "logits/chosen": 0.05736211687326431,
      "logits/rejected": 0.047862760722637177,
      "logps/chosen": -428.587890625,
      "logps/rejected": -487.86029052734375,
      "loss": 0.091,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.11271530389785767,
      "rewards/margins": 0.05474567413330078,
      "rewards/rejected": -0.16746096312999725,
      "step": 1380
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.291526635858829e-07,
      "logits/chosen": 0.033168576657772064,
      "logits/rejected": 0.07321670651435852,
      "logps/chosen": -406.710205078125,
      "logps/rejected": -515.9356079101562,
      "loss": 0.0829,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.09568953514099121,
      "rewards/margins": 0.058294862508773804,
      "rewards/rejected": -0.1539844125509262,
      "step": 1390
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0668583168519608e-07,
      "logits/chosen": 0.047509074211120605,
      "logits/rejected": 0.06223832815885544,
      "logps/chosen": -388.98193359375,
      "logps/rejected": -471.21136474609375,
      "loss": 0.0839,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.09332209825515747,
      "rewards/margins": 0.04638640210032463,
      "rewards/rejected": -0.1397085040807724,
      "step": 1400
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8533046995651755e-07,
      "logits/chosen": 0.06117665767669678,
      "logits/rejected": 0.05806068703532219,
      "logps/chosen": -398.0728454589844,
      "logps/rejected": -476.2325744628906,
      "loss": 0.0928,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.10759732872247696,
      "rewards/margins": 0.06671231985092163,
      "rewards/rejected": -0.1743096262216568,
      "step": 1410
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6509692841987557e-07,
      "logits/chosen": 0.09554895013570786,
      "logits/rejected": 0.11183027923107147,
      "logps/chosen": -373.590576171875,
      "logps/rejected": -459.16046142578125,
      "loss": 0.0958,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.09168444573879242,
      "rewards/margins": 0.05873497575521469,
      "rewards/rejected": -0.1504194289445877,
      "step": 1420
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4599501339758043e-07,
      "logits/chosen": 0.14753518998622894,
      "logits/rejected": 0.15233811736106873,
      "logps/chosen": -391.5714416503906,
      "logps/rejected": -454.29901123046875,
      "loss": 0.0838,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07346367090940475,
      "rewards/margins": 0.0692339688539505,
      "rewards/rejected": -0.14269763231277466,
      "step": 1430
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2803398276152236e-07,
      "logits/chosen": 0.0984094962477684,
      "logits/rejected": 0.158823162317276,
      "logps/chosen": -421.304931640625,
      "logps/rejected": -486.080810546875,
      "loss": 0.0911,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.10145977884531021,
      "rewards/margins": 0.05414135009050369,
      "rewards/rejected": -0.1556011140346527,
      "step": 1440
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1122254144628308e-07,
      "logits/chosen": 0.03182172030210495,
      "logits/rejected": 0.06180509179830551,
      "logps/chosen": -370.4424743652344,
      "logps/rejected": -450.4090881347656,
      "loss": 0.0875,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.09365768730640411,
      "rewards/margins": 0.041229672729969025,
      "rewards/rejected": -0.13488735258579254,
      "step": 1450
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.556883723023047e-08,
      "logits/chosen": 0.03198453038930893,
      "logits/rejected": 0.09605802595615387,
      "logps/chosen": -458.57366943359375,
      "logps/rejected": -517.7696533203125,
      "loss": 0.0928,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.10384533554315567,
      "rewards/margins": 0.06892184168100357,
      "rewards/rejected": -0.17276716232299805,
      "step": 1460
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.108045678664133e-08,
      "logits/chosen": 0.046630989760160446,
      "logits/rejected": 0.11219167709350586,
      "logps/chosen": -429.394287109375,
      "logps/rejected": -471.2891540527344,
      "loss": 0.0851,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.10102847963571548,
      "rewards/margins": 0.04838350787758827,
      "rewards/rejected": -0.14941199123859406,
      "step": 1470
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.776442200677136e-08,
      "logits/chosen": 0.07663191854953766,
      "logits/rejected": 0.08462060987949371,
      "logps/chosen": -418.231201171875,
      "logps/rejected": -460.2142639160156,
      "loss": 0.0978,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08978335559368134,
      "rewards/margins": 0.05147411674261093,
      "rewards/rejected": -0.14125746488571167,
      "step": 1480
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.562718659664651e-08,
      "logits/chosen": 0.05606203153729439,
      "logits/rejected": 0.03971995785832405,
      "logps/chosen": -446.9505920410156,
      "logps/rejected": -492.7408752441406,
      "loss": 0.1003,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.11127449572086334,
      "rewards/margins": 0.049640461802482605,
      "rewards/rejected": -0.16091494262218475,
      "step": 1490
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.467463294923091e-08,
      "logits/chosen": 0.07213045656681061,
      "logits/rejected": 0.11403784900903702,
      "logps/chosen": -408.98272705078125,
      "logps/rejected": -505.5140686035156,
      "loss": 0.0818,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.10484303534030914,
      "rewards/margins": 0.07243579626083374,
      "rewards/rejected": -0.17727883160114288,
      "step": 1500
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.491206929348595e-08,
      "logits/chosen": 0.09976326674222946,
      "logits/rejected": 0.11376956850290298,
      "logps/chosen": -462.38946533203125,
      "logps/rejected": -540.771484375,
      "loss": 0.083,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.11446479707956314,
      "rewards/margins": 0.06572078168392181,
      "rewards/rejected": -0.18018554151058197,
      "step": 1510
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.634422712170209e-08,
      "logits/chosen": 0.06767505407333374,
      "logits/rejected": 0.08407372236251831,
      "logps/chosen": -445.62646484375,
      "logps/rejected": -456.46685791015625,
      "loss": 0.0938,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.11092151701450348,
      "rewards/margins": 0.028619635850191116,
      "rewards/rejected": -0.1395411491394043,
      "step": 1520
    },
    {
      "epoch": 0.96,
      "learning_rate": 1.8975258896347383e-08,
      "logits/chosen": 0.03723181039094925,
      "logits/rejected": 0.07889989018440247,
      "logps/chosen": -400.19281005859375,
      "logps/rejected": -445.9078063964844,
      "loss": 0.1039,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.09558527171611786,
      "rewards/margins": 0.049675554037094116,
      "rewards/rejected": -0.14526084065437317,
      "step": 1530
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.2808736037550618e-08,
      "logits/chosen": 0.022983688861131668,
      "logits/rejected": 0.07783511281013489,
      "logps/chosen": -435.6316833496094,
      "logps/rejected": -445.15118408203125,
      "loss": 0.0877,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.104395292699337,
      "rewards/margins": 0.03368845209479332,
      "rewards/rejected": -0.13808374106884003,
      "step": 1540
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.847647192186226e-09,
      "logits/chosen": 0.08826269954442978,
      "logits/rejected": 0.08017078787088394,
      "logps/chosen": -440.181884765625,
      "logps/rejected": -492.92510986328125,
      "loss": 0.0749,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.09225556254386902,
      "rewards/margins": 0.06564488261938095,
      "rewards/rejected": -0.15790043771266937,
      "step": 1550
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.094396785408461e-09,
      "logits/chosen": 0.050535012036561966,
      "logits/rejected": 0.09702993929386139,
      "logps/chosen": -396.10906982421875,
      "logps/rejected": -434.914794921875,
      "loss": 0.0964,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.09921000152826309,
      "rewards/margins": 0.04018466919660568,
      "rewards/rejected": -0.13939465582370758,
      "step": 1560
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.5508038553291304e-09,
      "logits/chosen": 0.06177855283021927,
      "logits/rejected": 0.08348623663187027,
      "logps/chosen": -436.14166259765625,
      "logps/rejected": -509.82373046875,
      "loss": 0.0862,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.09413688629865646,
      "rewards/margins": 0.05878259614109993,
      "rewards/rejected": -0.15291950106620789,
      "step": 1570
    },
    {
      "epoch": 1.0,
      "learning_rate": 2.1810117140974584e-10,
      "logits/chosen": 0.024281805381178856,
      "logits/rejected": 0.09278794378042221,
      "logps/chosen": -491.00299072265625,
      "logps/rejected": -472.14349365234375,
      "loss": 0.1018,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.11561523377895355,
      "rewards/margins": 0.04595177248120308,
      "rewards/rejected": -0.16156700253486633,
      "step": 1580
    },
    {
      "epoch": 1.0,
      "step": 1586,
      "total_flos": 0.0,
      "train_loss": 0.09621614837856943,
      "train_runtime": 21999.9134,
      "train_samples_per_second": 3.461,
      "train_steps_per_second": 0.072
    }
  ],
  "logging_steps": 10,
  "max_steps": 1586,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}