{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 625,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 7.936507936507937e-08,
      "logits/chosen": 0.3197985589504242,
      "logits/rejected": 0.2375701665878296,
      "logps/chosen": -309.8530578613281,
      "logps/rejected": -345.0298156738281,
      "loss": 0.3912,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.936507936507937e-07,
      "logits/chosen": 0.13694973289966583,
      "logits/rejected": 0.3060193359851837,
      "logps/chosen": -419.0776062011719,
      "logps/rejected": -389.68438720703125,
      "loss": 0.3933,
      "rewards/accuracies": 0.3194444477558136,
      "rewards/chosen": -0.0009195790044032037,
      "rewards/margins": -8.771694410825148e-05,
      "rewards/rejected": -0.0008318620966747403,
      "step": 10
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5873015873015873e-06,
      "logits/chosen": 0.16718418896198273,
      "logits/rejected": 0.27480465173721313,
      "logps/chosen": -394.2663879394531,
      "logps/rejected": -397.93658447265625,
      "loss": 0.378,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0013991680461913347,
      "rewards/margins": -8.433303446508944e-05,
      "rewards/rejected": -0.001314834924414754,
      "step": 20
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.380952380952381e-06,
      "logits/chosen": 0.24418623745441437,
      "logits/rejected": 0.2495376318693161,
      "logps/chosen": -425.3749084472656,
      "logps/rejected": -403.7620849609375,
      "loss": 0.3871,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.0024094637483358383,
      "rewards/margins": 6.462958117481321e-05,
      "rewards/rejected": -0.0024740933440625668,
      "step": 30
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1746031746031746e-06,
      "logits/chosen": 0.24052706360816956,
      "logits/rejected": 0.20116524398326874,
      "logps/chosen": -398.9176330566406,
      "logps/rejected": -420.3963928222656,
      "loss": 0.3636,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.0041575380600988865,
      "rewards/margins": 4.97179280500859e-05,
      "rewards/rejected": -0.004207254853099585,
      "step": 40
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.968253968253968e-06,
      "logits/chosen": 0.2002558708190918,
      "logits/rejected": 0.2225637137889862,
      "logps/chosen": -400.6484680175781,
      "logps/rejected": -394.4924011230469,
      "loss": 0.4006,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.006101720035076141,
      "rewards/margins": 0.0004920283681713045,
      "rewards/rejected": -0.006593747995793819,
      "step": 50
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.761904761904762e-06,
      "logits/chosen": 0.24047236144542694,
      "logits/rejected": 0.20159101486206055,
      "logps/chosen": -390.0282287597656,
      "logps/rejected": -406.8359375,
      "loss": 0.3709,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.005089767277240753,
      "rewards/margins": 0.000577005441300571,
      "rewards/rejected": -0.005666772834956646,
      "step": 60
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998086282661188e-06,
      "logits/chosen": 0.18658548593521118,
      "logits/rejected": 0.23578360676765442,
      "logps/chosen": -413.97216796875,
      "logps/rejected": -415.7422790527344,
      "loss": 0.3699,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.007824992761015892,
      "rewards/margins": 0.0006475156405940652,
      "rewards/rejected": -0.008472507819533348,
      "step": 70
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.988720025682995e-06,
      "logits/chosen": 0.12549163401126862,
      "logits/rejected": 0.11077409982681274,
      "logps/chosen": -370.007568359375,
      "logps/rejected": -357.638427734375,
      "loss": 0.3601,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.011440587230026722,
      "rewards/margins": 0.0010136803612113,
      "rewards/rejected": -0.012454266659915447,
      "step": 80
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9715789537359126e-06,
      "logits/chosen": 0.03908854350447655,
      "logits/rejected": 0.18181201815605164,
      "logps/chosen": -408.84930419921875,
      "logps/rejected": -391.1748962402344,
      "loss": 0.3651,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.017708633095026016,
      "rewards/margins": 0.001199634512886405,
      "rewards/rejected": -0.018908265978097916,
      "step": 90
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.946716615897932e-06,
      "logits/chosen": 0.15611937642097473,
      "logits/rejected": 0.21340712904930115,
      "logps/chosen": -437.1705017089844,
      "logps/rejected": -466.39544677734375,
      "loss": 0.3702,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.016876522451639175,
      "rewards/margins": 0.004214240703731775,
      "rewards/rejected": -0.021090764552354813,
      "step": 100
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9142106826480114e-06,
      "logits/chosen": 0.1575128138065338,
      "logits/rejected": 0.14739355444908142,
      "logps/chosen": -394.54949951171875,
      "logps/rejected": -417.59930419921875,
      "loss": 0.3609,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.013111593201756477,
      "rewards/margins": 0.00490961829200387,
      "rewards/rejected": -0.01802121102809906,
      "step": 110
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.874162703221823e-06,
      "logits/chosen": 0.20379705727100372,
      "logits/rejected": 0.27394187450408936,
      "logps/chosen": -443.15185546875,
      "logps/rejected": -441.23095703125,
      "loss": 0.3603,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.025930130854249,
      "rewards/margins": 0.007792765740305185,
      "rewards/rejected": -0.0337228998541832,
      "step": 120
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.826697788369752e-06,
      "logits/chosen": 0.10621203482151031,
      "logits/rejected": 0.1633463203907013,
      "logps/chosen": -446.20947265625,
      "logps/rejected": -472.85174560546875,
      "loss": 0.3577,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.039154283702373505,
      "rewards/margins": 0.009123672731220722,
      "rewards/rejected": -0.04827795922756195,
      "step": 130
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7719642195082224e-06,
      "logits/chosen": 0.11126575618982315,
      "logits/rejected": 0.15343426167964935,
      "logps/chosen": -466.1980895996094,
      "logps/rejected": -454.23876953125,
      "loss": 0.3654,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.053963106125593185,
      "rewards/margins": 0.010927463881671429,
      "rewards/rejected": -0.06489057093858719,
      "step": 140
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.710132985485355e-06,
      "logits/chosen": 0.0875745341181755,
      "logits/rejected": 0.14972081780433655,
      "logps/chosen": -499.6947326660156,
      "logps/rejected": -533.8878173828125,
      "loss": 0.3411,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.08158211410045624,
      "rewards/margins": 0.02510920725762844,
      "rewards/rejected": -0.10669132322072983,
      "step": 150
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.641397248408122e-06,
      "logits/chosen": 0.06422760337591171,
      "logits/rejected": 0.15457448363304138,
      "logps/chosen": -565.0504760742188,
      "logps/rejected": -628.6942138671875,
      "loss": 0.3487,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.16031715273857117,
      "rewards/margins": 0.04973132163286209,
      "rewards/rejected": -0.21004848182201385,
      "step": 160
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5659717401997655e-06,
      "logits/chosen": 0.14616025984287262,
      "logits/rejected": 0.035494547337293625,
      "logps/chosen": -591.8494873046875,
      "logps/rejected": -686.7391357421875,
      "loss": 0.342,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20710352063179016,
      "rewards/margins": 0.07382966578006744,
      "rewards/rejected": -0.2809332013130188,
      "step": 170
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4840920917726425e-06,
      "logits/chosen": 0.02610364928841591,
      "logits/rejected": 0.07332785427570343,
      "logps/chosen": -585.5015869140625,
      "logps/rejected": -654.242919921875,
      "loss": 0.3385,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.19318720698356628,
      "rewards/margins": 0.08042450994253159,
      "rewards/rejected": -0.27361172437667847,
      "step": 180
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.396014096912182e-06,
      "logits/chosen": -0.055401187390089035,
      "logits/rejected": 0.06281381845474243,
      "logps/chosen": -607.0491333007812,
      "logps/rejected": -655.4883422851562,
      "loss": 0.3587,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.179766446352005,
      "rewards/margins": 0.04076399281620979,
      "rewards/rejected": -0.2205304205417633,
      "step": 190
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.302012913171584e-06,
      "logits/chosen": 0.041515715420246124,
      "logits/rejected": 0.10322318226099014,
      "logps/chosen": -518.5902709960938,
      "logps/rejected": -555.7604370117188,
      "loss": 0.3493,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.14564794301986694,
      "rewards/margins": 0.04818274453282356,
      "rewards/rejected": -0.1938306838274002,
      "step": 200
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.202382202273702e-06,
      "logits/chosen": 0.060168325901031494,
      "logits/rejected": 0.05509088560938835,
      "logps/chosen": -546.6846923828125,
      "logps/rejected": -593.6578369140625,
      "loss": 0.3723,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14728133380413055,
      "rewards/margins": 0.04971260204911232,
      "rewards/rejected": -0.19699391722679138,
      "step": 210
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.097433212705492e-06,
      "logits/chosen": 0.07657967507839203,
      "logits/rejected": 0.014337155036628246,
      "logps/chosen": -568.3507080078125,
      "logps/rejected": -619.3447875976562,
      "loss": 0.3164,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.16198821365833282,
      "rewards/margins": 0.06240609288215637,
      "rewards/rejected": -0.2243943214416504,
      "step": 220
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.987493807371033e-06,
      "logits/chosen": -0.006547008641064167,
      "logits/rejected": 0.13627514243125916,
      "logps/chosen": -569.6785278320312,
      "logps/rejected": -639.2614135742188,
      "loss": 0.3225,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.17529377341270447,
      "rewards/margins": 0.0657162293791771,
      "rewards/rejected": -0.24100999534130096,
      "step": 230
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.872907439340758e-06,
      "logits/chosen": -0.017886776477098465,
      "logits/rejected": 0.0037322058342397213,
      "logps/chosen": -559.8616943359375,
      "logps/rejected": -608.9739990234375,
      "loss": 0.332,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.18361715972423553,
      "rewards/margins": 0.05353798344731331,
      "rewards/rejected": -0.23715515434741974,
      "step": 240
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.75403207889666e-06,
      "logits/chosen": -0.013735148124396801,
      "logits/rejected": 0.019972026348114014,
      "logps/chosen": -543.68017578125,
      "logps/rejected": -580.9366455078125,
      "loss": 0.3555,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.17838358879089355,
      "rewards/margins": 0.04401790350675583,
      "rewards/rejected": -0.2224014699459076,
      "step": 250
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.631239095225417e-06,
      "logits/chosen": -0.046024613082408905,
      "logits/rejected": 0.13009200990200043,
      "logps/chosen": -629.1693115234375,
      "logps/rejected": -624.858642578125,
      "loss": 0.3556,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -0.19326834380626678,
      "rewards/margins": 0.042895831167697906,
      "rewards/rejected": -0.2361641824245453,
      "step": 260
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5049120962530608e-06,
      "logits/chosen": 0.04170585796236992,
      "logits/rejected": 0.03916405886411667,
      "logps/chosen": -612.9544067382812,
      "logps/rejected": -684.2191162109375,
      "loss": 0.3172,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.18931707739830017,
      "rewards/margins": 0.06901886314153671,
      "rewards/rejected": -0.2583359181880951,
      "step": 270
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3754457302455464e-06,
      "logits/chosen": -0.016118571162223816,
      "logits/rejected": 0.050165075808763504,
      "logps/chosen": -590.7628173828125,
      "logps/rejected": -657.3172607421875,
      "loss": 0.3254,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.1856985241174698,
      "rewards/margins": 0.061163417994976044,
      "rewards/rejected": -0.24686193466186523,
      "step": 280
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2432444529190714e-06,
      "logits/chosen": -0.013759548775851727,
      "logits/rejected": 0.10973884910345078,
      "logps/chosen": -643.6578979492188,
      "logps/rejected": -724.0432739257812,
      "loss": 0.3321,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21854694187641144,
      "rewards/margins": 0.08554595708847046,
      "rewards/rejected": -0.3040928840637207,
      "step": 290
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1087212639117057e-06,
      "logits/chosen": 0.05435953661799431,
      "logits/rejected": 0.04930294305086136,
      "logps/chosen": -610.7354125976562,
      "logps/rejected": -708.9710693359375,
      "loss": 0.3449,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.22193463146686554,
      "rewards/margins": 0.09556058794260025,
      "rewards/rejected": -0.317495197057724,
      "step": 300
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9722964165636263e-06,
      "logits/chosen": 0.016163092106580734,
      "logits/rejected": -0.008114620111882687,
      "logps/chosen": -599.1492309570312,
      "logps/rejected": -699.5606689453125,
      "loss": 0.3026,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.21535012125968933,
      "rewards/margins": 0.09686703979969025,
      "rewards/rejected": -0.31221720576286316,
      "step": 310
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8343961050366275e-06,
      "logits/chosen": -0.02114369347691536,
      "logits/rejected": 0.00704212486743927,
      "logps/chosen": -597.2741088867188,
      "logps/rejected": -663.7066650390625,
      "loss": 0.2955,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.20012113451957703,
      "rewards/margins": 0.08274127542972565,
      "rewards/rejected": -0.28286242485046387,
      "step": 320
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.695451132874385e-06,
      "logits/chosen": -0.005913554225116968,
      "logits/rejected": -0.03215361759066582,
      "logps/chosen": -634.7625122070312,
      "logps/rejected": -682.1895751953125,
      "loss": 0.349,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.2384355515241623,
      "rewards/margins": 0.057650744915008545,
      "rewards/rejected": -0.29608631134033203,
      "step": 330
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5558955671628964e-06,
      "logits/chosen": 0.014790521934628487,
      "logits/rejected": 0.03520793467760086,
      "logps/chosen": -708.5465087890625,
      "logps/rejected": -726.1888427734375,
      "loss": 0.3108,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.24060730636119843,
      "rewards/margins": 0.06836961209774017,
      "rewards/rejected": -0.3089769184589386,
      "step": 340
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4161653824955654e-06,
      "logits/chosen": -0.039772890508174896,
      "logits/rejected": 0.04145686700940132,
      "logps/chosen": -643.654296875,
      "logps/rejected": -722.4707641601562,
      "loss": 0.2987,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.2103274166584015,
      "rewards/margins": 0.08200379461050034,
      "rewards/rejected": -0.29233118891716003,
      "step": 350
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2766970989791697e-06,
      "logits/chosen": -0.07129567116498947,
      "logits/rejected": 0.038871683180332184,
      "logps/chosen": -605.6844482421875,
      "logps/rejected": -640.6483764648438,
      "loss": 0.3339,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.21578896045684814,
      "rewards/margins": 0.06505810469388962,
      "rewards/rejected": -0.28084704279899597,
      "step": 360
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1379264185356545e-06,
      "logits/chosen": -0.062216080725193024,
      "logits/rejected": -0.024451395496726036,
      "logps/chosen": -646.650634765625,
      "logps/rejected": -700.7503051757812,
      "loss": 0.3549,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.24396809935569763,
      "rewards/margins": 0.07057716697454453,
      "rewards/rejected": -0.31454527378082275,
      "step": 370
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.000286863759934e-06,
      "logits/chosen": -0.08249249309301376,
      "logits/rejected": 0.03427312523126602,
      "logps/chosen": -680.2882690429688,
      "logps/rejected": -714.1500244140625,
      "loss": 0.3508,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.2394016683101654,
      "rewards/margins": 0.05815623328089714,
      "rewards/rejected": -0.29755789041519165,
      "step": 380
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8642084235859764e-06,
      "logits/chosen": -0.034602344036102295,
      "logits/rejected": 0.007808700203895569,
      "logps/chosen": -624.1826171875,
      "logps/rejected": -688.9442138671875,
      "loss": 0.3546,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.23162564635276794,
      "rewards/margins": 0.046284474432468414,
      "rewards/rejected": -0.27791014313697815,
      "step": 390
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7301162099921013e-06,
      "logits/chosen": -0.10363703966140747,
      "logits/rejected": 0.05012492090463638,
      "logps/chosen": -563.6665649414062,
      "logps/rejected": -604.61572265625,
      "loss": 0.3045,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.19194665551185608,
      "rewards/margins": 0.07340183109045029,
      "rewards/rejected": -0.26534849405288696,
      "step": 400
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5984291299420117e-06,
      "logits/chosen": 0.026764582842588425,
      "logits/rejected": 0.04089599847793579,
      "logps/chosen": -624.69970703125,
      "logps/rejected": -700.4164428710938,
      "loss": 0.3382,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.21267059445381165,
      "rewards/margins": 0.0644538626074791,
      "rewards/rejected": -0.27712446451187134,
      "step": 410
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4695585767104092e-06,
      "logits/chosen": -0.08264167606830597,
      "logits/rejected": 0.056310832500457764,
      "logps/chosen": -592.313720703125,
      "logps/rejected": -640.4153442382812,
      "loss": 0.3128,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20638258755207062,
      "rewards/margins": 0.06300722062587738,
      "rewards/rejected": -0.269389808177948,
      "step": 420
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3439071446815452e-06,
      "logits/chosen": 0.007897214964032173,
      "logits/rejected": -0.022024786099791527,
      "logps/chosen": -576.2522583007812,
      "logps/rejected": -674.4778442382812,
      "loss": 0.3061,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.19270029664039612,
      "rewards/margins": 0.08912625163793564,
      "rewards/rejected": -0.28182655572891235,
      "step": 430
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2218673716356919e-06,
      "logits/chosen": -0.07224872708320618,
      "logits/rejected": -0.04243229702115059,
      "logps/chosen": -667.8207397460938,
      "logps/rejected": -694.7523193359375,
      "loss": 0.3343,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.21047215163707733,
      "rewards/margins": 0.05125568434596062,
      "rewards/rejected": -0.26172783970832825,
      "step": 440
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.103820512452661e-06,
      "logits/chosen": -0.004954156465828419,
      "logits/rejected": -0.06225745752453804,
      "logps/chosen": -611.6270141601562,
      "logps/rejected": -663.220703125,
      "loss": 0.3585,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.22025656700134277,
      "rewards/margins": 0.04412217065691948,
      "rewards/rejected": -0.26437875628471375,
      "step": 450
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.901353480633468e-07,
      "logits/chosen": -0.05928473919630051,
      "logits/rejected": -0.07472564280033112,
      "logps/chosen": -678.4139404296875,
      "logps/rejected": -729.8592529296875,
      "loss": 0.3345,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.23494932055473328,
      "rewards/margins": 0.06922940909862518,
      "rewards/rejected": -0.30417874455451965,
      "step": 460
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.811670333701544e-07,
      "logits/chosen": 0.0089465556666255,
      "logits/rejected": -0.0320318266749382,
      "logps/chosen": -632.8341064453125,
      "logps/rejected": -666.9921875,
      "loss": 0.3629,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.23375169932842255,
      "rewards/margins": 0.047677937895059586,
      "rewards/rejected": -0.28142961859703064,
      "step": 470
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.772559877354341e-07,
      "logits/chosen": 0.01971885748207569,
      "logits/rejected": 0.05568293482065201,
      "logps/chosen": -603.0645751953125,
      "logps/rejected": -670.7252807617188,
      "loss": 0.3554,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.20837394893169403,
      "rewards/margins": 0.04435722157359123,
      "rewards/rejected": -0.25273117423057556,
      "step": 480
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.787268315040604e-07,
      "logits/chosen": 0.04185909777879715,
      "logits/rejected": -0.008848948404192924,
      "logps/chosen": -616.8929443359375,
      "logps/rejected": -697.3258666992188,
      "loss": 0.3378,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.19609177112579346,
      "rewards/margins": 0.06852120906114578,
      "rewards/rejected": -0.26461294293403625,
      "step": 490
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.858873718824829e-07,
      "logits/chosen": -0.1592080146074295,
      "logits/rejected": -0.008233757689595222,
      "logps/chosen": -566.5294799804688,
      "logps/rejected": -622.8501586914062,
      "loss": 0.3033,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.18146850168704987,
      "rewards/margins": 0.07721823453903198,
      "rewards/rejected": -0.25868672132492065,
      "step": 500
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.990276413423817e-07,
      "logits/chosen": -0.11390354484319687,
      "logits/rejected": -0.022463113069534302,
      "logps/chosen": -630.0293579101562,
      "logps/rejected": -702.0118408203125,
      "loss": 0.3212,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.20778632164001465,
      "rewards/margins": 0.07734315097332001,
      "rewards/rejected": -0.28512945771217346,
      "step": 510
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.184189915529796e-07,
      "logits/chosen": -0.11935200542211533,
      "logits/rejected": -0.05328110605478287,
      "logps/chosen": -697.4544677734375,
      "logps/rejected": -787.1935424804688,
      "loss": 0.316,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.22982342541217804,
      "rewards/margins": 0.09117700159549713,
      "rewards/rejected": -0.3210003972053528,
      "step": 520
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4431324567258176e-07,
      "logits/chosen": -0.04139793664216995,
      "logits/rejected": 0.02297029457986355,
      "logps/chosen": -595.4196166992188,
      "logps/rejected": -648.87353515625,
      "loss": 0.3245,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.20907723903656006,
      "rewards/margins": 0.06023558974266052,
      "rewards/rejected": -0.2693127989768982,
      "step": 530
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.769419116476052e-07,
      "logits/chosen": -0.012856354005634785,
      "logits/rejected": 0.03404129296541214,
      "logps/chosen": -591.6539306640625,
      "logps/rejected": -639.3442993164062,
      "loss": 0.3519,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.21816472709178925,
      "rewards/margins": 0.05062199383974075,
      "rewards/rejected": -0.2687866985797882,
      "step": 540
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1651545897676512e-07,
      "logits/chosen": -0.1339225172996521,
      "logits/rejected": -6.989687972236425e-05,
      "logps/chosen": -603.4436645507812,
      "logps/rejected": -656.8947143554688,
      "loss": 0.3171,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.20348867774009705,
      "rewards/margins": 0.06371064484119415,
      "rewards/rejected": -0.2671993374824524,
      "step": 550
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6322266119983222e-07,
      "logits/chosen": -0.010643618181347847,
      "logits/rejected": -0.10318191349506378,
      "logps/chosen": -538.0825805664062,
      "logps/rejected": -612.8212280273438,
      "loss": 0.3428,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.18894371390342712,
      "rewards/margins": 0.05722329765558243,
      "rewards/rejected": -0.24616701900959015,
      "step": 560
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1723000616502167e-07,
      "logits/chosen": -0.05230097100138664,
      "logits/rejected": 0.0051549882628023624,
      "logps/chosen": -597.4989013671875,
      "logps/rejected": -646.1577758789062,
      "loss": 0.3383,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.20286564528942108,
      "rewards/margins": 0.06327156722545624,
      "rewards/rejected": -0.2661372423171997,
      "step": 570
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.868117591737585e-08,
      "logits/chosen": -0.11382152885198593,
      "logits/rejected": 0.017693499103188515,
      "logps/chosen": -600.3404541015625,
      "logps/rejected": -700.9383544921875,
      "loss": 0.3332,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.20399217307567596,
      "rewards/margins": 0.08086130768060684,
      "rewards/rejected": -0.284853458404541,
      "step": 580
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.769659783295383e-08,
      "logits/chosen": -0.041316356509923935,
      "logits/rejected": -0.09137067198753357,
      "logps/chosen": -627.2562255859375,
      "logps/rejected": -690.1580200195312,
      "loss": 0.3578,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.22119614481925964,
      "rewards/margins": 0.04840771108865738,
      "rewards/rejected": -0.2696038782596588,
      "step": 590
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4373068401120358e-08,
      "logits/chosen": -0.048679061233997345,
      "logits/rejected": 0.08877386897802353,
      "logps/chosen": -593.13134765625,
      "logps/rejected": -638.4296264648438,
      "loss": 0.36,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.21022383868694305,
      "rewards/margins": 0.04604775831103325,
      "rewards/rejected": -0.2562715709209442,
      "step": 600
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.78345083022425e-09,
      "logits/chosen": 0.013545943424105644,
      "logits/rejected": 0.013492127880454063,
      "logps/chosen": -616.1806640625,
      "logps/rejected": -662.4678955078125,
      "loss": 0.3495,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.217128723859787,
      "rewards/margins": 0.04821699112653732,
      "rewards/rejected": -0.2653457224369049,
      "step": 610
    },
    {
      "epoch": 0.99,
      "learning_rate": 9.764474213677654e-10,
      "logits/chosen": -0.05919772386550903,
      "logits/rejected": 0.03523395583033562,
      "logps/chosen": -687.3334350585938,
      "logps/rejected": -745.0026245117188,
      "loss": 0.3215,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.2285897433757782,
      "rewards/margins": 0.0788644552230835,
      "rewards/rejected": -0.3074541687965393,
      "step": 620
    },
    {
      "epoch": 1.0,
      "step": 625,
      "total_flos": 0.0,
      "train_loss": 0.34297545976638794,
      "train_runtime": 3774.4903,
      "train_samples_per_second": 2.649,
      "train_steps_per_second": 0.166
    }
  ],
  "logging_steps": 10,
  "max_steps": 625,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}