{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9982003599280144,
  "eval_steps": 500,
  "global_step": 416,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.1904761904761906e-07,
      "logits/chosen": -0.10997498780488968,
      "logits/rejected": -0.14968934655189514,
      "logps/chosen": -470.78253173828125,
      "logps/rejected": -601.5414428710938,
      "loss": 0.21,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.1904761904761906e-06,
      "logits/chosen": -0.1849229484796524,
      "logits/rejected": -0.13570153713226318,
      "logps/chosen": -495.9288330078125,
      "logps/rejected": -719.5632934570312,
      "loss": 0.2073,
      "rewards/accuracies": 0.4027777910232544,
      "rewards/chosen": -1.4412669315788662e-06,
      "rewards/margins": -4.9562087951926515e-05,
      "rewards/rejected": 4.812082988792099e-05,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.380952380952381e-06,
      "logits/chosen": -0.21864363551139832,
      "logits/rejected": -0.1575668752193451,
      "logps/chosen": -500.5243225097656,
      "logps/rejected": -724.0389404296875,
      "loss": 0.2106,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0024510668590664864,
      "rewards/margins": 0.002445765770971775,
      "rewards/rejected": -0.004896833095699549,
      "step": 20
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.5714285714285718e-06,
      "logits/chosen": -0.11167863756418228,
      "logits/rejected": -0.1275203675031662,
      "logps/chosen": -545.2283935546875,
      "logps/rejected": -782.0553588867188,
      "loss": 0.2009,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.013457772321999073,
      "rewards/margins": 0.01753374934196472,
      "rewards/rejected": -0.03099152073264122,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.761904761904762e-06,
      "logits/chosen": -0.20949259400367737,
      "logits/rejected": -0.17874333262443542,
      "logps/chosen": -546.1171875,
      "logps/rejected": -841.0467529296875,
      "loss": 0.1651,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.05306588485836983,
      "rewards/margins": 0.06392969191074371,
      "rewards/rejected": -0.11699558794498444,
      "step": 40
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.994357350311441e-06,
      "logits/chosen": -0.19548015296459198,
      "logits/rejected": -0.24876411259174347,
      "logps/chosen": -626.0526123046875,
      "logps/rejected": -1041.28076171875,
      "loss": 0.1523,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.1381954401731491,
      "rewards/margins": 0.14297422766685486,
      "rewards/rejected": -0.28116968274116516,
      "step": 50
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.97147773390341e-06,
      "logits/chosen": -0.2218095362186432,
      "logits/rejected": -0.2248532772064209,
      "logps/chosen": -627.7503662109375,
      "logps/rejected": -929.7164306640625,
      "loss": 0.1461,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.14269283413887024,
      "rewards/margins": 0.11241813004016876,
      "rewards/rejected": -0.2551110088825226,
      "step": 60
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.931169703639282e-06,
      "logits/chosen": -0.23902878165245056,
      "logits/rejected": -0.19728106260299683,
      "logps/chosen": -686.7578735351562,
      "logps/rejected": -1015.8434448242188,
      "loss": 0.1503,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.1398530751466751,
      "rewards/margins": 0.1030840128660202,
      "rewards/rejected": -0.24293705821037292,
      "step": 70
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.873717504456219e-06,
      "logits/chosen": -0.22387316823005676,
      "logits/rejected": -0.2015857696533203,
      "logps/chosen": -625.0352783203125,
      "logps/rejected": -1033.6270751953125,
      "loss": 0.1483,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.1254601627588272,
      "rewards/margins": 0.12839995324611664,
      "rewards/rejected": -0.25386008620262146,
      "step": 80
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7995262788689865e-06,
      "logits/chosen": -0.27522459626197815,
      "logits/rejected": -0.21160630881786346,
      "logps/chosen": -697.7965698242188,
      "logps/rejected": -1014.5755004882812,
      "loss": 0.1439,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.1593645215034485,
      "rewards/margins": 0.13819821178913116,
      "rewards/rejected": -0.29756274819374084,
      "step": 90
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.709119209978242e-06,
      "logits/chosen": -0.25298193097114563,
      "logits/rejected": -0.2807552218437195,
      "logps/chosen": -657.3956298828125,
      "logps/rejected": -1090.194091796875,
      "loss": 0.1371,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.15086904168128967,
      "rewards/margins": 0.16771209239959717,
      "rewards/rejected": -0.31858113408088684,
      "step": 100
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.603133832077953e-06,
      "logits/chosen": -0.2874082922935486,
      "logits/rejected": -0.18505045771598816,
      "logps/chosen": -685.4857788085938,
      "logps/rejected": -932.3782958984375,
      "loss": 0.1492,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1552571952342987,
      "rewards/margins": 0.113719142973423,
      "rewards/rejected": -0.2689763307571411,
      "step": 110
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.482317534878901e-06,
      "logits/chosen": -0.2046702355146408,
      "logits/rejected": -0.21371881663799286,
      "logps/chosen": -615.016357421875,
      "logps/rejected": -1014.6843872070312,
      "loss": 0.1446,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.13098101317882538,
      "rewards/margins": 0.15675313770771027,
      "rewards/rejected": -0.2877342104911804,
      "step": 120
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3475222930516484e-06,
      "logits/chosen": -0.30239051580429077,
      "logits/rejected": -0.2552723288536072,
      "logps/chosen": -661.1831665039062,
      "logps/rejected": -1063.7322998046875,
      "loss": 0.1373,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.16380271315574646,
      "rewards/margins": 0.14136558771133423,
      "rewards/rejected": -0.3051683008670807,
      "step": 130
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.199698658255298e-06,
      "logits/chosen": -0.23284006118774414,
      "logits/rejected": -0.24393205344676971,
      "logps/chosen": -689.3310546875,
      "logps/rejected": -1185.5355224609375,
      "loss": 0.1333,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.1679478883743286,
      "rewards/margins": 0.18742243945598602,
      "rewards/rejected": -0.35537031292915344,
      "step": 140
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.039889056019159e-06,
      "logits/chosen": -0.20814350247383118,
      "logits/rejected": -0.24712209403514862,
      "logps/chosen": -631.2597045898438,
      "logps/rejected": -1041.0517578125,
      "loss": 0.1331,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.15335983037948608,
      "rewards/margins": 0.17034170031547546,
      "rewards/rejected": -0.32370156049728394,
      "step": 150
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.869220434746509e-06,
      "logits/chosen": -0.2555493712425232,
      "logits/rejected": -0.2478325366973877,
      "logps/chosen": -709.0995483398438,
      "logps/rejected": -1125.80615234375,
      "loss": 0.1273,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.16779521107673645,
      "rewards/margins": 0.157444566488266,
      "rewards/rejected": -0.32523980736732483,
      "step": 160
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.688896318678322e-06,
      "logits/chosen": -0.24241690337657928,
      "logits/rejected": -0.2430168092250824,
      "logps/chosen": -652.2470703125,
      "logps/rejected": -1029.941162109375,
      "loss": 0.1405,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.1708507239818573,
      "rewards/margins": 0.14247409999370575,
      "rewards/rejected": -0.31332483887672424,
      "step": 170
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5001883208580668e-06,
      "logits/chosen": -0.2642208933830261,
      "logits/rejected": -0.2481091320514679,
      "logps/chosen": -667.6624145507812,
      "logps/rejected": -1021.7717895507812,
      "loss": 0.1306,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.18226784467697144,
      "rewards/margins": 0.14150500297546387,
      "rewards/rejected": -0.3237728476524353,
      "step": 180
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.30442717594657e-06,
      "logits/chosen": -0.1869751662015915,
      "logits/rejected": -0.23872609436511993,
      "logps/chosen": -644.8170776367188,
      "logps/rejected": -1029.4417724609375,
      "loss": 0.1211,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1459895819425583,
      "rewards/margins": 0.15111900866031647,
      "rewards/rejected": -0.29710859060287476,
      "step": 190
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.102993356121938e-06,
      "logits/chosen": -0.260442852973938,
      "logits/rejected": -0.2213623970746994,
      "logps/chosen": -672.3732299804688,
      "logps/rejected": -1037.558837890625,
      "loss": 0.1341,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.1695290207862854,
      "rewards/margins": 0.14989891648292542,
      "rewards/rejected": -0.3194279074668884,
      "step": 200
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.8973073362395e-06,
      "logits/chosen": -0.25881263613700867,
      "logits/rejected": -0.20856423676013947,
      "logps/chosen": -658.4823608398438,
      "logps/rejected": -1057.578369140625,
      "loss": 0.1328,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.17251142859458923,
      "rewards/margins": 0.17602583765983582,
      "rewards/rejected": -0.34853726625442505,
      "step": 210
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6888195769001147e-06,
      "logits/chosen": -0.2791309952735901,
      "logits/rejected": -0.27674758434295654,
      "logps/chosen": -642.2847900390625,
      "logps/rejected": -1090.7037353515625,
      "loss": 0.1269,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.18547888100147247,
      "rewards/margins": 0.1763353794813156,
      "rewards/rejected": -0.3618142306804657,
      "step": 220
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.479000296064417e-06,
      "logits/chosen": -0.24742920696735382,
      "logits/rejected": -0.2712458372116089,
      "logps/chosen": -732.4454956054688,
      "logps/rejected": -1207.4635009765625,
      "loss": 0.1238,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.18814927339553833,
      "rewards/margins": 0.19270290434360504,
      "rewards/rejected": -0.38085219264030457,
      "step": 230
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.269329101341745e-06,
      "logits/chosen": -0.30566203594207764,
      "logits/rejected": -0.2690572142601013,
      "logps/chosen": -696.6936645507812,
      "logps/rejected": -1079.8812255859375,
      "loss": 0.1214,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.196825310587883,
      "rewards/margins": 0.17798085510730743,
      "rewards/rejected": -0.37480616569519043,
      "step": 240
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.06128455606496e-06,
      "logits/chosen": -0.23360852897167206,
      "logits/rejected": -0.20390859246253967,
      "logps/chosen": -674.8441162109375,
      "logps/rejected": -1029.4664306640625,
      "loss": 0.1324,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1800800859928131,
      "rewards/margins": 0.17153646051883698,
      "rewards/rejected": -0.3516165018081665,
      "step": 250
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.856333752729311e-06,
      "logits/chosen": -0.23972924053668976,
      "logits/rejected": -0.2775985598564148,
      "logps/chosen": -682.693115234375,
      "logps/rejected": -1193.5947265625,
      "loss": 0.1254,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.19959069788455963,
      "rewards/margins": 0.19032658636569977,
      "rewards/rejected": -0.3899172842502594,
      "step": 260
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6559219673215784e-06,
      "logits/chosen": -0.30556970834732056,
      "logits/rejected": -0.3009134829044342,
      "logps/chosen": -681.2161865234375,
      "logps/rejected": -1142.70361328125,
      "loss": 0.1318,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.20223207771778107,
      "rewards/margins": 0.17814533412456512,
      "rewards/rejected": -0.3803774416446686,
      "step": 270
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4614624674952843e-06,
      "logits/chosen": -0.26370662450790405,
      "logits/rejected": -0.23301748931407928,
      "logps/chosen": -725.9805908203125,
      "logps/rejected": -1188.698974609375,
      "loss": 0.1291,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.20386967062950134,
      "rewards/margins": 0.1635088473558426,
      "rewards/rejected": -0.3673785328865051,
      "step": 280
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2743265464628787e-06,
      "logits/chosen": -0.26803213357925415,
      "logits/rejected": -0.23297333717346191,
      "logps/chosen": -660.8951416015625,
      "logps/rejected": -1116.446533203125,
      "loss": 0.1294,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.1704072505235672,
      "rewards/margins": 0.18539056181907654,
      "rewards/rejected": -0.35579782724380493,
      "step": 290
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0958338528840893e-06,
      "logits/chosen": -0.23666846752166748,
      "logits/rejected": -0.264843612909317,
      "logps/chosen": -715.3922119140625,
      "logps/rejected": -1168.862548828125,
      "loss": 0.1267,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.18868893384933472,
      "rewards/margins": 0.19361743330955505,
      "rewards/rejected": -0.38230639696121216,
      "step": 300
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.272430849423175e-07,
      "logits/chosen": -0.2275594025850296,
      "logits/rejected": -0.20153971016407013,
      "logps/chosen": -691.9971313476562,
      "logps/rejected": -1152.3519287109375,
      "loss": 0.1327,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.1932949721813202,
      "rewards/margins": 0.19385525584220886,
      "rewards/rejected": -0.38715022802352905,
      "step": 310
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.697431142327633e-07,
      "logits/chosen": -0.2559944987297058,
      "logits/rejected": -0.2104511708021164,
      "logps/chosen": -690.9197998046875,
      "logps/rejected": -1113.470703125,
      "loss": 0.1352,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.19007542729377747,
      "rewards/margins": 0.16325220465660095,
      "rewards/rejected": -0.3533276319503784,
      "step": 320
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.244446020550182e-07,
      "logits/chosen": -0.21340703964233398,
      "logits/rejected": -0.23157565295696259,
      "logps/chosen": -706.4766235351562,
      "logps/rejected": -1090.2486572265625,
      "loss": 0.1214,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.19076435267925262,
      "rewards/margins": 0.16976633667945862,
      "rewards/rejected": -0.36053070425987244,
      "step": 330
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.923721672305148e-07,
      "logits/chosen": -0.2297631800174713,
      "logits/rejected": -0.18899887800216675,
      "logps/chosen": -723.7625732421875,
      "logps/rejected": -1124.594482421875,
      "loss": 0.1285,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.19292166829109192,
      "rewards/margins": 0.18659308552742004,
      "rewards/rejected": -0.37951475381851196,
      "step": 340
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.7445716067596506e-07,
      "logits/chosen": -0.22677800059318542,
      "logits/rejected": -0.22308814525604248,
      "logps/chosen": -663.9019775390625,
      "logps/rejected": -1152.65283203125,
      "loss": 0.1117,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1815580129623413,
      "rewards/margins": 0.20128455758094788,
      "rewards/rejected": -0.3828425407409668,
      "step": 350
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.7153109768518926e-07,
      "logits/chosen": -0.249908447265625,
      "logits/rejected": -0.1893308460712433,
      "logps/chosen": -674.8004150390625,
      "logps/rejected": -1103.431396484375,
      "loss": 0.1331,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.18865013122558594,
      "rewards/margins": 0.16974930465221405,
      "rewards/rejected": -0.3583994507789612,
      "step": 360
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8431979423369607e-07,
      "logits/chosen": -0.20425882935523987,
      "logits/rejected": -0.2509860098361969,
      "logps/chosen": -787.9830322265625,
      "logps/rejected": -1210.3958740234375,
      "loss": 0.1263,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.20410247147083282,
      "rewards/margins": 0.1842099130153656,
      "rewards/rejected": -0.3883123993873596,
      "step": 370
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1343824865573422e-07,
      "logits/chosen": -0.23611795902252197,
      "logits/rejected": -0.2374495565891266,
      "logps/chosen": -683.4967041015625,
      "logps/rejected": -1033.9107666015625,
      "loss": 0.1211,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.18874987959861755,
      "rewards/margins": 0.15766242146492004,
      "rewards/rejected": -0.34641233086586,
      "step": 380
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.9386304787299175e-08,
      "logits/chosen": -0.2402607947587967,
      "logits/rejected": -0.2553133964538574,
      "logps/chosen": -653.82177734375,
      "logps/rejected": -1058.991943359375,
      "loss": 0.1348,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.18406951427459717,
      "rewards/margins": 0.15722426772117615,
      "rewards/rejected": -0.3412937819957733,
      "step": 390
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.2545127157831416e-08,
      "logits/chosen": -0.2702855169773102,
      "logits/rejected": -0.2571588158607483,
      "logps/chosen": -723.2869262695312,
      "logps/rejected": -1087.939208984375,
      "loss": 0.1216,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.19856762886047363,
      "rewards/margins": 0.1647205799818039,
      "rewards/rejected": -0.36328819394111633,
      "step": 400
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.1745130869123564e-09,
      "logits/chosen": -0.2467096745967865,
      "logits/rejected": -0.2630787193775177,
      "logps/chosen": -715.5072021484375,
      "logps/rejected": -1024.0079345703125,
      "loss": 0.1273,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20651917159557343,
      "rewards/margins": 0.14182263612747192,
      "rewards/rejected": -0.34834182262420654,
      "step": 410
    },
    {
      "epoch": 1.0,
      "step": 416,
      "total_flos": 0.0,
      "train_loss": 0.13873135742659753,
      "train_runtime": 5562.1192,
      "train_samples_per_second": 3.596,
      "train_steps_per_second": 0.075
    }
  ],
  "logging_steps": 10,
  "max_steps": 416,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}