diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,3060 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9995965030262273,
+  "eval_steps": 500,
+  "global_step": 1858,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.6881720430107528e-09,
+      "logits/chosen": -2.5808520317077637,
+      "logits/rejected": -2.0101242065429688,
+      "logps/chosen": -299.3489990234375,
+      "logps/rejected": -186.63014221191406,
+      "loss": 1.2656,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "rewards/safe_rewards": 0.0,
+      "rewards/unsafe_rewards": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.6881720430107527e-08,
+      "logits/chosen": -2.38761043548584,
+      "logits/rejected": -2.2287850379943848,
+      "logps/chosen": -201.83148193359375,
+      "logps/rejected": -189.46726989746094,
+      "loss": 1.4296,
+      "rewards/accuracies": 0.4305555522441864,
+      "rewards/chosen": 2.8226104404893704e-05,
+      "rewards/margins": -9.960395254893228e-05,
+      "rewards/rejected": 0.00012783010606653988,
+      "rewards/safe_rewards": -0.0001673989463597536,
+      "rewards/unsafe_rewards": 0.0002238511951873079,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.3763440860215054e-08,
+      "logits/chosen": -2.3484911918640137,
+      "logits/rejected": -2.053339719772339,
+      "logps/chosen": -226.3044891357422,
+      "logps/rejected": -181.17330932617188,
+      "loss": 1.463,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -4.27155100624077e-05,
+      "rewards/margins": 3.7895108562224777e-06,
+      "rewards/rejected": -4.650496339309029e-05,
+      "rewards/safe_rewards": -0.0004773393739014864,
+      "rewards/unsafe_rewards": 0.0003919084556400776,
+      "step": 20
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 8.064516129032257e-08,
+      "logits/chosen": -2.3405332565307617,
+      "logits/rejected": -2.145922899246216,
+      "logps/chosen": -215.05410766601562,
+      "logps/rejected": -189.3188018798828,
+      "loss": 1.431,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 3.503418338368647e-05,
+      "rewards/margins": 0.0005787784466519952,
+      "rewards/rejected": -0.0005437443032860756,
+      "rewards/safe_rewards": -0.00011984705633949488,
+      "rewards/unsafe_rewards": 0.00018991540127899498,
+      "step": 30
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0752688172043011e-07,
+      "logits/chosen": -2.2765462398529053,
+      "logits/rejected": -1.974180817604065,
+      "logps/chosen": -180.71937561035156,
+      "logps/rejected": -173.9296417236328,
+      "loss": 1.4304,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.00015593590796925128,
+      "rewards/margins": 0.0017944574356079102,
+      "rewards/rejected": -0.0016385214403271675,
+      "rewards/safe_rewards": 0.00035788281820714474,
+      "rewards/unsafe_rewards": -4.6010944060981274e-05,
+      "step": 40
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3440860215053762e-07,
+      "logits/chosen": -2.403860569000244,
+      "logits/rejected": -2.0332884788513184,
+      "logps/chosen": -209.592529296875,
+      "logps/rejected": -167.6835174560547,
+      "loss": 1.4344,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.0008560878923162818,
+      "rewards/margins": 0.004450940527021885,
+      "rewards/rejected": -0.0035948525182902813,
+      "rewards/safe_rewards": 0.00018845750309992582,
+      "rewards/unsafe_rewards": 0.0015237184707075357,
+      "step": 50
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6129032258064515e-07,
+      "logits/chosen": -2.330204486846924,
+      "logits/rejected": -2.1555492877960205,
+      "logps/chosen": -185.8196563720703,
+      "logps/rejected": -185.08883666992188,
+      "loss": 1.4264,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.0013676225207746029,
+      "rewards/margins": 0.0021811036858707666,
+      "rewards/rejected": -0.003548726439476013,
+      "rewards/safe_rewards": -0.000750910839997232,
+      "rewards/unsafe_rewards": -0.001984334085136652,
+      "step": 60
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8817204301075268e-07,
+      "logits/chosen": -2.3298559188842773,
+      "logits/rejected": -2.0787758827209473,
+      "logps/chosen": -202.39566040039062,
+      "logps/rejected": -184.2627410888672,
+      "loss": 1.3858,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0020925761200487614,
+      "rewards/margins": 0.006319403648376465,
+      "rewards/rejected": -0.008411980234086514,
+      "rewards/safe_rewards": -0.0030772520694881678,
+      "rewards/unsafe_rewards": -0.0011078999377787113,
+      "step": 70
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.1505376344086022e-07,
+      "logits/chosen": -2.3292670249938965,
+      "logits/rejected": -2.1124508380889893,
+      "logps/chosen": -220.2982940673828,
+      "logps/rejected": -195.1908721923828,
+      "loss": 1.3919,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.008099144324660301,
+      "rewards/margins": 0.012995732948184013,
+      "rewards/rejected": -0.021094877272844315,
+      "rewards/safe_rewards": -0.010834941640496254,
+      "rewards/unsafe_rewards": -0.005363349802792072,
+      "step": 80
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.4193548387096775e-07,
+      "logits/chosen": -2.3152518272399902,
+      "logits/rejected": -2.1177124977111816,
+      "logps/chosen": -209.4881134033203,
+      "logps/rejected": -170.4688720703125,
+      "loss": 1.4229,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.03572874516248703,
+      "rewards/margins": 0.025636380538344383,
+      "rewards/rejected": -0.061365120112895966,
+      "rewards/safe_rewards": -0.03464391082525253,
+      "rewards/unsafe_rewards": -0.03681357204914093,
+      "step": 90
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.6881720430107523e-07,
+      "logits/chosen": -2.312987804412842,
+      "logits/rejected": -2.1187119483947754,
+      "logps/chosen": -204.9591827392578,
+      "logps/rejected": -180.0883331298828,
+      "loss": 1.4099,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0888073593378067,
+      "rewards/margins": 0.04026350378990173,
+      "rewards/rejected": -0.12907087802886963,
+      "rewards/safe_rewards": -0.08975542336702347,
+      "rewards/unsafe_rewards": -0.08785931766033173,
+      "step": 100
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.956989247311828e-07,
+      "logits/chosen": -2.2454307079315186,
+      "logits/rejected": -2.0064516067504883,
+      "logps/chosen": -224.6505889892578,
+      "logps/rejected": -190.16635131835938,
+      "loss": 1.4442,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.12704427540302277,
+      "rewards/margins": 0.054799921810626984,
+      "rewards/rejected": -0.18184418976306915,
+      "rewards/safe_rewards": -0.12200836837291718,
+      "rewards/unsafe_rewards": -0.13208015263080597,
+      "step": 110
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.225806451612903e-07,
+      "logits/chosen": -2.26438570022583,
+      "logits/rejected": -1.9198744297027588,
+      "logps/chosen": -222.88461303710938,
+      "logps/rejected": -173.9338836669922,
+      "loss": 1.3613,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.1163511872291565,
+      "rewards/margins": 0.09466449916362762,
+      "rewards/rejected": -0.21101567149162292,
+      "rewards/safe_rewards": -0.11146645247936249,
+      "rewards/unsafe_rewards": -0.1212359219789505,
+      "step": 120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.4946236559139783e-07,
+      "logits/chosen": -2.19810152053833,
+      "logits/rejected": -1.9682261943817139,
+      "logps/chosen": -241.05618286132812,
+      "logps/rejected": -201.30264282226562,
+      "loss": 1.2805,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.26873043179512024,
+      "rewards/margins": 0.12516936659812927,
+      "rewards/rejected": -0.39389973878860474,
+      "rewards/safe_rewards": -0.21829214692115784,
+      "rewards/unsafe_rewards": -0.3191686272621155,
+      "step": 130
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.7634408602150537e-07,
+      "logits/chosen": -2.192406177520752,
+      "logits/rejected": -1.845873236656189,
+      "logps/chosen": -241.24447631835938,
+      "logps/rejected": -219.4851531982422,
+      "loss": 1.2394,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.2985479533672333,
+      "rewards/margins": 0.09591639041900635,
+      "rewards/rejected": -0.394464373588562,
+      "rewards/safe_rewards": -0.2801482379436493,
+      "rewards/unsafe_rewards": -0.31694772839546204,
+      "step": 140
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.0322580645161285e-07,
+      "logits/chosen": -2.0371975898742676,
+      "logits/rejected": -1.7497107982635498,
+      "logps/chosen": -249.6632537841797,
+      "logps/rejected": -225.79537963867188,
+      "loss": 1.1556,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4271857738494873,
+      "rewards/margins": 0.13365456461906433,
+      "rewards/rejected": -0.5608403086662292,
+      "rewards/safe_rewards": -0.42928582429885864,
+      "rewards/unsafe_rewards": -0.4250856935977936,
+      "step": 150
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.3010752688172043e-07,
+      "logits/chosen": -1.9401572942733765,
+      "logits/rejected": -1.5656859874725342,
+      "logps/chosen": -246.42398071289062,
+      "logps/rejected": -226.51400756835938,
+      "loss": 1.2948,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2622266113758087,
+      "rewards/margins": 0.10594137012958527,
+      "rewards/rejected": -0.3681679964065552,
+      "rewards/safe_rewards": -0.24256543815135956,
+      "rewards/unsafe_rewards": -0.28188782930374146,
+      "step": 160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.569892473118279e-07,
+      "logits/chosen": -1.9621531963348389,
+      "logits/rejected": -1.6663320064544678,
+      "logps/chosen": -253.3407440185547,
+      "logps/rejected": -226.83035278320312,
+      "loss": 1.1636,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.38964638113975525,
+      "rewards/margins": 0.20659947395324707,
+      "rewards/rejected": -0.5962458848953247,
+      "rewards/safe_rewards": -0.3696528375148773,
+      "rewards/unsafe_rewards": -0.4096398949623108,
+      "step": 170
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.838709677419355e-07,
+      "logits/chosen": -2.0335605144500732,
+      "logits/rejected": -1.65840744972229,
+      "logps/chosen": -232.89804077148438,
+      "logps/rejected": -238.404296875,
+      "loss": 1.153,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.40930503606796265,
+      "rewards/margins": 0.26214295625686646,
+      "rewards/rejected": -0.6714479327201843,
+      "rewards/safe_rewards": -0.42569422721862793,
+      "rewards/unsafe_rewards": -0.39291584491729736,
+      "step": 180
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999929391798331e-07,
+      "logits/chosen": -2.087995767593384,
+      "logits/rejected": -1.7015736103057861,
+      "logps/chosen": -233.88204956054688,
+      "logps/rejected": -231.9954071044922,
+      "loss": 1.1701,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.39558762311935425,
+      "rewards/margins": 0.25256142020225525,
+      "rewards/rejected": -0.6481491327285767,
+      "rewards/safe_rewards": -0.3778998851776123,
+      "rewards/unsafe_rewards": -0.4132753312587738,
+      "step": 190
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.9991350953333e-07,
+      "logits/chosen": -2.0101492404937744,
+      "logits/rejected": -1.688194990158081,
+      "logps/chosen": -269.3438720703125,
+      "logps/rejected": -272.6773376464844,
+      "loss": 1.1309,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.49628791213035583,
+      "rewards/margins": 0.21785268187522888,
+      "rewards/rejected": -0.7141406536102295,
+      "rewards/safe_rewards": -0.47186246514320374,
+      "rewards/unsafe_rewards": -0.5207133293151855,
+      "step": 200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997458523498236e-07,
+      "logits/chosen": -2.1067311763763428,
+      "logits/rejected": -1.8388845920562744,
+      "logps/chosen": -260.73553466796875,
+      "logps/rejected": -250.10794067382812,
+      "loss": 1.1343,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.685817539691925,
+      "rewards/margins": 0.2200475037097931,
+      "rewards/rejected": -0.9058650732040405,
+      "rewards/safe_rewards": -0.6602068543434143,
+      "rewards/unsafe_rewards": -0.7114282250404358,
+      "step": 210
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.99490026817712e-07,
+      "logits/chosen": -2.0778934955596924,
+      "logits/rejected": -1.7963426113128662,
+      "logps/chosen": -245.4992218017578,
+      "logps/rejected": -252.6900177001953,
+      "loss": 1.149,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.5112438201904297,
+      "rewards/margins": 0.3192596137523651,
+      "rewards/rejected": -0.8305034637451172,
+      "rewards/safe_rewards": -0.469885915517807,
+      "rewards/unsafe_rewards": -0.5526017546653748,
+      "step": 220
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.991461232516674e-07,
+      "logits/chosen": -2.041980266571045,
+      "logits/rejected": -1.7099599838256836,
+      "logps/chosen": -286.7567443847656,
+      "logps/rejected": -276.52020263671875,
+      "loss": 1.1816,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.566757082939148,
+      "rewards/margins": 0.22073951363563538,
+      "rewards/rejected": -0.7874965667724609,
+      "rewards/safe_rewards": -0.5972923040390015,
+      "rewards/unsafe_rewards": -0.5362219214439392,
+      "step": 230
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.98714263060751e-07,
+      "logits/chosen": -2.079230546951294,
+      "logits/rejected": -1.6963192224502563,
+      "logps/chosen": -245.6575927734375,
+      "logps/rejected": -231.0216064453125,
+      "loss": 1.1425,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.5724560022354126,
+      "rewards/margins": 0.2712286710739136,
+      "rewards/rejected": -0.8436846733093262,
+      "rewards/safe_rewards": -0.5816723108291626,
+      "rewards/unsafe_rewards": -0.5632396936416626,
+      "step": 240
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.98194598705552e-07,
+      "logits/chosen": -1.9290701150894165,
+      "logits/rejected": -1.7140071392059326,
+      "logps/chosen": -283.2284851074219,
+      "logps/rejected": -276.1746826171875,
+      "loss": 1.2033,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.9349914789199829,
+      "rewards/margins": 0.15353193879127502,
+      "rewards/rejected": -1.088523268699646,
+      "rewards/safe_rewards": -0.9215167164802551,
+      "rewards/unsafe_rewards": -0.9484661817550659,
+      "step": 250
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.975873136443648e-07,
+      "logits/chosen": -2.1985442638397217,
+      "logits/rejected": -1.9197852611541748,
+      "logps/chosen": -303.9427795410156,
+      "logps/rejected": -298.4468688964844,
+      "loss": 1.0412,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7289992570877075,
+      "rewards/margins": 0.2902063727378845,
+      "rewards/rejected": -1.0192055702209473,
+      "rewards/safe_rewards": -0.8002890348434448,
+      "rewards/unsafe_rewards": -0.6577093005180359,
+      "step": 260
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.968926222684212e-07,
+      "logits/chosen": -2.028428792953491,
+      "logits/rejected": -1.8198667764663696,
+      "logps/chosen": -262.1617126464844,
+      "logps/rejected": -279.1395568847656,
+      "loss": 1.0273,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.6195804476737976,
+      "rewards/margins": 0.3619759678840637,
+      "rewards/rejected": -0.9815564155578613,
+      "rewards/safe_rewards": -0.6148445010185242,
+      "rewards/unsafe_rewards": -0.6243164539337158,
+      "step": 270
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.961107698262044e-07,
+      "logits/chosen": -1.9460862874984741,
+      "logits/rejected": -1.6114823818206787,
+      "logps/chosen": -289.10284423828125,
+      "logps/rejected": -280.1482849121094,
+      "loss": 1.0933,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.70842045545578,
+      "rewards/margins": 0.2855362296104431,
+      "rewards/rejected": -0.9939567446708679,
+      "rewards/safe_rewards": -0.6650065183639526,
+      "rewards/unsafe_rewards": -0.7518342733383179,
+      "step": 280
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.952420323368673e-07,
+      "logits/chosen": -2.0242223739624023,
+      "logits/rejected": -1.8324100971221924,
+      "logps/chosen": -237.0861358642578,
+      "logps/rejected": -266.013916015625,
+      "loss": 1.1438,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.4568088948726654,
+      "rewards/margins": 0.3911629617214203,
+      "rewards/rejected": -0.8479719161987305,
+      "rewards/safe_rewards": -0.49832311272621155,
+      "rewards/unsafe_rewards": -0.41529473662376404,
+      "step": 290
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.942867164927899e-07,
+      "logits/chosen": -1.9857969284057617,
+      "logits/rejected": -1.7354557514190674,
+      "logps/chosen": -262.5624084472656,
+      "logps/rejected": -261.7809143066406,
+      "loss": 1.1983,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.6027632355690002,
+      "rewards/margins": 0.323146253824234,
+      "rewards/rejected": -0.9259093999862671,
+      "rewards/safe_rewards": -0.6246525645256042,
+      "rewards/unsafe_rewards": -0.5808738470077515,
+      "step": 300
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.932451595513062e-07,
+      "logits/chosen": -2.0492186546325684,
+      "logits/rejected": -1.6627346277236938,
+      "logps/chosen": -287.6437072753906,
+      "logps/rejected": -296.08685302734375,
+      "loss": 0.9844,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.6867466568946838,
+      "rewards/margins": 0.3914056420326233,
+      "rewards/rejected": -1.0781524181365967,
+      "rewards/safe_rewards": -0.7488200664520264,
+      "rewards/unsafe_rewards": -0.6246733069419861,
+      "step": 310
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.921177292156419e-07,
+      "logits/chosen": -2.0579938888549805,
+      "logits/rejected": -1.661026954650879,
+      "logps/chosen": -275.15484619140625,
+      "logps/rejected": -296.12921142578125,
+      "loss": 0.9846,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.6913672089576721,
+      "rewards/margins": 0.45374804735183716,
+      "rewards/rejected": -1.1451152563095093,
+      "rewards/safe_rewards": -0.6368889808654785,
+      "rewards/unsafe_rewards": -0.7458454370498657,
+      "step": 320
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.909048235051033e-07,
+      "logits/chosen": -1.8785194158554077,
+      "logits/rejected": -1.6157915592193604,
+      "logps/chosen": -287.54022216796875,
+      "logps/rejected": -308.53485107421875,
+      "loss": 1.05,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.881294846534729,
+      "rewards/margins": 0.377378910779953,
+      "rewards/rejected": -1.2586736679077148,
+      "rewards/safe_rewards": -0.8856824040412903,
+      "rewards/unsafe_rewards": -0.876907229423523,
+      "step": 330
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.896068706145631e-07,
+      "logits/chosen": -2.043916702270508,
+      "logits/rejected": -1.7367770671844482,
+      "logps/chosen": -311.2376708984375,
+      "logps/rejected": -299.87225341796875,
+      "loss": 1.0624,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.005437970161438,
+      "rewards/margins": 0.386522501707077,
+      "rewards/rejected": -1.3919605016708374,
+      "rewards/safe_rewards": -1.0309627056121826,
+      "rewards/unsafe_rewards": -0.9799133539199829,
+      "step": 340
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.882243287632946e-07,
+      "logits/chosen": -2.2845442295074463,
+      "logits/rejected": -2.015984058380127,
+      "logps/chosen": -249.07626342773438,
+      "logps/rejected": -270.349609375,
+      "loss": 1.0634,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.5606988668441772,
+      "rewards/margins": 0.3308585584163666,
+      "rewards/rejected": -0.8915573954582214,
+      "rewards/safe_rewards": -0.5756514668464661,
+      "rewards/unsafe_rewards": -0.5457462072372437,
+      "step": 350
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.867576860332048e-07,
+      "logits/chosen": -2.2117257118225098,
+      "logits/rejected": -1.9762624502182007,
+      "logps/chosen": -250.96493530273438,
+      "logps/rejected": -287.42401123046875,
+      "loss": 1.0082,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.7890839576721191,
+      "rewards/margins": 0.39461660385131836,
+      "rewards/rejected": -1.1837005615234375,
+      "rewards/safe_rewards": -0.8431307673454285,
+      "rewards/unsafe_rewards": -0.7350370287895203,
+      "step": 360
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.85207460196526e-07,
+      "logits/chosen": -2.083486557006836,
+      "logits/rejected": -1.7575275897979736,
+      "logps/chosen": -315.93890380859375,
+      "logps/rejected": -334.19989013671875,
+      "loss": 1.0535,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1155636310577393,
+      "rewards/margins": 0.3764941096305847,
+      "rewards/rejected": -1.4920578002929688,
+      "rewards/safe_rewards": -1.1053030490875244,
+      "rewards/unsafe_rewards": -1.125824213027954,
+      "step": 370
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.835741985330259e-07,
+      "logits/chosen": -2.07675838470459,
+      "logits/rejected": -1.7431271076202393,
+      "logps/chosen": -267.29132080078125,
+      "logps/rejected": -277.96624755859375,
+      "loss": 1.0022,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.6648051142692566,
+      "rewards/margins": 0.39567989110946655,
+      "rewards/rejected": -1.0604850053787231,
+      "rewards/safe_rewards": -0.6165894865989685,
+      "rewards/unsafe_rewards": -0.7130206823348999,
+      "step": 380
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.818584776367992e-07,
+      "logits/chosen": -1.928900122642517,
+      "logits/rejected": -1.7115558385849,
+      "logps/chosen": -286.6376647949219,
+      "logps/rejected": -315.5821838378906,
+      "loss": 1.0452,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.8273309469223022,
+      "rewards/margins": 0.4084450602531433,
+      "rewards/rejected": -1.2357759475708008,
+      "rewards/safe_rewards": -0.8857539892196655,
+      "rewards/unsafe_rewards": -0.7689078450202942,
+      "step": 390
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.800609032127122e-07,
+      "logits/chosen": -1.9402987957000732,
+      "logits/rejected": -1.653032898902893,
+      "logps/chosen": -315.54443359375,
+      "logps/rejected": -311.26129150390625,
+      "loss": 1.096,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1000012159347534,
+      "rewards/margins": 0.32444682717323303,
+      "rewards/rejected": -1.4244478940963745,
+      "rewards/safe_rewards": -1.031652569770813,
+      "rewards/unsafe_rewards": -1.1683496236801147,
+      "step": 400
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.78182109862569e-07,
+      "logits/chosen": -1.957297682762146,
+      "logits/rejected": -1.815405249595642,
+      "logps/chosen": -275.63458251953125,
+      "logps/rejected": -291.7708435058594,
+      "loss": 1.1237,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.8423420786857605,
+      "rewards/margins": 0.27593573927879333,
+      "rewards/rejected": -1.1182777881622314,
+      "rewards/safe_rewards": -0.7917040586471558,
+      "rewards/unsafe_rewards": -0.8929800987243652,
+      "step": 410
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7622276086107677e-07,
+      "logits/chosen": -2.0669121742248535,
+      "logits/rejected": -1.759894609451294,
+      "logps/chosen": -285.4234924316406,
+      "logps/rejected": -295.45294189453125,
+      "loss": 1.0445,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7513025999069214,
+      "rewards/margins": 0.3429652750492096,
+      "rewards/rejected": -1.0942678451538086,
+      "rewards/safe_rewards": -0.8147345781326294,
+      "rewards/unsafe_rewards": -0.6878706216812134,
+      "step": 420
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.741835479216879e-07,
+      "logits/chosen": -2.0404961109161377,
+      "logits/rejected": -1.6269737482070923,
+      "logps/chosen": -326.9522399902344,
+      "logps/rejected": -324.0838623046875,
+      "loss": 1.0151,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.8008167147636414,
+      "rewards/margins": 0.46369487047195435,
+      "rewards/rejected": -1.2645115852355957,
+      "rewards/safe_rewards": -0.8669862747192383,
+      "rewards/unsafe_rewards": -0.7346470952033997,
+      "step": 430
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.720651909524036e-07,
+      "logits/chosen": -2.04176664352417,
+      "logits/rejected": -1.7474247217178345,
+      "logps/chosen": -265.5859069824219,
+      "logps/rejected": -273.75836181640625,
+      "loss": 1.0613,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.6951051950454712,
+      "rewards/margins": 0.3854682147502899,
+      "rewards/rejected": -1.0805734395980835,
+      "rewards/safe_rewards": -0.8021440505981445,
+      "rewards/unsafe_rewards": -0.5880664587020874,
+      "step": 440
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.698684378016222e-07,
+      "logits/chosen": -1.9997985363006592,
+      "logits/rejected": -1.7316805124282837,
+      "logps/chosen": -258.0675354003906,
+      "logps/rejected": -275.7622375488281,
+      "loss": 1.057,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.7074462175369263,
+      "rewards/margins": 0.33383387327194214,
+      "rewards/rejected": -1.0412800312042236,
+      "rewards/safe_rewards": -0.6980961561203003,
+      "rewards/unsafe_rewards": -0.7167961597442627,
+      "step": 450
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.675940639941256e-07,
+      "logits/chosen": -2.0017848014831543,
+      "logits/rejected": -1.6473373174667358,
+      "logps/chosen": -284.409912109375,
+      "logps/rejected": -302.1802978515625,
+      "loss": 0.9964,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.7972325086593628,
+      "rewards/margins": 0.45086246728897095,
+      "rewards/rejected": -1.248094916343689,
+      "rewards/safe_rewards": -0.7986913919448853,
+      "rewards/unsafe_rewards": -0.7957736849784851,
+      "step": 460
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.6524287245729286e-07,
+      "logits/chosen": -1.8008419275283813,
+      "logits/rejected": -1.545585036277771,
+      "logps/chosen": -285.72833251953125,
+      "logps/rejected": -295.92877197265625,
+      "loss": 1.0215,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.9112635850906372,
+      "rewards/margins": 0.3833921253681183,
+      "rewards/rejected": -1.2946555614471436,
+      "rewards/safe_rewards": -0.9739904403686523,
+      "rewards/unsafe_rewards": -0.8485366702079773,
+      "step": 470
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.628156932376418e-07,
+      "logits/chosen": -1.935253381729126,
+      "logits/rejected": -1.5615403652191162,
+      "logps/chosen": -283.60662841796875,
+      "logps/rejected": -275.73553466796875,
+      "loss": 1.0577,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8973907232284546,
+      "rewards/margins": 0.34667691588401794,
+      "rewards/rejected": -1.244067668914795,
+      "rewards/safe_rewards": -0.9187390208244324,
+      "rewards/unsafe_rewards": -0.8760424852371216,
+      "step": 480
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.603133832077953e-07,
+      "logits/chosen": -2.0226516723632812,
+      "logits/rejected": -1.757315993309021,
+      "logps/chosen": -319.8951416015625,
+      "logps/rejected": -344.83306884765625,
+      "loss": 1.0078,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8721585273742676,
+      "rewards/margins": 0.43064364790916443,
+      "rewards/rejected": -1.302802324295044,
+      "rewards/safe_rewards": -0.8747318983078003,
+      "rewards/unsafe_rewards": -0.8695852160453796,
+      "step": 490
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5773682576397776e-07,
+      "logits/chosen": -1.8940719366073608,
+      "logits/rejected": -1.6204118728637695,
+      "logps/chosen": -282.92388916015625,
+      "logps/rejected": -298.21685791015625,
+      "loss": 1.0274,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8302618265151978,
+      "rewards/margins": 0.41684699058532715,
+      "rewards/rejected": -1.2471086978912354,
+      "rewards/safe_rewards": -0.8429195284843445,
+      "rewards/unsafe_rewards": -0.8176040649414062,
+      "step": 500
+    },
+    {
+      "epoch": 0.27,
+      "eval_logits/chosen": -1.461648941040039,
+      "eval_logits/rejected": -0.9854875206947327,
+      "eval_logps/chosen": -232.3561248779297,
+      "eval_logps/rejected": -222.5215606689453,
+      "eval_loss": 0.37307849526405334,
+      "eval_rewards/accuracies": 0.7075266242027283,
+      "eval_rewards/chosen": -1.0191720724105835,
+      "eval_rewards/margins": 0.28133097290992737,
+      "eval_rewards/rejected": -1.300503134727478,
+      "eval_rewards/safe_rewards": -1.0088554620742798,
+      "eval_rewards/unsafe_rewards": -1.028071403503418,
+      "eval_runtime": 1058.7885,
+      "eval_samples_per_second": 31.209,
+      "eval_steps_per_second": 0.976,
+      "step": 500
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5508693051414774e-07,
+      "logits/chosen": -1.9713261127471924,
+      "logits/rejected": -1.7668718099594116,
+      "logps/chosen": -295.57354736328125,
+      "logps/rejected": -315.67010498046875,
+      "loss": 1.0145,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.992205023765564,
+      "rewards/margins": 0.4039764404296875,
+      "rewards/rejected": -1.3961814641952515,
+      "rewards/safe_rewards": -0.9630918502807617,
+      "rewards/unsafe_rewards": -1.0213181972503662,
+      "step": 510
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.52364632956877e-07,
+      "logits/chosen": -2.020744562149048,
+      "logits/rejected": -1.7648818492889404,
+      "logps/chosen": -306.5279235839844,
+      "logps/rejected": -284.1745910644531,
+      "loss": 1.1779,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9038923978805542,
+      "rewards/margins": 0.2524818778038025,
+      "rewards/rejected": -1.1563743352890015,
+      "rewards/safe_rewards": -0.9023400545120239,
+      "rewards/unsafe_rewards": -0.9054449200630188,
+      "step": 520
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4957089415108895e-07,
+      "logits/chosen": -1.965404748916626,
+      "logits/rejected": -1.6829001903533936,
+      "logps/chosen": -266.58013916015625,
+      "logps/rejected": -311.5513916015625,
+      "loss": 0.9603,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.7168244123458862,
+      "rewards/margins": 0.5065333247184753,
+      "rewards/rejected": -1.2233576774597168,
+      "rewards/safe_rewards": -0.7431866526603699,
+      "rewards/unsafe_rewards": -0.6904621720314026,
+      "step": 530
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.467067003767745e-07,
+      "logits/chosen": -1.9147508144378662,
+      "logits/rejected": -1.4926608800888062,
+      "logps/chosen": -276.83740234375,
+      "logps/rejected": -303.65716552734375,
+      "loss": 1.0585,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.8392030000686646,
+      "rewards/margins": 0.5333585739135742,
+      "rewards/rejected": -1.3725616931915283,
+      "rewards/safe_rewards": -0.8698552250862122,
+      "rewards/unsafe_rewards": -0.8085508346557617,
+      "step": 540
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.437730627868027e-07,
+      "logits/chosen": -1.8325055837631226,
+      "logits/rejected": -1.4082276821136475,
+      "logps/chosen": -256.5406799316406,
+      "logps/rejected": -277.9321594238281,
+      "loss": 0.9491,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.7962465286254883,
+      "rewards/margins": 0.5230618715286255,
+      "rewards/rejected": -1.3193082809448242,
+      "rewards/safe_rewards": -0.7890614867210388,
+      "rewards/unsafe_rewards": -0.803431510925293,
+      "step": 550
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.4077101704995163e-07,
+      "logits/chosen": -1.9758007526397705,
+      "logits/rejected": -1.716571569442749,
+      "logps/chosen": -271.619140625,
+      "logps/rejected": -284.5029602050781,
+      "loss": 1.0222,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.7951744198799133,
+      "rewards/margins": 0.3159455358982086,
+      "rewards/rejected": -1.1111198663711548,
+      "rewards/safe_rewards": -0.751990020275116,
+      "rewards/unsafe_rewards": -0.8383587002754211,
+      "step": 560
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3770162298528356e-07,
+      "logits/chosen": -1.9304263591766357,
+      "logits/rejected": -1.6568174362182617,
+      "logps/chosen": -294.840087890625,
+      "logps/rejected": -296.0787658691406,
+      "loss": 1.0612,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.9997127652168274,
+      "rewards/margins": 0.4206429421901703,
+      "rewards/rejected": -1.4203556776046753,
+      "rewards/safe_rewards": -0.9456769824028015,
+      "rewards/unsafe_rewards": -1.053748369216919,
+      "step": 570
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3456596418799476e-07,
+      "logits/chosen": -1.8863794803619385,
+      "logits/rejected": -1.6375776529312134,
+      "logps/chosen": -308.4244689941406,
+      "logps/rejected": -312.00836181640625,
+      "loss": 0.9379,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0467333793640137,
+      "rewards/margins": 0.35309094190597534,
+      "rewards/rejected": -1.3998241424560547,
+      "rewards/safe_rewards": -1.0730044841766357,
+      "rewards/unsafe_rewards": -1.0204620361328125,
+      "step": 580
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.313651476468715e-07,
+      "logits/chosen": -1.9189523458480835,
+      "logits/rejected": -1.6596009731292725,
+      "logps/chosen": -302.38214111328125,
+      "logps/rejected": -310.23687744140625,
+      "loss": 0.9881,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.0581415891647339,
+      "rewards/margins": 0.3177962601184845,
+      "rewards/rejected": -1.375937819480896,
+      "rewards/safe_rewards": -1.1356565952301025,
+      "rewards/unsafe_rewards": -0.9806265830993652,
+      "step": 590
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.2810030335348693e-07,
+      "logits/chosen": -1.9322669506072998,
+      "logits/rejected": -1.6003156900405884,
+      "logps/chosen": -305.31378173828125,
+      "logps/rejected": -295.27435302734375,
+      "loss": 1.0005,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.9873701930046082,
+      "rewards/margins": 0.315813809633255,
+      "rewards/rejected": -1.3031837940216064,
+      "rewards/safe_rewards": -0.9451528787612915,
+      "rewards/unsafe_rewards": -1.0295875072479248,
+      "step": 600
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2477258390327806e-07,
+      "logits/chosen": -1.9405990839004517,
+      "logits/rejected": -1.6028436422348022,
+      "logps/chosen": -266.96820068359375,
+      "logps/rejected": -300.30810546875,
+      "loss": 0.952,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.8195999264717102,
+      "rewards/margins": 0.4460810720920563,
+      "rewards/rejected": -1.2656810283660889,
+      "rewards/safe_rewards": -0.8683537244796753,
+      "rewards/unsafe_rewards": -0.7708461880683899,
+      "step": 610
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2138316408864197e-07,
+      "logits/chosen": -1.9435718059539795,
+      "logits/rejected": -1.5219794511795044,
+      "logps/chosen": -273.6173400878906,
+      "logps/rejected": -296.8672790527344,
+      "loss": 0.8771,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.7538294196128845,
+      "rewards/margins": 0.5905435085296631,
+      "rewards/rejected": -1.3443728685379028,
+      "rewards/safe_rewards": -0.7166475653648376,
+      "rewards/unsafe_rewards": -0.7910112142562866,
+      "step": 620
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.179332404841962e-07,
+      "logits/chosen": -1.6942704916000366,
+      "logits/rejected": -1.2099497318267822,
+      "logps/chosen": -327.61376953125,
+      "logps/rejected": -344.44866943359375,
+      "loss": 0.9468,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.1748586893081665,
+      "rewards/margins": 0.5536222457885742,
+      "rewards/rejected": -1.7284809350967407,
+      "rewards/safe_rewards": -1.184999942779541,
+      "rewards/unsafe_rewards": -1.1647173166275024,
+      "step": 630
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.1442403102434954e-07,
+      "logits/chosen": -1.7647279500961304,
+      "logits/rejected": -1.3675248622894287,
+      "logps/chosen": -310.9916076660156,
+      "logps/rejected": -321.579345703125,
+      "loss": 1.0042,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9101033210754395,
+      "rewards/margins": 0.5184718370437622,
+      "rewards/rejected": -1.428575038909912,
+      "rewards/safe_rewards": -0.9146644473075867,
+      "rewards/unsafe_rewards": -0.9055421948432922,
+      "step": 640
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.108567745733318e-07,
+      "logits/chosen": -1.7713772058486938,
+      "logits/rejected": -1.3516300916671753,
+      "logps/chosen": -255.63150024414062,
+      "logps/rejected": -284.5046081542969,
+      "loss": 1.0245,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.821179986000061,
+      "rewards/margins": 0.4137188792228699,
+      "rewards/rejected": -1.2348989248275757,
+      "rewards/safe_rewards": -0.8415560722351074,
+      "rewards/unsafe_rewards": -0.8008037805557251,
+      "step": 650
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.0723273048783426e-07,
+      "logits/chosen": -1.8442039489746094,
+      "logits/rejected": -1.5036883354187012,
+      "logps/chosen": -304.8941650390625,
+      "logps/rejected": -295.1714172363281,
+      "loss": 1.0401,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.8060005903244019,
+      "rewards/margins": 0.4549127221107483,
+      "rewards/rejected": -1.2609132528305054,
+      "rewards/safe_rewards": -0.7164521813392639,
+      "rewards/unsafe_rewards": -0.8955489993095398,
+      "step": 660
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.0355317817241697e-07,
+      "logits/chosen": -1.8206180334091187,
+      "logits/rejected": -1.4083284139633179,
+      "logps/chosen": -316.703369140625,
+      "logps/rejected": -288.38323974609375,
+      "loss": 1.0164,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.8776229619979858,
+      "rewards/margins": 0.3921293020248413,
+      "rewards/rejected": -1.2697522640228271,
+      "rewards/safe_rewards": -0.8068926930427551,
+      "rewards/unsafe_rewards": -0.9483532905578613,
+      "step": 670
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.998194166278367e-07,
+      "logits/chosen": -1.7814273834228516,
+      "logits/rejected": -1.490839958190918,
+      "logps/chosen": -295.8428955078125,
+      "logps/rejected": -305.77972412109375,
+      "loss": 1.0484,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.0455328226089478,
+      "rewards/margins": 0.34231680631637573,
+      "rewards/rejected": -1.3878495693206787,
+      "rewards/safe_rewards": -1.0646705627441406,
+      "rewards/unsafe_rewards": -1.0263949632644653,
+      "step": 680
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9603276399245855e-07,
+      "logits/chosen": -1.8057140111923218,
+      "logits/rejected": -1.3926641941070557,
+      "logps/chosen": -323.49139404296875,
+      "logps/rejected": -323.6500549316406,
+      "loss": 1.0317,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.1086748838424683,
+      "rewards/margins": 0.44791918992996216,
+      "rewards/rejected": -1.556593894958496,
+      "rewards/safe_rewards": -1.087368130683899,
+      "rewards/unsafe_rewards": -1.1299816370010376,
+      "step": 690
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.9219455707691e-07,
+      "logits/chosen": -1.895880937576294,
+      "logits/rejected": -1.5151503086090088,
+      "logps/chosen": -295.07525634765625,
+      "logps/rejected": -304.41241455078125,
+      "loss": 0.9783,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9127877950668335,
+      "rewards/margins": 0.4130992889404297,
+      "rewards/rejected": -1.3258870840072632,
+      "rewards/safe_rewards": -0.9074820280075073,
+      "rewards/unsafe_rewards": -0.9180935025215149,
+      "step": 700
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.883061508921439e-07,
+      "logits/chosen": -1.9308559894561768,
+      "logits/rejected": -1.6609346866607666,
+      "logps/chosen": -277.08172607421875,
+      "logps/rejected": -316.75994873046875,
+      "loss": 1.0027,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.7832189798355103,
+      "rewards/margins": 0.37469321489334106,
+      "rewards/rejected": -1.1579121351242065,
+      "rewards/safe_rewards": -0.7912700176239014,
+      "rewards/unsafe_rewards": -0.7751679420471191,
+      "step": 710
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8436891817107555e-07,
+      "logits/chosen": -1.761370301246643,
+      "logits/rejected": -1.5160869359970093,
+      "logps/chosen": -285.6172790527344,
+      "logps/rejected": -315.7353210449219,
+      "loss": 1.0302,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.9513989686965942,
+      "rewards/margins": 0.45606541633605957,
+      "rewards/rejected": -1.4074645042419434,
+      "rewards/safe_rewards": -1.0125133991241455,
+      "rewards/unsafe_rewards": -0.890284538269043,
+      "step": 720
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8038424888396414e-07,
+      "logits/chosen": -1.8194071054458618,
+      "logits/rejected": -1.4451847076416016,
+      "logps/chosen": -295.29241943359375,
+      "logps/rejected": -320.0098571777344,
+      "loss": 0.9634,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8949946165084839,
+      "rewards/margins": 0.4506065249443054,
+      "rewards/rejected": -1.3456013202667236,
+      "rewards/safe_rewards": -0.855640709400177,
+      "rewards/unsafe_rewards": -0.9343485832214355,
+      "step": 730
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.763535497477079e-07,
+      "logits/chosen": -1.8460315465927124,
+      "logits/rejected": -1.4700871706008911,
+      "logps/chosen": -310.37518310546875,
+      "logps/rejected": -316.61236572265625,
+      "loss": 0.9976,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.0227649211883545,
+      "rewards/margins": 0.43325185775756836,
+      "rewards/rejected": -1.4560167789459229,
+      "rewards/safe_rewards": -1.0539515018463135,
+      "rewards/unsafe_rewards": -0.9915785789489746,
+      "step": 740
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7227824372922795e-07,
+      "logits/chosen": -1.8418188095092773,
+      "logits/rejected": -1.479627013206482,
+      "logps/chosen": -286.8067321777344,
+      "logps/rejected": -302.6395568847656,
+      "loss": 0.9795,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.9858185052871704,
+      "rewards/margins": 0.41545191407203674,
+      "rewards/rejected": -1.4012705087661743,
+      "rewards/safe_rewards": -0.9399738311767578,
+      "rewards/unsafe_rewards": -1.031663179397583,
+      "step": 750
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.681597695431148e-07,
+      "logits/chosen": -1.749204397201538,
+      "logits/rejected": -1.3665492534637451,
+      "logps/chosen": -289.353271484375,
+      "logps/rejected": -327.75677490234375,
+      "loss": 0.9449,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.9073783755302429,
+      "rewards/margins": 0.5846059918403625,
+      "rewards/rejected": -1.4919843673706055,
+      "rewards/safe_rewards": -0.9077240228652954,
+      "rewards/unsafe_rewards": -0.9070326089859009,
+      "step": 760
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.639995811437159e-07,
+      "logits/chosen": -1.667654037475586,
+      "logits/rejected": -1.314335584640503,
+      "logps/chosen": -289.2320556640625,
+      "logps/rejected": -326.54229736328125,
+      "loss": 0.9656,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9039511680603027,
+      "rewards/margins": 0.5181721448898315,
+      "rewards/rejected": -1.4221234321594238,
+      "rewards/safe_rewards": -0.9193918108940125,
+      "rewards/unsafe_rewards": -0.8885105848312378,
+      "step": 770
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.597991472118426e-07,
+      "logits/chosen": -1.7382431030273438,
+      "logits/rejected": -1.3142716884613037,
+      "logps/chosen": -300.62677001953125,
+      "logps/rejected": -313.65814208984375,
+      "loss": 1.055,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8587741851806641,
+      "rewards/margins": 0.427274614572525,
+      "rewards/rejected": -1.2860486507415771,
+      "rewards/safe_rewards": -0.8905943036079407,
+      "rewards/unsafe_rewards": -0.8269540071487427,
+      "step": 780
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5555995063627836e-07,
+      "logits/chosen": -1.8035333156585693,
+      "logits/rejected": -1.4783815145492554,
+      "logps/chosen": -321.8671875,
+      "logps/rejected": -317.4945068359375,
+      "loss": 0.9705,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.9154338836669922,
+      "rewards/margins": 0.400259792804718,
+      "rewards/rejected": -1.3156936168670654,
+      "rewards/safe_rewards": -0.946729838848114,
+      "rewards/unsafe_rewards": -0.8841378092765808,
+      "step": 790
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.512834879902715e-07,
+      "logits/chosen": -1.7292006015777588,
+      "logits/rejected": -1.3531233072280884,
+      "logps/chosen": -297.385009765625,
+      "logps/rejected": -316.39666748046875,
+      "loss": 0.9624,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9740325808525085,
+      "rewards/margins": 0.43682414293289185,
+      "rewards/rejected": -1.4108567237854004,
+      "rewards/safe_rewards": -0.979016900062561,
+      "rewards/unsafe_rewards": -0.9690481424331665,
+      "step": 800
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4697126900319616e-07,
+      "logits/chosen": -1.6081682443618774,
+      "logits/rejected": -1.1794389486312866,
+      "logps/chosen": -295.1870422363281,
+      "logps/rejected": -306.71160888671875,
+      "loss": 1.0047,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.9457162022590637,
+      "rewards/margins": 0.4736977517604828,
+      "rewards/rejected": -1.4194139242172241,
+      "rewards/safe_rewards": -0.8538358807563782,
+      "rewards/unsafe_rewards": -1.0375964641571045,
+      "step": 810
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.426248160275693e-07,
+      "logits/chosen": -1.7311818599700928,
+      "logits/rejected": -1.3600565195083618,
+      "logps/chosen": -278.1633605957031,
+      "logps/rejected": -299.32635498046875,
+      "loss": 1.0259,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.8063637018203735,
+      "rewards/margins": 0.43319129943847656,
+      "rewards/rejected": -1.2395551204681396,
+      "rewards/safe_rewards": -0.859821617603302,
+      "rewards/unsafe_rewards": -0.7529059052467346,
+      "step": 820
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3824566350161094e-07,
+      "logits/chosen": -1.785316824913025,
+      "logits/rejected": -1.323632836341858,
+      "logps/chosen": -275.93060302734375,
+      "logps/rejected": -278.7740783691406,
+      "loss": 0.9551,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.675979495048523,
+      "rewards/margins": 0.428046852350235,
+      "rewards/rejected": -1.104026436805725,
+      "rewards/safe_rewards": -0.6841101050376892,
+      "rewards/unsafe_rewards": -0.6678491234779358,
+      "step": 830
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.338353574075381e-07,
+      "logits/chosen": -1.5819236040115356,
+      "logits/rejected": -1.3700740337371826,
+      "logps/chosen": -266.2092590332031,
+      "logps/rejected": -284.17657470703125,
+      "loss": 1.1627,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.9084580540657043,
+      "rewards/margins": 0.3083532750606537,
+      "rewards/rejected": -1.216811180114746,
+      "rewards/safe_rewards": -0.9423872828483582,
+      "rewards/unsafe_rewards": -0.8745288848876953,
+      "step": 840
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2939545472578314e-07,
+      "logits/chosen": -1.7262417078018188,
+      "logits/rejected": -1.2280857563018799,
+      "logps/chosen": -324.2544860839844,
+      "logps/rejected": -322.0166320800781,
+      "loss": 1.002,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9809592366218567,
+      "rewards/margins": 0.39795732498168945,
+      "rewards/rejected": -1.3789165019989014,
+      "rewards/safe_rewards": -0.9736999273300171,
+      "rewards/unsafe_rewards": -0.9882184863090515,
+      "step": 850
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2492752288532916e-07,
+      "logits/chosen": -1.7719913721084595,
+      "logits/rejected": -1.3637843132019043,
+      "logps/chosen": -289.90692138671875,
+      "logps/rejected": -295.659423828125,
+      "loss": 1.0013,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.8620317578315735,
+      "rewards/margins": 0.4116358757019043,
+      "rewards/rejected": -1.273667573928833,
+      "rewards/safe_rewards": -0.8024783134460449,
+      "rewards/unsafe_rewards": -0.9215850830078125,
+      "step": 860
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.204331392103574e-07,
+      "logits/chosen": -1.8806778192520142,
+      "logits/rejected": -1.389723539352417,
+      "logps/chosen": -291.46832275390625,
+      "logps/rejected": -283.912841796875,
+      "loss": 0.9879,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.7795971035957336,
+      "rewards/margins": 0.4201742112636566,
+      "rewards/rejected": -1.1997714042663574,
+      "rewards/safe_rewards": -0.8063246607780457,
+      "rewards/unsafe_rewards": -0.7528696060180664,
+      "step": 870
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.159138903634006e-07,
+      "logits/chosen": -1.6265941858291626,
+      "logits/rejected": -1.263318657875061,
+      "logps/chosen": -308.3788757324219,
+      "logps/rejected": -303.78753662109375,
+      "loss": 0.9939,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.9974681735038757,
+      "rewards/margins": 0.31481030583381653,
+      "rewards/rejected": -1.312278389930725,
+      "rewards/safe_rewards": -0.9676594734191895,
+      "rewards/unsafe_rewards": -1.0272767543792725,
+      "step": 880
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1137137178519977e-07,
+      "logits/chosen": -1.5345653295516968,
+      "logits/rejected": -1.2038419246673584,
+      "logps/chosen": -277.8311767578125,
+      "logps/rejected": -317.1457214355469,
+      "loss": 0.9773,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.9578744769096375,
+      "rewards/margins": 0.4849475026130676,
+      "rewards/rejected": -1.4428222179412842,
+      "rewards/safe_rewards": -0.9253692626953125,
+      "rewards/unsafe_rewards": -0.9903799891471863,
+      "step": 890
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.068071871314626e-07,
+      "logits/chosen": -1.477648138999939,
+      "logits/rejected": -1.1965210437774658,
+      "logps/chosen": -279.1951599121094,
+      "logps/rejected": -292.29833984375,
+      "loss": 0.9773,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.0047625303268433,
+      "rewards/margins": 0.3235151171684265,
+      "rewards/rejected": -1.328277826309204,
+      "rewards/safe_rewards": -1.013329267501831,
+      "rewards/unsafe_rewards": -0.9961959719657898,
+      "step": 900
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.022229477067205e-07,
+      "logits/chosen": -1.6116926670074463,
+      "logits/rejected": -1.2270816564559937,
+      "logps/chosen": -320.8763732910156,
+      "logps/rejected": -318.58355712890625,
+      "loss": 0.8831,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.0378270149230957,
+      "rewards/margins": 0.4454011917114258,
+      "rewards/rejected": -1.4832282066345215,
+      "rewards/safe_rewards": -1.0344152450561523,
+      "rewards/unsafe_rewards": -1.0412386655807495,
+      "step": 910
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.976202718954869e-07,
+      "logits/chosen": -1.5823721885681152,
+      "logits/rejected": -1.1224069595336914,
+      "logps/chosen": -333.028564453125,
+      "logps/rejected": -350.3519592285156,
+      "loss": 1.0709,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.2154980897903442,
+      "rewards/margins": 0.46428003907203674,
+      "rewards/rejected": -1.6797780990600586,
+      "rewards/safe_rewards": -1.1762654781341553,
+      "rewards/unsafe_rewards": -1.2547308206558228,
+      "step": 920
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.930007845909146e-07,
+      "logits/chosen": -1.5423873662948608,
+      "logits/rejected": -1.2088254690170288,
+      "logps/chosen": -333.67645263671875,
+      "logps/rejected": -360.3753662109375,
+      "loss": 0.9999,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.3605718612670898,
+      "rewards/margins": 0.422391802072525,
+      "rewards/rejected": -1.782963752746582,
+      "rewards/safe_rewards": -1.422663688659668,
+      "rewards/unsafe_rewards": -1.2984803915023804,
+      "step": 930
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8836611662115634e-07,
+      "logits/chosen": -1.6511509418487549,
+      "logits/rejected": -1.275423288345337,
+      "logps/chosen": -321.2275085449219,
+      "logps/rejected": -312.20733642578125,
+      "loss": 1.0348,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.010995626449585,
+      "rewards/margins": 0.42944231629371643,
+      "rewards/rejected": -1.440437912940979,
+      "rewards/safe_rewards": -0.9675294160842896,
+      "rewards/unsafe_rewards": -1.0544618368148804,
+      "step": 940
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8371790417362986e-07,
+      "logits/chosen": -1.5915000438690186,
+      "logits/rejected": -1.2896572351455688,
+      "logps/chosen": -281.606201171875,
+      "logps/rejected": -308.20916748046875,
+      "loss": 1.0666,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.9426482319831848,
+      "rewards/margins": 0.37754327058792114,
+      "rewards/rejected": -1.3201916217803955,
+      "rewards/safe_rewards": -0.8953598141670227,
+      "rewards/unsafe_rewards": -0.9899368286132812,
+      "step": 950
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7905778821739056e-07,
+      "logits/chosen": -1.478126049041748,
+      "logits/rejected": -1.1587542295455933,
+      "logps/chosen": -301.42181396484375,
+      "logps/rejected": -304.0444030761719,
+      "loss": 0.9759,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.0664873123168945,
+      "rewards/margins": 0.36265167593955994,
+      "rewards/rejected": -1.4291390180587769,
+      "rewards/safe_rewards": -1.0449130535125732,
+      "rewards/unsafe_rewards": -1.0880613327026367,
+      "step": 960
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.74387413923817e-07,
+      "logits/chosen": -1.4300400018692017,
+      "logits/rejected": -1.1722289323806763,
+      "logps/chosen": -327.14739990234375,
+      "logps/rejected": -327.6680908203125,
+      "loss": 0.977,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.026637315750122,
+      "rewards/margins": 0.39583832025527954,
+      "rewards/rejected": -1.4224755764007568,
+      "rewards/safe_rewards": -1.0289757251739502,
+      "rewards/unsafe_rewards": -1.024298906326294,
+      "step": 970
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.69708430085812e-07,
+      "logits/chosen": -1.6212282180786133,
+      "logits/rejected": -1.1375267505645752,
+      "logps/chosen": -331.76654052734375,
+      "logps/rejected": -341.3858947753906,
+      "loss": 1.0282,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0838706493377686,
+      "rewards/margins": 0.5636218190193176,
+      "rewards/rejected": -1.6474926471710205,
+      "rewards/safe_rewards": -1.0060853958129883,
+      "rewards/unsafe_rewards": -1.161656141281128,
+      "step": 980
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6502248853572504e-07,
+      "logits/chosen": -1.5994454622268677,
+      "logits/rejected": -1.2842458486557007,
+      "logps/chosen": -291.59063720703125,
+      "logps/rejected": -317.38970947265625,
+      "loss": 1.0111,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0371322631835938,
+      "rewards/margins": 0.5142688155174255,
+      "rewards/rejected": -1.551400899887085,
+      "rewards/safe_rewards": -1.0258736610412598,
+      "rewards/unsafe_rewards": -1.0483907461166382,
+      "step": 990
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6033124356220325e-07,
+      "logits/chosen": -1.5264742374420166,
+      "logits/rejected": -1.1301841735839844,
+      "logps/chosen": -305.7037048339844,
+      "logps/rejected": -310.7588806152344,
+      "loss": 0.9569,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.029294490814209,
+      "rewards/margins": 0.4845431447029114,
+      "rewards/rejected": -1.5138375759124756,
+      "rewards/safe_rewards": -0.948198139667511,
+      "rewards/unsafe_rewards": -1.1103906631469727,
+      "step": 1000
+    },
+    {
+      "epoch": 0.54,
+      "eval_logits/chosen": -1.0711549520492554,
+      "eval_logits/rejected": -0.5821475982666016,
+      "eval_logps/chosen": -221.7958984375,
+      "eval_logps/rejected": -212.7307586669922,
+      "eval_loss": 0.34972870349884033,
+      "eval_rewards/accuracies": 0.7209583520889282,
+      "eval_rewards/chosen": -0.913569986820221,
+      "eval_rewards/margins": 0.28902512788772583,
+      "eval_rewards/rejected": -1.2025949954986572,
+      "eval_rewards/safe_rewards": -0.9005662798881531,
+      "eval_rewards/unsafe_rewards": -0.9165622591972351,
+      "eval_runtime": 1062.9878,
+      "eval_samples_per_second": 31.086,
+      "eval_steps_per_second": 0.972,
+      "step": 1000
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.55636351326173e-07,
+      "logits/chosen": -1.4447015523910522,
+      "logits/rejected": -1.0418920516967773,
+      "logps/chosen": -320.98358154296875,
+      "logps/rejected": -330.81341552734375,
+      "loss": 0.8958,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.149300217628479,
+      "rewards/margins": 0.562938928604126,
+      "rewards/rejected": -1.7122390270233154,
+      "rewards/safe_rewards": -1.1756014823913574,
+      "rewards/unsafe_rewards": -1.122998595237732,
+      "step": 1010
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.509394692761622e-07,
+      "logits/chosen": -1.4893229007720947,
+      "logits/rejected": -1.0318996906280518,
+      "logps/chosen": -320.03070068359375,
+      "logps/rejected": -331.4481201171875,
+      "loss": 0.9718,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1435344219207764,
+      "rewards/margins": 0.5211852788925171,
+      "rewards/rejected": -1.6647193431854248,
+      "rewards/safe_rewards": -1.1441400051116943,
+      "rewards/unsafe_rewards": -1.1429284811019897,
+      "step": 1020
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.462422555631674e-07,
+      "logits/chosen": -1.3502875566482544,
+      "logits/rejected": -0.8036888241767883,
+      "logps/chosen": -344.90618896484375,
+      "logps/rejected": -345.6969909667969,
+      "loss": 0.953,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.4247171878814697,
+      "rewards/margins": 0.532504141330719,
+      "rewards/rejected": -1.957221269607544,
+      "rewards/safe_rewards": -1.399427890777588,
+      "rewards/unsafe_rewards": -1.4500062465667725,
+      "step": 1030
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.415463684552728e-07,
+      "logits/chosen": -1.279679536819458,
+      "logits/rejected": -0.875691294670105,
+      "logps/chosen": -329.5700378417969,
+      "logps/rejected": -347.35162353515625,
+      "loss": 1.0239,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.3259227275848389,
+      "rewards/margins": 0.47411665320396423,
+      "rewards/rejected": -1.800039529800415,
+      "rewards/safe_rewards": -1.3383002281188965,
+      "rewards/unsafe_rewards": -1.3135454654693604,
+      "step": 1040
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3685346575222807e-07,
+      "logits/chosen": -1.4535144567489624,
+      "logits/rejected": -0.9614871740341187,
+      "logps/chosen": -325.8673400878906,
+      "logps/rejected": -333.7716064453125,
+      "loss": 0.9704,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.082326889038086,
+      "rewards/margins": 0.5055097341537476,
+      "rewards/rejected": -1.5878366231918335,
+      "rewards/safe_rewards": -1.087576150894165,
+      "rewards/unsafe_rewards": -1.0770776271820068,
+      "step": 1050
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.321652042001919e-07,
+      "logits/chosen": -1.425490140914917,
+      "logits/rejected": -0.9585882425308228,
+      "logps/chosen": -332.23651123046875,
+      "logps/rejected": -367.16656494140625,
+      "loss": 0.9341,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.210457682609558,
+      "rewards/margins": 0.5372633934020996,
+      "rewards/rejected": -1.7477210760116577,
+      "rewards/safe_rewards": -1.155447244644165,
+      "rewards/unsafe_rewards": -1.265467882156372,
+      "step": 1060
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2748323890684662e-07,
+      "logits/chosen": -1.5478570461273193,
+      "logits/rejected": -0.9782267808914185,
+      "logps/chosen": -313.54412841796875,
+      "logps/rejected": -330.4336853027344,
+      "loss": 0.9344,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0107817649841309,
+      "rewards/margins": 0.6438864469528198,
+      "rewards/rejected": -1.6546680927276611,
+      "rewards/safe_rewards": -1.0349432229995728,
+      "rewards/unsafe_rewards": -0.986620306968689,
+      "step": 1070
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2280922275709213e-07,
+      "logits/chosen": -1.4732444286346436,
+      "logits/rejected": -1.1423754692077637,
+      "logps/chosen": -317.08258056640625,
+      "logps/rejected": -328.4356689453125,
+      "loss": 0.982,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.0598576068878174,
+      "rewards/margins": 0.41645368933677673,
+      "rewards/rejected": -1.4763113260269165,
+      "rewards/safe_rewards": -1.0165767669677734,
+      "rewards/unsafe_rewards": -1.1031386852264404,
+      "step": 1080
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1814480582952375e-07,
+      "logits/chosen": -1.489725112915039,
+      "logits/rejected": -1.1142539978027344,
+      "logps/chosen": -309.04681396484375,
+      "logps/rejected": -333.715576171875,
+      "loss": 0.9727,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.0693882703781128,
+      "rewards/margins": 0.4686933159828186,
+      "rewards/rejected": -1.5380815267562866,
+      "rewards/safe_rewards": -0.9820979237556458,
+      "rewards/unsafe_rewards": -1.156678557395935,
+      "step": 1090
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1349163481390187e-07,
+      "logits/chosen": -1.421409010887146,
+      "logits/rejected": -1.031862497329712,
+      "logps/chosen": -316.99139404296875,
+      "logps/rejected": -340.88909912109375,
+      "loss": 0.9693,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1647942066192627,
+      "rewards/margins": 0.4819185137748718,
+      "rewards/rejected": -1.6467128992080688,
+      "rewards/safe_rewards": -1.0639079809188843,
+      "rewards/unsafe_rewards": -1.2656804323196411,
+      "step": 1100
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0885135242981647e-07,
+      "logits/chosen": -1.4362539052963257,
+      "logits/rejected": -1.005382776260376,
+      "logps/chosen": -351.58978271484375,
+      "logps/rejected": -323.1505126953125,
+      "loss": 0.9265,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1940038204193115,
+      "rewards/margins": 0.4320971369743347,
+      "rewards/rejected": -1.6261011362075806,
+      "rewards/safe_rewards": -1.2753981351852417,
+      "rewards/unsafe_rewards": -1.112609624862671,
+      "step": 1110
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0422559684675494e-07,
+      "logits/chosen": -1.4388701915740967,
+      "logits/rejected": -0.9763511419296265,
+      "logps/chosen": -328.2054748535156,
+      "logps/rejected": -331.94427490234375,
+      "loss": 0.9247,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.1099811792373657,
+      "rewards/margins": 0.4838770031929016,
+      "rewards/rejected": -1.5938583612442017,
+      "rewards/safe_rewards": -1.143110752105713,
+      "rewards/unsafe_rewards": -1.076851725578308,
+      "step": 1120
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9961600110577457e-07,
+      "logits/chosen": -1.3994672298431396,
+      "logits/rejected": -0.9330530166625977,
+      "logps/chosen": -320.69989013671875,
+      "logps/rejected": -341.1868591308594,
+      "loss": 1.007,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.1178644895553589,
+      "rewards/margins": 0.4091947674751282,
+      "rewards/rejected": -1.5270591974258423,
+      "rewards/safe_rewards": -1.0592812299728394,
+      "rewards/unsafe_rewards": -1.1764475107192993,
+      "step": 1130
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.950241925429867e-07,
+      "logits/chosen": -1.5059046745300293,
+      "logits/rejected": -0.9375900030136108,
+      "logps/chosen": -301.2913513183594,
+      "logps/rejected": -323.166259765625,
+      "loss": 0.9384,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.0121370553970337,
+      "rewards/margins": 0.6072208285331726,
+      "rewards/rejected": -1.6193578243255615,
+      "rewards/safe_rewards": -1.0190235376358032,
+      "rewards/unsafe_rewards": -1.0052505731582642,
+      "step": 1140
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9045179221505495e-07,
+      "logits/chosen": -1.4665344953536987,
+      "logits/rejected": -1.1525509357452393,
+      "logps/chosen": -336.7375183105469,
+      "logps/rejected": -342.58258056640625,
+      "loss": 0.9456,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0744739770889282,
+      "rewards/margins": 0.4735342860221863,
+      "rewards/rejected": -1.5480082035064697,
+      "rewards/safe_rewards": -1.0436086654663086,
+      "rewards/unsafe_rewards": -1.1053390502929688,
+      "step": 1150
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8590041432690893e-07,
+      "logits/chosen": -1.3896484375,
+      "logits/rejected": -1.1142855882644653,
+      "logps/chosen": -297.66619873046875,
+      "logps/rejected": -315.8662109375,
+      "loss": 0.977,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.100203037261963,
+      "rewards/margins": 0.3822095990180969,
+      "rewards/rejected": -1.4824126958847046,
+      "rewards/safe_rewards": -1.1140623092651367,
+      "rewards/unsafe_rewards": -1.0863438844680786,
+      "step": 1160
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.813716656618788e-07,
+      "logits/chosen": -1.331933856010437,
+      "logits/rejected": -1.0040611028671265,
+      "logps/chosen": -300.43267822265625,
+      "logps/rejected": -321.0027770996094,
+      "loss": 0.969,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.1213122606277466,
+      "rewards/margins": 0.4405437111854553,
+      "rewards/rejected": -1.5618560314178467,
+      "rewards/safe_rewards": -1.0390920639038086,
+      "rewards/unsafe_rewards": -1.2035324573516846,
+      "step": 1170
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7686714501444788e-07,
+      "logits/chosen": -1.4456322193145752,
+      "logits/rejected": -0.8397674560546875,
+      "logps/chosen": -326.2867736816406,
+      "logps/rejected": -333.5230407714844,
+      "loss": 0.9325,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.1393133401870728,
+      "rewards/margins": 0.5787724256515503,
+      "rewards/rejected": -1.7180858850479126,
+      "rewards/safe_rewards": -1.2189667224884033,
+      "rewards/unsafe_rewards": -1.0596599578857422,
+      "step": 1180
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7238844262582768e-07,
+      "logits/chosen": -1.363268494606018,
+      "logits/rejected": -1.1216974258422852,
+      "logps/chosen": -321.5309143066406,
+      "logps/rejected": -357.5130920410156,
+      "loss": 0.9391,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.1455228328704834,
+      "rewards/margins": 0.5131853222846985,
+      "rewards/rejected": -1.6587082147598267,
+      "rewards/safe_rewards": -1.0672725439071655,
+      "rewards/unsafe_rewards": -1.2237731218338013,
+      "step": 1190
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.679371396225504e-07,
+      "logits/chosen": -1.4268032312393188,
+      "logits/rejected": -0.9499413371086121,
+      "logps/chosen": -310.05328369140625,
+      "logps/rejected": -351.35272216796875,
+      "loss": 0.9212,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.089523434638977,
+      "rewards/margins": 0.6114920973777771,
+      "rewards/rejected": -1.7010157108306885,
+      "rewards/safe_rewards": -1.027521014213562,
+      "rewards/unsafe_rewards": -1.151525855064392,
+      "step": 1200
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6351480745828096e-07,
+      "logits/chosen": -1.4248908758163452,
+      "logits/rejected": -1.0554434061050415,
+      "logps/chosen": -324.76751708984375,
+      "logps/rejected": -340.4091491699219,
+      "loss": 0.8506,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.2382574081420898,
+      "rewards/margins": 0.46283411979675293,
+      "rewards/rejected": -1.7010915279388428,
+      "rewards/safe_rewards": -1.2276763916015625,
+      "rewards/unsafe_rewards": -1.2488384246826172,
+      "step": 1210
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5912300735904248e-07,
+      "logits/chosen": -1.5351839065551758,
+      "logits/rejected": -1.176792860031128,
+      "logps/chosen": -334.9280090332031,
+      "logps/rejected": -334.9038391113281,
+      "loss": 0.9582,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1213009357452393,
+      "rewards/margins": 0.4566107392311096,
+      "rewards/rejected": -1.577911615371704,
+      "rewards/safe_rewards": -1.2159931659698486,
+      "rewards/unsafe_rewards": -1.0266087055206299,
+      "step": 1220
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5476328977205395e-07,
+      "logits/chosen": -1.5172470808029175,
+      "logits/rejected": -1.0838744640350342,
+      "logps/chosen": -318.88702392578125,
+      "logps/rejected": -331.92626953125,
+      "loss": 0.929,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.1251935958862305,
+      "rewards/margins": 0.5502282381057739,
+      "rewards/rejected": -1.675421953201294,
+      "rewards/safe_rewards": -1.2570269107818604,
+      "rewards/unsafe_rewards": -0.9933602213859558,
+      "step": 1230
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5043719381837112e-07,
+      "logits/chosen": -1.4699004888534546,
+      "logits/rejected": -1.1323144435882568,
+      "logps/chosen": -332.270751953125,
+      "logps/rejected": -346.4047546386719,
+      "loss": 0.9436,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1165038347244263,
+      "rewards/margins": 0.47613364458084106,
+      "rewards/rejected": -1.5926374197006226,
+      "rewards/safe_rewards": -1.0986835956573486,
+      "rewards/unsafe_rewards": -1.1343239545822144,
+      "step": 1240
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.461462467495284e-07,
+      "logits/chosen": -1.445765495300293,
+      "logits/rejected": -1.0371205806732178,
+      "logps/chosen": -287.15216064453125,
+      "logps/rejected": -333.86676025390625,
+      "loss": 0.8531,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.092584490776062,
+      "rewards/margins": 0.5968576669692993,
+      "rewards/rejected": -1.6894422769546509,
+      "rewards/safe_rewards": -1.1179049015045166,
+      "rewards/unsafe_rewards": -1.067264199256897,
+      "step": 1250
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4189196340836865e-07,
+      "logits/chosen": -1.6602089405059814,
+      "logits/rejected": -1.2219452857971191,
+      "logps/chosen": -297.80426025390625,
+      "logps/rejected": -314.51458740234375,
+      "loss": 0.9095,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9903711080551147,
+      "rewards/margins": 0.4589962363243103,
+      "rewards/rejected": -1.4493674039840698,
+      "rewards/safe_rewards": -1.0244104862213135,
+      "rewards/unsafe_rewards": -0.9563320279121399,
+      "step": 1260
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3767584569425561e-07,
+      "logits/chosen": -1.6483261585235596,
+      "logits/rejected": -1.1660915613174438,
+      "logps/chosen": -309.3149719238281,
+      "logps/rejected": -320.50726318359375,
+      "loss": 0.9112,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0096858739852905,
+      "rewards/margins": 0.4956924319267273,
+      "rewards/rejected": -1.5053783655166626,
+      "rewards/safe_rewards": -1.045543909072876,
+      "rewards/unsafe_rewards": -0.973828136920929,
+      "step": 1270
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.334993820328541e-07,
+      "logits/chosen": -1.501082420349121,
+      "logits/rejected": -1.1109731197357178,
+      "logps/chosen": -289.6914978027344,
+      "logps/rejected": -326.8876953125,
+      "loss": 0.9211,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1317590475082397,
+      "rewards/margins": 0.6352017521858215,
+      "rewards/rejected": -1.766960859298706,
+      "rewards/safe_rewards": -1.0952441692352295,
+      "rewards/unsafe_rewards": -1.1682740449905396,
+      "step": 1280
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.2936404685066852e-07,
+      "logits/chosen": -1.487445592880249,
+      "logits/rejected": -1.1660425662994385,
+      "logps/chosen": -333.1927185058594,
+      "logps/rejected": -355.2337341308594,
+      "loss": 0.9879,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1514599323272705,
+      "rewards/margins": 0.4698604643344879,
+      "rewards/rejected": -1.621320366859436,
+      "rewards/safe_rewards": -1.2052063941955566,
+      "rewards/unsafe_rewards": -1.0977137088775635,
+      "step": 1290
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.252713000545221e-07,
+      "logits/chosen": -1.6853973865509033,
+      "logits/rejected": -1.2987568378448486,
+      "logps/chosen": -322.5224914550781,
+      "logps/rejected": -331.0067138671875,
+      "loss": 0.8714,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.0373764038085938,
+      "rewards/margins": 0.5552427768707275,
+      "rewards/rejected": -1.5926191806793213,
+      "rewards/safe_rewards": -1.0258121490478516,
+      "rewards/unsafe_rewards": -1.0489407777786255,
+      "step": 1300
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2122258651616304e-07,
+      "logits/chosen": -1.6230665445327759,
+      "logits/rejected": -1.1880443096160889,
+      "logps/chosen": -313.47772216796875,
+      "logps/rejected": -304.3802185058594,
+      "loss": 0.9486,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.0324283838272095,
+      "rewards/margins": 0.39181455969810486,
+      "rewards/rejected": -1.4242427349090576,
+      "rewards/safe_rewards": -0.9644128680229187,
+      "rewards/unsafe_rewards": -1.1004436016082764,
+      "step": 1310
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1721933556217792e-07,
+      "logits/chosen": -1.5601403713226318,
+      "logits/rejected": -1.2365220785140991,
+      "logps/chosen": -307.1771240234375,
+      "logps/rejected": -328.2700500488281,
+      "loss": 0.9875,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.066193699836731,
+      "rewards/margins": 0.4565289616584778,
+      "rewards/rejected": -1.5227227210998535,
+      "rewards/safe_rewards": -1.0890748500823975,
+      "rewards/unsafe_rewards": -1.0433127880096436,
+      "step": 1320
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1326296046939333e-07,
+      "logits/chosen": -1.5020397901535034,
+      "logits/rejected": -1.146071195602417,
+      "logps/chosen": -292.02313232421875,
+      "logps/rejected": -311.25604248046875,
+      "loss": 0.926,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.971126914024353,
+      "rewards/margins": 0.559014081954956,
+      "rewards/rejected": -1.5301411151885986,
+      "rewards/safe_rewards": -0.9235084652900696,
+      "rewards/unsafe_rewards": -1.0187455415725708,
+      "step": 1330
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0935485796594351e-07,
+      "logits/chosen": -1.5235176086425781,
+      "logits/rejected": -1.0568602085113525,
+      "logps/chosen": -333.9339904785156,
+      "logps/rejected": -333.97576904296875,
+      "loss": 1.032,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.0942230224609375,
+      "rewards/margins": 0.5436626672744751,
+      "rewards/rejected": -1.6378856897354126,
+      "rewards/safe_rewards": -1.079145073890686,
+      "rewards/unsafe_rewards": -1.109300971031189,
+      "step": 1340
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0549640773818028e-07,
+      "logits/chosen": -1.411492943763733,
+      "logits/rejected": -1.1914324760437012,
+      "logps/chosen": -315.5474853515625,
+      "logps/rejected": -322.1836242675781,
+      "loss": 0.9732,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1521949768066406,
+      "rewards/margins": 0.41504979133605957,
+      "rewards/rejected": -1.5672447681427002,
+      "rewards/safe_rewards": -1.2113215923309326,
+      "rewards/unsafe_rewards": -1.0930684804916382,
+      "step": 1350
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0168897194359921e-07,
+      "logits/chosen": -1.5103179216384888,
+      "logits/rejected": -1.1291395425796509,
+      "logps/chosen": -344.5137634277344,
+      "logps/rejected": -345.2103576660156,
+      "loss": 0.9403,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.1670544147491455,
+      "rewards/margins": 0.4278257489204407,
+      "rewards/rejected": -1.5948803424835205,
+      "rewards/safe_rewards": -1.1103397607803345,
+      "rewards/unsafe_rewards": -1.2237694263458252,
+      "step": 1360
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.793389472995392e-08,
+      "logits/chosen": -1.437276840209961,
+      "logits/rejected": -0.9090574383735657,
+      "logps/chosen": -319.93829345703125,
+      "logps/rejected": -318.78802490234375,
+      "loss": 0.8305,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.0438860654830933,
+      "rewards/margins": 0.630165696144104,
+      "rewards/rejected": -1.6740516424179077,
+      "rewards/safe_rewards": -0.9974796175956726,
+      "rewards/unsafe_rewards": -1.0902923345565796,
+      "step": 1370
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.423250176072874e-08,
+      "logits/chosen": -1.4429516792297363,
+      "logits/rejected": -1.0162016153335571,
+      "logps/chosen": -314.0827331542969,
+      "logps/rejected": -311.3514404296875,
+      "loss": 1.0966,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.2516918182373047,
+      "rewards/margins": 0.3869698643684387,
+      "rewards/rejected": -1.6386617422103882,
+      "rewards/safe_rewards": -1.2116239070892334,
+      "rewards/unsafe_rewards": -1.291759967803955,
+      "step": 1380
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.058609974713654e-08,
+      "logits/chosen": -1.5069031715393066,
+      "logits/rejected": -1.0585293769836426,
+      "logps/chosen": -312.8193054199219,
+      "logps/rejected": -344.32769775390625,
+      "loss": 0.8858,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.0806872844696045,
+      "rewards/margins": 0.6248958110809326,
+      "rewards/rejected": -1.7055833339691162,
+      "rewards/safe_rewards": -1.065792441368103,
+      "rewards/unsafe_rewards": -1.0955822467803955,
+      "step": 1390
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.699597598680753e-08,
+      "logits/chosen": -1.4285691976547241,
+      "logits/rejected": -1.0217626094818115,
+      "logps/chosen": -298.48516845703125,
+      "logps/rejected": -319.5094909667969,
+      "loss": 0.8598,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0639463663101196,
+      "rewards/margins": 0.5066736340522766,
+      "rewards/rejected": -1.570619821548462,
+      "rewards/safe_rewards": -1.049989938735962,
+      "rewards/unsafe_rewards": -1.0779026746749878,
+      "step": 1400
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.346339790933166e-08,
+      "logits/chosen": -1.4796028137207031,
+      "logits/rejected": -1.0254989862442017,
+      "logps/chosen": -303.11322021484375,
+      "logps/rejected": -319.5587463378906,
+      "loss": 0.9735,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.1025015115737915,
+      "rewards/margins": 0.5202707052230835,
+      "rewards/rejected": -1.622771978378296,
+      "rewards/safe_rewards": -1.048610806465149,
+      "rewards/unsafe_rewards": -1.1563920974731445,
+      "step": 1410
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 7.998961262881506e-08,
+      "logits/chosen": -1.4263174533843994,
+      "logits/rejected": -0.9260984659194946,
+      "logps/chosen": -325.7618408203125,
+      "logps/rejected": -318.91070556640625,
+      "loss": 0.9355,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.0862057209014893,
+      "rewards/margins": 0.51823890209198,
+      "rewards/rejected": -1.6044447422027588,
+      "rewards/safe_rewards": -1.1601320505142212,
+      "rewards/unsafe_rewards": -1.0122793912887573,
+      "step": 1420
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.657584650360846e-08,
+      "logits/chosen": -1.2710342407226562,
+      "logits/rejected": -0.989566445350647,
+      "logps/chosen": -300.971923828125,
+      "logps/rejected": -315.53387451171875,
+      "loss": 0.9896,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.1461577415466309,
+      "rewards/margins": 0.5026684999465942,
+      "rewards/rejected": -1.648826241493225,
+      "rewards/safe_rewards": -1.1745736598968506,
+      "rewards/unsafe_rewards": -1.117741584777832,
+      "step": 1430
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.322330470336313e-08,
+      "logits/chosen": -1.38018798828125,
+      "logits/rejected": -0.90345698595047,
+      "logps/chosen": -322.70196533203125,
+      "logps/rejected": -353.0235595703125,
+      "loss": 0.9274,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.1377968788146973,
+      "rewards/margins": 0.5669258236885071,
+      "rewards/rejected": -1.7047227621078491,
+      "rewards/safe_rewards": -1.0534660816192627,
+      "rewards/unsafe_rewards": -1.2221280336380005,
+      "step": 1440
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.993317078356709e-08,
+      "logits/chosen": -1.4155539274215698,
+      "logits/rejected": -1.210967779159546,
+      "logps/chosen": -331.1578674316406,
+      "logps/rejected": -324.1207580566406,
+      "loss": 0.9583,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.2102378606796265,
+      "rewards/margins": 0.2986965477466583,
+      "rewards/rejected": -1.5089343786239624,
+      "rewards/safe_rewards": -1.2407358884811401,
+      "rewards/unsafe_rewards": -1.1797398328781128,
+      "step": 1450
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.67066062677118e-08,
+      "logits/chosen": -1.5069057941436768,
+      "logits/rejected": -1.0829797983169556,
+      "logps/chosen": -308.49664306640625,
+      "logps/rejected": -310.1492004394531,
+      "loss": 1.0037,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.0759823322296143,
+      "rewards/margins": 0.45751920342445374,
+      "rewards/rejected": -1.5335016250610352,
+      "rewards/safe_rewards": -1.0966331958770752,
+      "rewards/unsafe_rewards": -1.0553314685821533,
+      "step": 1460
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.354475023723685e-08,
+      "logits/chosen": -1.4374122619628906,
+      "logits/rejected": -1.0555397272109985,
+      "logps/chosen": -348.89227294921875,
+      "logps/rejected": -351.38665771484375,
+      "loss": 0.949,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.1484794616699219,
+      "rewards/margins": 0.5996941924095154,
+      "rewards/rejected": -1.748173475265503,
+      "rewards/safe_rewards": -1.1115808486938477,
+      "rewards/unsafe_rewards": -1.1853783130645752,
+      "step": 1470
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.044871892939746e-08,
+      "logits/chosen": -1.5760042667388916,
+      "logits/rejected": -1.1585383415222168,
+      "logps/chosen": -320.93597412109375,
+      "logps/rejected": -338.1377868652344,
+      "loss": 0.9439,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0907033681869507,
+      "rewards/margins": 0.4960748255252838,
+      "rewards/rejected": -1.5867780447006226,
+      "rewards/safe_rewards": -1.1278786659240723,
+      "rewards/unsafe_rewards": -1.0535279512405396,
+      "step": 1480
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.741960534319676e-08,
+      "logits/chosen": -1.5081236362457275,
+      "logits/rejected": -1.237660527229309,
+      "logps/chosen": -281.8060607910156,
+      "logps/rejected": -306.55889892578125,
+      "loss": 0.9089,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.0866477489471436,
+      "rewards/margins": 0.4203459322452545,
+      "rewards/rejected": -1.5069936513900757,
+      "rewards/safe_rewards": -1.198955774307251,
+      "rewards/unsafe_rewards": -0.9743399620056152,
+      "step": 1490
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.44584788535217e-08,
+      "logits/chosen": -1.5474069118499756,
+      "logits/rejected": -1.1630122661590576,
+      "logps/chosen": -322.1282653808594,
+      "logps/rejected": -332.53656005859375,
+      "loss": 0.8619,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0482981204986572,
+      "rewards/margins": 0.5498597621917725,
+      "rewards/rejected": -1.5981578826904297,
+      "rewards/safe_rewards": -1.0035431385040283,
+      "rewards/unsafe_rewards": -1.0930533409118652,
+      "step": 1500
+    },
+    {
+      "epoch": 0.81,
+      "eval_logits/chosen": -1.146567940711975,
+      "eval_logits/rejected": -0.7005062699317932,
+      "eval_logps/chosen": -221.80184936523438,
+      "eval_logps/rejected": -210.18832397460938,
+      "eval_loss": 0.34287312626838684,
+      "eval_rewards/accuracies": 0.7268877029418945,
+      "eval_rewards/chosen": -0.913629412651062,
+      "eval_rewards/margins": 0.2635413706302643,
+      "eval_rewards/rejected": -1.1771708726882935,
+      "eval_rewards/safe_rewards": -0.9047471284866333,
+      "eval_rewards/unsafe_rewards": -0.9192255139350891,
+      "eval_runtime": 1122.8444,
+      "eval_samples_per_second": 29.429,
+      "eval_steps_per_second": 0.92,
+      "step": 1500
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.156638483361933e-08,
+      "logits/chosen": -1.612953782081604,
+      "logits/rejected": -1.2291548252105713,
+      "logps/chosen": -316.44293212890625,
+      "logps/rejected": -337.2187194824219,
+      "loss": 0.9167,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.9855987429618835,
+      "rewards/margins": 0.5536761283874512,
+      "rewards/rejected": -1.5392746925354004,
+      "rewards/safe_rewards": -1.0115060806274414,
+      "rewards/unsafe_rewards": -0.9596911668777466,
+      "step": 1510
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.8744344286046236e-08,
+      "logits/chosen": -1.4979829788208008,
+      "logits/rejected": -1.1606972217559814,
+      "logps/chosen": -323.786865234375,
+      "logps/rejected": -326.4385681152344,
+      "loss": 0.9593,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0706676244735718,
+      "rewards/margins": 0.37281331419944763,
+      "rewards/rejected": -1.4434809684753418,
+      "rewards/safe_rewards": -1.150662899017334,
+      "rewards/unsafe_rewards": -0.99067223072052,
+      "step": 1520
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.599335348222169e-08,
+      "logits/chosen": -1.5021213293075562,
+      "logits/rejected": -1.2472387552261353,
+      "logps/chosen": -329.1572265625,
+      "logps/rejected": -364.0357971191406,
+      "loss": 0.9118,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0721365213394165,
+      "rewards/margins": 0.5561688542366028,
+      "rewards/rejected": -1.628305435180664,
+      "rewards/safe_rewards": -1.036094307899475,
+      "rewards/unsafe_rewards": -1.108178734779358,
+      "step": 1530
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.331438361071163e-08,
+      "logits/chosen": -1.5342875719070435,
+      "logits/rejected": -1.3547062873840332,
+      "logps/chosen": -334.36212158203125,
+      "logps/rejected": -347.2046813964844,
+      "loss": 0.9704,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9460422396659851,
+      "rewards/margins": 0.44854864478111267,
+      "rewards/rejected": -1.394590973854065,
+      "rewards/safe_rewards": -0.965340256690979,
+      "rewards/unsafe_rewards": -0.9267444610595703,
+      "step": 1540
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.0708380434367864e-08,
+      "logits/chosen": -1.5614886283874512,
+      "logits/rejected": -1.1849619150161743,
+      "logps/chosen": -300.528564453125,
+      "logps/rejected": -326.10638427734375,
+      "loss": 0.8839,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0258691310882568,
+      "rewards/margins": 0.4937085211277008,
+      "rewards/rejected": -1.5195776224136353,
+      "rewards/safe_rewards": -1.0579198598861694,
+      "rewards/unsafe_rewards": -0.993818461894989,
+      "step": 1550
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.817626395644305e-08,
+      "logits/chosen": -1.5818434953689575,
+      "logits/rejected": -1.210106611251831,
+      "logps/chosen": -297.70025634765625,
+      "logps/rejected": -307.71710205078125,
+      "loss": 1.0046,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9973915219306946,
+      "rewards/margins": 0.3978816568851471,
+      "rewards/rejected": -1.3952730894088745,
+      "rewards/safe_rewards": -0.9759462475776672,
+      "rewards/unsafe_rewards": -1.0188367366790771,
+      "step": 1560
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.571892809580013e-08,
+      "logits/chosen": -1.5090782642364502,
+      "logits/rejected": -1.1929179430007935,
+      "logps/chosen": -307.1462097167969,
+      "logps/rejected": -320.7843017578125,
+      "loss": 0.9616,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1602782011032104,
+      "rewards/margins": 0.36262303590774536,
+      "rewards/rejected": -1.5229012966156006,
+      "rewards/safe_rewards": -1.113433837890625,
+      "rewards/unsafe_rewards": -1.207122564315796,
+      "step": 1570
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.333724037132976e-08,
+      "logits/chosen": -1.5555639266967773,
+      "logits/rejected": -1.2410228252410889,
+      "logps/chosen": -306.10662841796875,
+      "logps/rejected": -334.8534851074219,
+      "loss": 0.96,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.9998570680618286,
+      "rewards/margins": 0.5056936740875244,
+      "rewards/rejected": -1.5055506229400635,
+      "rewards/safe_rewards": -0.9567073583602905,
+      "rewards/unsafe_rewards": -1.0430065393447876,
+      "step": 1580
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.1032041595688506e-08,
+      "logits/chosen": -1.4647419452667236,
+      "logits/rejected": -1.0212126970291138,
+      "logps/chosen": -309.408935546875,
+      "logps/rejected": -335.38299560546875,
+      "loss": 0.9033,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1254334449768066,
+      "rewards/margins": 0.5138076543807983,
+      "rewards/rejected": -1.6392412185668945,
+      "rewards/safe_rewards": -1.1220200061798096,
+      "rewards/unsafe_rewards": -1.1288467645645142,
+      "step": 1590
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.880414557846453e-08,
+      "logits/chosen": -1.4411920309066772,
+      "logits/rejected": -1.2204091548919678,
+      "logps/chosen": -293.62689208984375,
+      "logps/rejected": -315.4649353027344,
+      "loss": 0.8851,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0861139297485352,
+      "rewards/margins": 0.4907267689704895,
+      "rewards/rejected": -1.5768407583236694,
+      "rewards/safe_rewards": -1.0654656887054443,
+      "rewards/unsafe_rewards": -1.106762170791626,
+      "step": 1600
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.6654338838876662e-08,
+      "logits/chosen": -1.585949420928955,
+      "logits/rejected": -1.0787384510040283,
+      "logps/chosen": -322.25885009765625,
+      "logps/rejected": -315.6914978027344,
+      "loss": 0.8857,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0791314840316772,
+      "rewards/margins": 0.5700170397758484,
+      "rewards/rejected": -1.6491485834121704,
+      "rewards/safe_rewards": -1.1312898397445679,
+      "rewards/unsafe_rewards": -1.0269731283187866,
+      "step": 1610
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.4583380328107805e-08,
+      "logits/chosen": -1.5202205181121826,
+      "logits/rejected": -1.1094920635223389,
+      "logps/chosen": -331.93377685546875,
+      "logps/rejected": -334.93719482421875,
+      "loss": 0.9394,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0624914169311523,
+      "rewards/margins": 0.5675247311592102,
+      "rewards/rejected": -1.6300160884857178,
+      "rewards/safe_rewards": -1.0210199356079102,
+      "rewards/unsafe_rewards": -1.1039628982543945,
+      "step": 1620
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.259200116137039e-08,
+      "logits/chosen": -1.4817931652069092,
+      "logits/rejected": -1.1653249263763428,
+      "logps/chosen": -334.85223388671875,
+      "logps/rejected": -357.4493713378906,
+      "loss": 0.9782,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.1782487630844116,
+      "rewards/margins": 0.4701511263847351,
+      "rewards/rejected": -1.6483999490737915,
+      "rewards/safe_rewards": -1.1807775497436523,
+      "rewards/unsafe_rewards": -1.175719976425171,
+      "step": 1630
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.068090435979958e-08,
+      "logits/chosen": -1.4055829048156738,
+      "logits/rejected": -1.156343698501587,
+      "logps/chosen": -306.995361328125,
+      "logps/rejected": -317.0059509277344,
+      "loss": 0.9645,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.0118372440338135,
+      "rewards/margins": 0.44742053747177124,
+      "rewards/rejected": -1.45925772190094,
+      "rewards/safe_rewards": -1.0128872394561768,
+      "rewards/unsafe_rewards": -1.010787010192871,
+      "step": 1640
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.8850764602263423e-08,
+      "logits/chosen": -1.4388693571090698,
+      "logits/rejected": -1.0342535972595215,
+      "logps/chosen": -311.6680603027344,
+      "logps/rejected": -348.0157165527344,
+      "loss": 0.9259,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1476449966430664,
+      "rewards/margins": 0.49749964475631714,
+      "rewards/rejected": -1.6451447010040283,
+      "rewards/safe_rewards": -1.1766583919525146,
+      "rewards/unsafe_rewards": -1.1186316013336182,
+      "step": 1650
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.710222798718028e-08,
+      "logits/chosen": -1.498641014099121,
+      "logits/rejected": -1.1954280138015747,
+      "logps/chosen": -323.10626220703125,
+      "logps/rejected": -355.72705078125,
+      "loss": 0.8871,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.1219761371612549,
+      "rewards/margins": 0.4705938696861267,
+      "rewards/rejected": -1.5925698280334473,
+      "rewards/safe_rewards": -1.170627474784851,
+      "rewards/unsafe_rewards": -1.07332444190979,
+      "step": 1660
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.5435911804424356e-08,
+      "logits/chosen": -1.545506477355957,
+      "logits/rejected": -1.2171175479888916,
+      "logps/chosen": -331.9698181152344,
+      "logps/rejected": -340.3002014160156,
+      "loss": 0.9938,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.0450648069381714,
+      "rewards/margins": 0.47697582840919495,
+      "rewards/rejected": -1.522040605545044,
+      "rewards/safe_rewards": -1.095984697341919,
+      "rewards/unsafe_rewards": -0.9941450357437134,
+      "step": 1670
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.3852404317403199e-08,
+      "logits/chosen": -1.41542649269104,
+      "logits/rejected": -1.1583986282348633,
+      "logps/chosen": -297.29962158203125,
+      "logps/rejected": -336.06988525390625,
+      "loss": 0.9583,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1032707691192627,
+      "rewards/margins": 0.3784652352333069,
+      "rewards/rejected": -1.4817359447479248,
+      "rewards/safe_rewards": -1.15559983253479,
+      "rewards/unsafe_rewards": -1.0509414672851562,
+      "step": 1680
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.235226455538113e-08,
+      "logits/chosen": -1.4842908382415771,
+      "logits/rejected": -1.2061156034469604,
+      "logps/chosen": -318.2637939453125,
+      "logps/rejected": -340.8109436035156,
+      "loss": 1.0008,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1303695440292358,
+      "rewards/margins": 0.47945070266723633,
+      "rewards/rejected": -1.6098201274871826,
+      "rewards/safe_rewards": -1.1323390007019043,
+      "rewards/unsafe_rewards": -1.128400206565857,
+      "step": 1690
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.0936022116124321e-08,
+      "logits/chosen": -1.4996792078018188,
+      "logits/rejected": -1.116720199584961,
+      "logps/chosen": -307.4630432128906,
+      "logps/rejected": -331.1900634765625,
+      "loss": 0.866,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.0742332935333252,
+      "rewards/margins": 0.5489377975463867,
+      "rewards/rejected": -1.6231712102890015,
+      "rewards/safe_rewards": -1.107750654220581,
+      "rewards/unsafe_rewards": -1.0407161712646484,
+      "step": 1700
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.60417697893534e-09,
+      "logits/chosen": -1.482126235961914,
+      "logits/rejected": -1.1564598083496094,
+      "logps/chosen": -312.1965026855469,
+      "logps/rejected": -340.93817138671875,
+      "loss": 0.9667,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.156433343887329,
+      "rewards/margins": 0.43530288338661194,
+      "rewards/rejected": -1.5917361974716187,
+      "rewards/safe_rewards": -1.0536446571350098,
+      "rewards/unsafe_rewards": -1.2592222690582275,
+      "step": 1710
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.357199328144576e-09,
+      "logits/chosen": -1.4593350887298584,
+      "logits/rejected": -1.200596570968628,
+      "logps/chosen": -356.2051086425781,
+      "logps/rejected": -369.406494140625,
+      "loss": 0.8611,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.1415997743606567,
+      "rewards/margins": 0.49805140495300293,
+      "rewards/rejected": -1.6396510601043701,
+      "rewards/safe_rewards": -1.2040703296661377,
+      "rewards/unsafe_rewards": -1.0791290998458862,
+      "step": 1720
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.1955293871198144e-09,
+      "logits/chosen": -1.350987195968628,
+      "logits/rejected": -1.1763075590133667,
+      "logps/chosen": -296.27191162109375,
+      "logps/rejected": -321.37518310546875,
+      "loss": 0.9819,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.2195558547973633,
+      "rewards/margins": 0.39501625299453735,
+      "rewards/rejected": -1.6145721673965454,
+      "rewards/safe_rewards": -1.2167112827301025,
+      "rewards/unsafe_rewards": -1.2224003076553345,
+      "step": 1730
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.119577262853254e-09,
+      "logits/chosen": -1.441007375717163,
+      "logits/rejected": -1.0252482891082764,
+      "logps/chosen": -299.6694641113281,
+      "logps/rejected": -312.8736267089844,
+      "loss": 0.9801,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.121921420097351,
+      "rewards/margins": 0.5479141473770142,
+      "rewards/rejected": -1.6698356866836548,
+      "rewards/safe_rewards": -1.0443205833435059,
+      "rewards/unsafe_rewards": -1.1995223760604858,
+      "step": 1740
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.129722801180542e-09,
+      "logits/chosen": -1.4107353687286377,
+      "logits/rejected": -1.0875647068023682,
+      "logps/chosen": -322.1299743652344,
+      "logps/rejected": -341.2066650390625,
+      "loss": 0.8399,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.189591407775879,
+      "rewards/margins": 0.5180362462997437,
+      "rewards/rejected": -1.707627534866333,
+      "rewards/safe_rewards": -1.3024308681488037,
+      "rewards/unsafe_rewards": -1.076751708984375,
+      "step": 1750
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.226315452682816e-09,
+      "logits/chosen": -1.4723705053329468,
+      "logits/rejected": -1.1786158084869385,
+      "logps/chosen": -305.52362060546875,
+      "logps/rejected": -326.5906677246094,
+      "loss": 0.9485,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.046170711517334,
+      "rewards/margins": 0.4863569736480713,
+      "rewards/rejected": -1.5325279235839844,
+      "rewards/safe_rewards": -1.0766370296478271,
+      "rewards/unsafe_rewards": -1.0157043933868408,
+      "step": 1760
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.4096741493194193e-09,
+      "logits/chosen": -1.527930498123169,
+      "logits/rejected": -1.2494192123413086,
+      "logps/chosen": -315.3348693847656,
+      "logps/rejected": -332.2015686035156,
+      "loss": 1.0269,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1415612697601318,
+      "rewards/margins": 0.39452338218688965,
+      "rewards/rejected": -1.5360848903656006,
+      "rewards/safe_rewards": -1.0917105674743652,
+      "rewards/unsafe_rewards": -1.1914122104644775,
+      "step": 1770
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.6800871918346846e-09,
+      "logits/chosen": -1.5815150737762451,
+      "logits/rejected": -1.1552437543869019,
+      "logps/chosen": -320.330078125,
+      "logps/rejected": -340.8857727050781,
+      "loss": 0.9364,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.0657531023025513,
+      "rewards/margins": 0.5418880581855774,
+      "rewards/rejected": -1.6076412200927734,
+      "rewards/safe_rewards": -1.1473314762115479,
+      "rewards/unsafe_rewards": -0.9841750264167786,
+      "step": 1780
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0378121479783796e-09,
+      "logits/chosen": -1.4162019491195679,
+      "logits/rejected": -1.0255894660949707,
+      "logps/chosen": -313.9689025878906,
+      "logps/rejected": -334.1581115722656,
+      "loss": 0.9987,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.1681249141693115,
+      "rewards/margins": 0.49787864089012146,
+      "rewards/rejected": -1.6660035848617554,
+      "rewards/safe_rewards": -1.1383296251296997,
+      "rewards/unsafe_rewards": -1.1979202032089233,
+      "step": 1790
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4830757615760247e-09,
+      "logits/chosen": -1.437466025352478,
+      "logits/rejected": -1.0774166584014893,
+      "logps/chosen": -325.25677490234375,
+      "logps/rejected": -333.42791748046875,
+      "loss": 0.9478,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.0991178750991821,
+      "rewards/margins": 0.44227179884910583,
+      "rewards/rejected": -1.5413895845413208,
+      "rewards/safe_rewards": -1.1737608909606934,
+      "rewards/unsafe_rewards": -1.024474859237671,
+      "step": 1800
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.0160738724809548e-09,
+      "logits/chosen": -1.496584177017212,
+      "logits/rejected": -1.0213630199432373,
+      "logps/chosen": -305.14263916015625,
+      "logps/rejected": -338.046875,
+      "loss": 0.881,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.1096470355987549,
+      "rewards/margins": 0.5430228114128113,
+      "rewards/rejected": -1.6526696681976318,
+      "rewards/safe_rewards": -1.1323860883712769,
+      "rewards/unsafe_rewards": -1.0869077444076538,
+      "step": 1810
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.369713474366212e-10,
+      "logits/chosen": -1.460850477218628,
+      "logits/rejected": -1.1146998405456543,
+      "logps/chosen": -342.700439453125,
+      "logps/rejected": -370.11865234375,
+      "loss": 0.8416,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.1760565042495728,
+      "rewards/margins": 0.5496792793273926,
+      "rewards/rejected": -1.7257359027862549,
+      "rewards/safe_rewards": -1.209657907485962,
+      "rewards/unsafe_rewards": -1.1424554586410522,
+      "step": 1820
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.459020218731512e-10,
+      "logits/chosen": -1.4401605129241943,
+      "logits/rejected": -1.135258674621582,
+      "logps/chosen": -297.53607177734375,
+      "logps/rejected": -319.9522399902344,
+      "loss": 0.872,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.0829004049301147,
+      "rewards/margins": 0.5533518195152283,
+      "rewards/rejected": -1.6362521648406982,
+      "rewards/safe_rewards": -1.041105031967163,
+      "rewards/unsafe_rewards": -1.124695897102356,
+      "step": 1830
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.429686526593088e-10,
+      "logits/chosen": -1.4089921712875366,
+      "logits/rejected": -1.140413522720337,
+      "logps/chosen": -320.8370666503906,
+      "logps/rejected": -343.36822509765625,
+      "loss": 1.0174,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.1735649108886719,
+      "rewards/margins": 0.45700669288635254,
+      "rewards/rejected": -1.6305716037750244,
+      "rewards/safe_rewards": -1.203802227973938,
+      "rewards/unsafe_rewards": -1.1433275938034058,
+      "step": 1840
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.824288182584622e-11,
+      "logits/chosen": -1.5680155754089355,
+      "logits/rejected": -1.1377493143081665,
+      "logps/chosen": -327.45550537109375,
+      "logps/rejected": -339.25115966796875,
+      "loss": 0.8677,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.1065986156463623,
+      "rewards/margins": 0.5338612198829651,
+      "rewards/rejected": -1.6404597759246826,
+      "rewards/safe_rewards": -1.171757459640503,
+      "rewards/unsafe_rewards": -1.0414397716522217,
+      "step": 1850
+    },
+    {
+      "epoch": 1.0,
+      "step": 1858,
+      "total_flos": 0.0,
+      "train_loss": 1.018996798697021,
+      "train_runtime": 22449.6551,
+      "train_samples_per_second": 2.649,
+      "train_steps_per_second": 0.083
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1858,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}