diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,14499 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 100,
+  "global_step": 8706,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00034458993797381116,
+      "grad_norm": 1.5267285108566284,
+      "learning_rate": 3.4443168771526976e-11,
+      "logits/chosen": -3.024087429046631,
+      "logits/rejected": -2.988196611404419,
+      "logps/chosen": -47.308799743652344,
+      "logps/rejected": -44.131954193115234,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0034458993797381117,
+      "grad_norm": 1.607421636581421,
+      "learning_rate": 3.444316877152698e-10,
+      "logits/chosen": -3.0891504287719727,
+      "logits/rejected": -3.071469306945801,
+      "logps/chosen": -51.67715072631836,
+      "logps/rejected": -51.65058135986328,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.3993055522441864,
+      "rewards/chosen": 6.8785157054662704e-06,
+      "rewards/margins": 2.2403159164241515e-05,
+      "rewards/rejected": -1.5524632544838823e-05,
+      "step": 10
+    },
+    {
+      "epoch": 0.006891798759476223,
+      "grad_norm": 1.465844750404358,
+      "learning_rate": 6.888633754305396e-10,
+      "logits/chosen": -3.0982673168182373,
+      "logits/rejected": -3.0706088542938232,
+      "logps/chosen": -56.0171012878418,
+      "logps/rejected": -54.545310974121094,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": 0.000170388346305117,
+      "rewards/margins": 1.0887908501899801e-05,
+      "rewards/rejected": 0.00015950041415635496,
+      "step": 20
+    },
+    {
+      "epoch": 0.010337698139214336,
+      "grad_norm": 1.8542964458465576,
+      "learning_rate": 1.0332950631458093e-09,
+      "logits/chosen": -3.1275365352630615,
+      "logits/rejected": -3.104400396347046,
+      "logps/chosen": -55.355979919433594,
+      "logps/rejected": -52.83089065551758,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4468750059604645,
+      "rewards/chosen": -4.464233643375337e-05,
+      "rewards/margins": -0.00014433874457608908,
+      "rewards/rejected": 9.969641541829333e-05,
+      "step": 30
+    },
+    {
+      "epoch": 0.013783597518952447,
+      "grad_norm": 1.5623725652694702,
+      "learning_rate": 1.3777267508610793e-09,
+      "logits/chosen": -3.1054978370666504,
+      "logits/rejected": -3.0787899494171143,
+      "logps/chosen": -56.42761993408203,
+      "logps/rejected": -53.678627014160156,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 5.21727379236836e-05,
+      "rewards/margins": 0.00023771081760060042,
+      "rewards/rejected": -0.00018553808331489563,
+      "step": 40
+    },
+    {
+      "epoch": 0.01722949689869056,
+      "grad_norm": 1.7456684112548828,
+      "learning_rate": 1.722158438576349e-09,
+      "logits/chosen": -3.081786632537842,
+      "logits/rejected": -3.0434436798095703,
+      "logps/chosen": -54.70775604248047,
+      "logps/rejected": -51.2340087890625,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 2.2655992779618828e-06,
+      "rewards/margins": -1.6194762793020345e-05,
+      "rewards/rejected": 1.8460374121787027e-05,
+      "step": 50
+    },
+    {
+      "epoch": 0.02067539627842867,
+      "grad_norm": 1.7984153032302856,
+      "learning_rate": 2.0665901262916186e-09,
+      "logits/chosen": -3.0937764644622803,
+      "logits/rejected": -3.074101686477661,
+      "logps/chosen": -54.38935470581055,
+      "logps/rejected": -53.957908630371094,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -9.771531767910346e-05,
+      "rewards/margins": -3.43201172654517e-05,
+      "rewards/rejected": -6.339520041365176e-05,
+      "step": 60
+    },
+    {
+      "epoch": 0.024121295658166782,
+      "grad_norm": 1.7539132833480835,
+      "learning_rate": 2.4110218140068887e-09,
+      "logits/chosen": -3.112018585205078,
+      "logits/rejected": -3.0958092212677,
+      "logps/chosen": -54.6849479675293,
+      "logps/rejected": -53.765899658203125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": 4.858314787270501e-05,
+      "rewards/margins": 0.00011905960127478465,
+      "rewards/rejected": -7.047644612612203e-05,
+      "step": 70
+    },
+    {
+      "epoch": 0.027567195037904894,
+      "grad_norm": 1.569575309753418,
+      "learning_rate": 2.7554535017221585e-09,
+      "logits/chosen": -3.058257818222046,
+      "logits/rejected": -3.038975238800049,
+      "logps/chosen": -53.057411193847656,
+      "logps/rejected": -53.54570388793945,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.503125011920929,
+      "rewards/chosen": -3.9076985558494925e-06,
+      "rewards/margins": -7.156379433581606e-05,
+      "rewards/rejected": 6.765608850400895e-05,
+      "step": 80
+    },
+    {
+      "epoch": 0.031013094417643005,
+      "grad_norm": 1.7667663097381592,
+      "learning_rate": 3.0998851894374283e-09,
+      "logits/chosen": -3.06956148147583,
+      "logits/rejected": -3.050424575805664,
+      "logps/chosen": -56.337074279785156,
+      "logps/rejected": -52.55491256713867,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 8.060954132815823e-05,
+      "rewards/margins": 0.00011852341413032264,
+      "rewards/rejected": -3.7913861888227984e-05,
+      "step": 90
+    },
+    {
+      "epoch": 0.03445899379738112,
+      "grad_norm": 1.753562331199646,
+      "learning_rate": 3.444316877152698e-09,
+      "logits/chosen": -3.0917346477508545,
+      "logits/rejected": -3.0621657371520996,
+      "logps/chosen": -56.142295837402344,
+      "logps/rejected": -53.62836456298828,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.00010823029151652008,
+      "rewards/margins": 9.168633550871164e-05,
+      "rewards/rejected": 1.6543965102755465e-05,
+      "step": 100
+    },
+    {
+      "epoch": 0.03445899379738112,
+      "eval_logits/chosen": -3.1633360385894775,
+      "eval_logits/rejected": -3.1576740741729736,
+      "eval_logps/chosen": -58.702396392822266,
+      "eval_logps/rejected": -63.167236328125,
+      "eval_loss": 0.6931648254394531,
+      "eval_rewards/accuracies": 0.49302974343299866,
+      "eval_rewards/chosen": 9.50019748415798e-05,
+      "eval_rewards/margins": -3.390433266758919e-05,
+      "eval_rewards/rejected": 0.00012890630750916898,
+      "eval_runtime": 382.8893,
+      "eval_samples_per_second": 11.241,
+      "eval_steps_per_second": 1.405,
+      "step": 100
+    },
+    {
+      "epoch": 0.03790489317711923,
+      "grad_norm": 1.6134201288223267,
+      "learning_rate": 3.788748564867967e-09,
+      "logits/chosen": -3.032473087310791,
+      "logits/rejected": -3.012482166290283,
+      "logps/chosen": -52.719703674316406,
+      "logps/rejected": -54.301780700683594,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5218750238418579,
+      "rewards/chosen": 0.00012056533887516707,
+      "rewards/margins": 0.00010502724762773141,
+      "rewards/rejected": 1.553810216137208e-05,
+      "step": 110
+    },
+    {
+      "epoch": 0.04135079255685734,
+      "grad_norm": 1.5367040634155273,
+      "learning_rate": 4.133180252583237e-09,
+      "logits/chosen": -3.037926197052002,
+      "logits/rejected": -3.0075340270996094,
+      "logps/chosen": -52.4307975769043,
+      "logps/rejected": -51.051971435546875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.503125011920929,
+      "rewards/chosen": 1.1001296115864534e-05,
+      "rewards/margins": 3.366530654602684e-05,
+      "rewards/rejected": -2.2664000425720587e-05,
+      "step": 120
+    },
+    {
+      "epoch": 0.044796691936595454,
+      "grad_norm": 1.7146553993225098,
+      "learning_rate": 4.4776119402985065e-09,
+      "logits/chosen": -3.1061904430389404,
+      "logits/rejected": -3.0898799896240234,
+      "logps/chosen": -53.57172393798828,
+      "logps/rejected": -53.8604736328125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 2.9032340535195544e-05,
+      "rewards/margins": 5.941182462265715e-05,
+      "rewards/rejected": -3.0379474992514588e-05,
+      "step": 130
+    },
+    {
+      "epoch": 0.048242591316333565,
+      "grad_norm": 1.858219027519226,
+      "learning_rate": 4.8220436280137775e-09,
+      "logits/chosen": -3.078770160675049,
+      "logits/rejected": -3.052644968032837,
+      "logps/chosen": -55.374237060546875,
+      "logps/rejected": -53.8642578125,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -7.880809425842017e-05,
+      "rewards/margins": -3.4557091566966847e-05,
+      "rewards/rejected": -4.425101724336855e-05,
+      "step": 140
+    },
+    {
+      "epoch": 0.051688490696071676,
+      "grad_norm": 1.6268450021743774,
+      "learning_rate": 5.166475315729047e-09,
+      "logits/chosen": -3.0249457359313965,
+      "logits/rejected": -3.013507127761841,
+      "logps/chosen": -54.10808181762695,
+      "logps/rejected": -54.107826232910156,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 1.3208697964728344e-05,
+      "rewards/margins": -3.807519533438608e-05,
+      "rewards/rejected": 5.128389602759853e-05,
+      "step": 150
+    },
+    {
+      "epoch": 0.05513439007580979,
+      "grad_norm": 1.6796091794967651,
+      "learning_rate": 5.510907003444317e-09,
+      "logits/chosen": -3.0442471504211426,
+      "logits/rejected": -3.028390645980835,
+      "logps/chosen": -54.01305389404297,
+      "logps/rejected": -51.273658752441406,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": 6.503812528535491e-06,
+      "rewards/margins": 6.548188684973866e-05,
+      "rewards/rejected": -5.897807568544522e-05,
+      "step": 160
+    },
+    {
+      "epoch": 0.0585802894555479,
+      "grad_norm": 1.6665263175964355,
+      "learning_rate": 5.855338691159586e-09,
+      "logits/chosen": -3.04443621635437,
+      "logits/rejected": -3.0218753814697266,
+      "logps/chosen": -53.77790069580078,
+      "logps/rejected": -52.0765266418457,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 1.9446970327408053e-05,
+      "rewards/margins": 5.564164530369453e-05,
+      "rewards/rejected": -3.619468043325469e-05,
+      "step": 170
+    },
+    {
+      "epoch": 0.06202618883528601,
+      "grad_norm": 1.6640942096710205,
+      "learning_rate": 6.1997703788748565e-09,
+      "logits/chosen": -3.0530142784118652,
+      "logits/rejected": -3.021761894226074,
+      "logps/chosen": -55.40903854370117,
+      "logps/rejected": -52.03125762939453,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4781250059604645,
+      "rewards/chosen": -0.00010554668551776558,
+      "rewards/margins": -0.00014609869685955346,
+      "rewards/rejected": 4.0552047721575946e-05,
+      "step": 180
+    },
+    {
+      "epoch": 0.06547208821502412,
+      "grad_norm": 1.525570034980774,
+      "learning_rate": 6.544202066590126e-09,
+      "logits/chosen": -3.1539559364318848,
+      "logits/rejected": -3.126908540725708,
+      "logps/chosen": -52.959022521972656,
+      "logps/rejected": -51.77215576171875,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.00014587806072086096,
+      "rewards/margins": -8.230133244069293e-05,
+      "rewards/rejected": -6.357675010804087e-05,
+      "step": 190
+    },
+    {
+      "epoch": 0.06891798759476224,
+      "grad_norm": 1.7355997562408447,
+      "learning_rate": 6.888633754305396e-09,
+      "logits/chosen": -3.0855188369750977,
+      "logits/rejected": -3.0648550987243652,
+      "logps/chosen": -54.34490203857422,
+      "logps/rejected": -53.99126052856445,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4906249940395355,
+      "rewards/chosen": 7.46866426197812e-05,
+      "rewards/margins": 0.00011284511856501922,
+      "rewards/rejected": -3.815847958321683e-05,
+      "step": 200
+    },
+    {
+      "epoch": 0.06891798759476224,
+      "eval_logits/chosen": -3.1633803844451904,
+      "eval_logits/rejected": -3.1577279567718506,
+      "eval_logps/chosen": -58.706581115722656,
+      "eval_logps/rejected": -63.1661262512207,
+      "eval_loss": 0.6931912302970886,
+      "eval_rewards/accuracies": 0.4888475835323334,
+      "eval_rewards/chosen": 5.3132996981730685e-05,
+      "eval_rewards/margins": -8.680764585733414e-05,
+      "eval_rewards/rejected": 0.00013994066102895886,
+      "eval_runtime": 383.4906,
+      "eval_samples_per_second": 11.223,
+      "eval_steps_per_second": 1.403,
+      "step": 200
+    },
+    {
+      "epoch": 0.07236388697450034,
+      "grad_norm": 1.7263901233673096,
+      "learning_rate": 7.2330654420206654e-09,
+      "logits/chosen": -3.0673437118530273,
+      "logits/rejected": -3.061281681060791,
+      "logps/chosen": -52.456809997558594,
+      "logps/rejected": -54.72541427612305,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.0764348189695738e-05,
+      "rewards/margins": 3.4362215956207365e-05,
+      "rewards/rejected": -4.512655868893489e-05,
+      "step": 210
+    },
+    {
+      "epoch": 0.07580978635423846,
+      "grad_norm": 1.7311609983444214,
+      "learning_rate": 7.577497129735934e-09,
+      "logits/chosen": -3.110809803009033,
+      "logits/rejected": -3.0862531661987305,
+      "logps/chosen": -53.693023681640625,
+      "logps/rejected": -53.7116584777832,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": 4.901724241790362e-05,
+      "rewards/margins": 0.00011947364691877738,
+      "rewards/rejected": -7.045637903502211e-05,
+      "step": 220
+    },
+    {
+      "epoch": 0.07925568573397657,
+      "grad_norm": 1.6573609113693237,
+      "learning_rate": 7.921928817451203e-09,
+      "logits/chosen": -3.040351390838623,
+      "logits/rejected": -3.014275550842285,
+      "logps/chosen": -56.13459396362305,
+      "logps/rejected": -53.79411697387695,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.534375011920929,
+      "rewards/chosen": 0.00013501883950084448,
+      "rewards/margins": 0.0001316798006882891,
+      "rewards/rejected": 3.33903994942375e-06,
+      "step": 230
+    },
+    {
+      "epoch": 0.08270158511371468,
+      "grad_norm": 1.8083306550979614,
+      "learning_rate": 8.266360505166474e-09,
+      "logits/chosen": -3.0464279651641846,
+      "logits/rejected": -3.0279934406280518,
+      "logps/chosen": -53.75872802734375,
+      "logps/rejected": -55.18073654174805,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5093749761581421,
+      "rewards/chosen": -6.338553794194013e-05,
+      "rewards/margins": -5.129817145643756e-05,
+      "rewards/rejected": -1.2087375580449589e-05,
+      "step": 240
+    },
+    {
+      "epoch": 0.08614748449345279,
+      "grad_norm": 1.5646331310272217,
+      "learning_rate": 8.610792192881744e-09,
+      "logits/chosen": -2.9863524436950684,
+      "logits/rejected": -2.9471404552459717,
+      "logps/chosen": -57.77240753173828,
+      "logps/rejected": -51.473472595214844,
+      "loss": 0.693,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": 0.0001028064070851542,
+      "rewards/margins": 0.00020027598657179624,
+      "rewards/rejected": -9.746955765876919e-05,
+      "step": 250
+    },
+    {
+      "epoch": 0.08959338387319091,
+      "grad_norm": 1.5849988460540771,
+      "learning_rate": 8.955223880597013e-09,
+      "logits/chosen": -3.0340161323547363,
+      "logits/rejected": -3.0105412006378174,
+      "logps/chosen": -57.33642578125,
+      "logps/rejected": -51.81958770751953,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": 4.5101998694008216e-05,
+      "rewards/margins": 0.00011759666813304648,
+      "rewards/rejected": -7.249465124914423e-05,
+      "step": 260
+    },
+    {
+      "epoch": 0.09303928325292901,
+      "grad_norm": 1.5757571458816528,
+      "learning_rate": 9.299655568312282e-09,
+      "logits/chosen": -3.0476603507995605,
+      "logits/rejected": -3.0184288024902344,
+      "logps/chosen": -54.269081115722656,
+      "logps/rejected": -52.059471130371094,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 6.857989501440898e-05,
+      "rewards/margins": 0.0001449831761419773,
+      "rewards/rejected": -7.640327385161072e-05,
+      "step": 270
+    },
+    {
+      "epoch": 0.09648518263266713,
+      "grad_norm": 1.7623542547225952,
+      "learning_rate": 9.644087256027555e-09,
+      "logits/chosen": -3.087009906768799,
+      "logits/rejected": -3.0740387439727783,
+      "logps/chosen": -52.84673309326172,
+      "logps/rejected": -53.47339630126953,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -5.518016405403614e-05,
+      "rewards/margins": 1.3806826245854609e-05,
+      "rewards/rejected": -6.898697756696492e-05,
+      "step": 280
+    },
+    {
+      "epoch": 0.09993108201240523,
+      "grad_norm": 1.4707379341125488,
+      "learning_rate": 9.988518943742824e-09,
+      "logits/chosen": -3.0488078594207764,
+      "logits/rejected": -3.0416297912597656,
+      "logps/chosen": -51.2463493347168,
+      "logps/rejected": -53.53486251831055,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.46562498807907104,
+      "rewards/chosen": -0.00011236511636525393,
+      "rewards/margins": -8.335065649589524e-05,
+      "rewards/rejected": -2.9014472602284513e-05,
+      "step": 290
+    },
+    {
+      "epoch": 0.10337698139214335,
+      "grad_norm": 1.7845849990844727,
+      "learning_rate": 1.0332950631458094e-08,
+      "logits/chosen": -3.0379014015197754,
+      "logits/rejected": -3.0148253440856934,
+      "logps/chosen": -54.379295349121094,
+      "logps/rejected": -55.731353759765625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 2.041672451014165e-05,
+      "rewards/margins": 6.121327169239521e-05,
+      "rewards/rejected": -4.079652717337012e-05,
+      "step": 300
+    },
+    {
+      "epoch": 0.10337698139214335,
+      "eval_logits/chosen": -3.163435935974121,
+      "eval_logits/rejected": -3.1578028202056885,
+      "eval_logps/chosen": -58.70713806152344,
+      "eval_logps/rejected": -63.16929626464844,
+      "eval_loss": 0.6931781768798828,
+      "eval_rewards/accuracies": 0.49326208233833313,
+      "eval_rewards/chosen": 4.754788096761331e-05,
+      "eval_rewards/margins": -6.0738573665730655e-05,
+      "eval_rewards/rejected": 0.00010828646190930158,
+      "eval_runtime": 383.4663,
+      "eval_samples_per_second": 11.224,
+      "eval_steps_per_second": 1.403,
+      "step": 300
+    },
+    {
+      "epoch": 0.10682288077188146,
+      "grad_norm": 1.6843689680099487,
+      "learning_rate": 1.0677382319173363e-08,
+      "logits/chosen": -3.0700087547302246,
+      "logits/rejected": -3.0554189682006836,
+      "logps/chosen": -53.794639587402344,
+      "logps/rejected": -53.267974853515625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -2.8351705623208545e-05,
+      "rewards/margins": 1.1917157280549873e-05,
+      "rewards/rejected": -4.026884198538028e-05,
+      "step": 310
+    },
+    {
+      "epoch": 0.11026878015161957,
+      "grad_norm": 1.8084667921066284,
+      "learning_rate": 1.1021814006888634e-08,
+      "logits/chosen": -3.116976261138916,
+      "logits/rejected": -3.098066568374634,
+      "logps/chosen": -53.38834762573242,
+      "logps/rejected": -52.58855438232422,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.484375,
+      "rewards/chosen": 1.0202318208030192e-06,
+      "rewards/margins": -1.5139745300984941e-05,
+      "rewards/rejected": 1.6159981896635145e-05,
+      "step": 320
+    },
+    {
+      "epoch": 0.11371467953135768,
+      "grad_norm": 1.6020495891571045,
+      "learning_rate": 1.1366245694603903e-08,
+      "logits/chosen": -3.058793306350708,
+      "logits/rejected": -3.056426525115967,
+      "logps/chosen": -53.05699920654297,
+      "logps/rejected": -53.55686569213867,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": -5.535797754419036e-05,
+      "rewards/margins": 8.838313078740612e-05,
+      "rewards/rejected": -0.00014374109741766006,
+      "step": 330
+    },
+    {
+      "epoch": 0.1171605789110958,
+      "grad_norm": 1.6713683605194092,
+      "learning_rate": 1.1710677382319173e-08,
+      "logits/chosen": -3.002620220184326,
+      "logits/rejected": -2.987659215927124,
+      "logps/chosen": -53.47795867919922,
+      "logps/rejected": -54.195335388183594,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -8.549810445401818e-05,
+      "rewards/margins": -6.205539830261841e-05,
+      "rewards/rejected": -2.3442707970389165e-05,
+      "step": 340
+    },
+    {
+      "epoch": 0.1206064782908339,
+      "grad_norm": 1.600785493850708,
+      "learning_rate": 1.2055109070034442e-08,
+      "logits/chosen": -3.106245517730713,
+      "logits/rejected": -3.077061176300049,
+      "logps/chosen": -57.41899490356445,
+      "logps/rejected": -51.7630500793457,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.00013625768769998103,
+      "rewards/margins": -0.00011847783753182739,
+      "rewards/rejected": -1.7779831978259608e-05,
+      "step": 350
+    },
+    {
+      "epoch": 0.12405237767057202,
+      "grad_norm": 1.6922481060028076,
+      "learning_rate": 1.2399540757749713e-08,
+      "logits/chosen": -3.044334888458252,
+      "logits/rejected": -3.027862310409546,
+      "logps/chosen": -53.98234176635742,
+      "logps/rejected": -54.54735565185547,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 5.651239007420372e-06,
+      "rewards/margins": 5.852256208527251e-07,
+      "rewards/rejected": 5.0660046326811425e-06,
+      "step": 360
+    },
+    {
+      "epoch": 0.12749827705031014,
+      "grad_norm": 1.6789193153381348,
+      "learning_rate": 1.2743972445464982e-08,
+      "logits/chosen": -3.0830962657928467,
+      "logits/rejected": -3.057596445083618,
+      "logps/chosen": -55.6155891418457,
+      "logps/rejected": -53.12651824951172,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5406249761581421,
+      "rewards/chosen": 8.971284842118621e-05,
+      "rewards/margins": 0.0002537999243941158,
+      "rewards/rejected": -0.0001640870759729296,
+      "step": 370
+    },
+    {
+      "epoch": 0.13094417643004824,
+      "grad_norm": 1.7992178201675415,
+      "learning_rate": 1.3088404133180252e-08,
+      "logits/chosen": -3.123126268386841,
+      "logits/rejected": -3.0895767211914062,
+      "logps/chosen": -55.21710968017578,
+      "logps/rejected": -51.77630615234375,
+      "loss": 0.693,
+      "rewards/accuracies": 0.528124988079071,
+      "rewards/chosen": 7.373380503850058e-05,
+      "rewards/margins": 0.0002209977974416688,
+      "rewards/rejected": -0.0001472639705752954,
+      "step": 380
+    },
+    {
+      "epoch": 0.13439007580978635,
+      "grad_norm": 1.6913421154022217,
+      "learning_rate": 1.3432835820895521e-08,
+      "logits/chosen": -3.0962982177734375,
+      "logits/rejected": -3.0691208839416504,
+      "logps/chosen": -53.0391845703125,
+      "logps/rejected": -51.70380783081055,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -9.544294880470261e-05,
+      "rewards/margins": 4.369412636151537e-05,
+      "rewards/rejected": -0.00013913707516621798,
+      "step": 390
+    },
+    {
+      "epoch": 0.13783597518952448,
+      "grad_norm": 1.7395342588424683,
+      "learning_rate": 1.3777267508610792e-08,
+      "logits/chosen": -3.044773817062378,
+      "logits/rejected": -3.0149877071380615,
+      "logps/chosen": -54.4503288269043,
+      "logps/rejected": -53.98418426513672,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 1.7747695892467164e-05,
+      "rewards/margins": 4.0229253500001505e-05,
+      "rewards/rejected": -2.2481553969555534e-05,
+      "step": 400
+    },
+    {
+      "epoch": 0.13783597518952448,
+      "eval_logits/chosen": -3.163205623626709,
+      "eval_logits/rejected": -3.1575379371643066,
+      "eval_logps/chosen": -58.70612335205078,
+      "eval_logps/rejected": -63.172672271728516,
+      "eval_loss": 0.6931561827659607,
+      "eval_rewards/accuracies": 0.48094794154167175,
+      "eval_rewards/chosen": 5.773279917775653e-05,
+      "eval_rewards/margins": -1.6799720469862223e-05,
+      "eval_rewards/rejected": 7.453252328559756e-05,
+      "eval_runtime": 383.5079,
+      "eval_samples_per_second": 11.223,
+      "eval_steps_per_second": 1.403,
+      "step": 400
+    },
+    {
+      "epoch": 0.14128187456926258,
+      "grad_norm": 1.5212793350219727,
+      "learning_rate": 1.4121699196326061e-08,
+      "logits/chosen": -3.087085008621216,
+      "logits/rejected": -3.060256242752075,
+      "logps/chosen": -54.10284423828125,
+      "logps/rejected": -53.159217834472656,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5093749761581421,
+      "rewards/chosen": -0.00010306591866537929,
+      "rewards/margins": 3.8969657907728106e-05,
+      "rewards/rejected": -0.00014203556929714978,
+      "step": 410
+    },
+    {
+      "epoch": 0.1447277739490007,
+      "grad_norm": 1.7078908681869507,
+      "learning_rate": 1.4466130884041331e-08,
+      "logits/chosen": -3.091862916946411,
+      "logits/rejected": -3.070795774459839,
+      "logps/chosen": -54.34466552734375,
+      "logps/rejected": -51.77064895629883,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -7.322328019654378e-05,
+      "rewards/margins": 4.082272062078118e-05,
+      "rewards/rejected": -0.00011404599354136735,
+      "step": 420
+    },
+    {
+      "epoch": 0.1481736733287388,
+      "grad_norm": 1.7153022289276123,
+      "learning_rate": 1.48105625717566e-08,
+      "logits/chosen": -3.0320191383361816,
+      "logits/rejected": -3.0166242122650146,
+      "logps/chosen": -51.84508514404297,
+      "logps/rejected": -53.62068557739258,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -4.921266372548416e-05,
+      "rewards/margins": -5.284211511025205e-05,
+      "rewards/rejected": 3.6294454730523285e-06,
+      "step": 430
+    },
+    {
+      "epoch": 0.15161957270847692,
+      "grad_norm": 1.5208419561386108,
+      "learning_rate": 1.5154994259471868e-08,
+      "logits/chosen": -3.0755865573883057,
+      "logits/rejected": -3.0622599124908447,
+      "logps/chosen": -51.51741409301758,
+      "logps/rejected": -52.52643966674805,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.6734196833567694e-05,
+      "rewards/margins": 6.7774104536511e-05,
+      "rewards/rejected": -8.450829773209989e-05,
+      "step": 440
+    },
+    {
+      "epoch": 0.15506547208821503,
+      "grad_norm": 1.5616028308868408,
+      "learning_rate": 1.5499425947187137e-08,
+      "logits/chosen": -3.071122646331787,
+      "logits/rejected": -3.047133207321167,
+      "logps/chosen": -56.311279296875,
+      "logps/rejected": -53.27568817138672,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.503125011920929,
+      "rewards/chosen": 6.249133548408281e-06,
+      "rewards/margins": 3.433436359046027e-05,
+      "rewards/rejected": -2.808523095154669e-05,
+      "step": 450
+    },
+    {
+      "epoch": 0.15851137146795313,
+      "grad_norm": 1.6252989768981934,
+      "learning_rate": 1.5843857634902407e-08,
+      "logits/chosen": -3.0730209350585938,
+      "logits/rejected": -3.047236919403076,
+      "logps/chosen": -52.66759490966797,
+      "logps/rejected": -50.75257873535156,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5093749761581421,
+      "rewards/chosen": 7.009728869888932e-05,
+      "rewards/margins": 4.2905041482299566e-05,
+      "rewards/rejected": 2.7192270863451995e-05,
+      "step": 460
+    },
+    {
+      "epoch": 0.16195727084769124,
+      "grad_norm": 1.5376890897750854,
+      "learning_rate": 1.618828932261768e-08,
+      "logits/chosen": -3.1453003883361816,
+      "logits/rejected": -3.1185054779052734,
+      "logps/chosen": -56.32984161376953,
+      "logps/rejected": -54.702659606933594,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": 0.00014750212721992284,
+      "rewards/margins": 0.0002940718550235033,
+      "rewards/rejected": -0.00014656971325166523,
+      "step": 470
+    },
+    {
+      "epoch": 0.16540317022742937,
+      "grad_norm": 1.7167824506759644,
+      "learning_rate": 1.653272101033295e-08,
+      "logits/chosen": -2.912266254425049,
+      "logits/rejected": -2.9052317142486572,
+      "logps/chosen": -53.0239372253418,
+      "logps/rejected": -55.635467529296875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.0001758922589942813,
+      "rewards/margins": 2.3157805117079988e-05,
+      "rewards/rejected": -0.00019905003136955202,
+      "step": 480
+    },
+    {
+      "epoch": 0.16884906960716747,
+      "grad_norm": 1.8802646398544312,
+      "learning_rate": 1.6877152698048218e-08,
+      "logits/chosen": -3.1187596321105957,
+      "logits/rejected": -3.0910351276397705,
+      "logps/chosen": -58.35840606689453,
+      "logps/rejected": -53.641639709472656,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.2699114197166637e-05,
+      "rewards/margins": -2.9918530344730243e-05,
+      "rewards/rejected": 7.219426152005326e-06,
+      "step": 490
+    },
+    {
+      "epoch": 0.17229496898690558,
+      "grad_norm": 1.5846978425979614,
+      "learning_rate": 1.7221584385763487e-08,
+      "logits/chosen": -3.012493133544922,
+      "logits/rejected": -2.9869420528411865,
+      "logps/chosen": -55.7285041809082,
+      "logps/rejected": -52.19916915893555,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -9.770771430339664e-05,
+      "rewards/margins": 8.709765825187787e-05,
+      "rewards/rejected": -0.0001848053652793169,
+      "step": 500
+    },
+    {
+      "epoch": 0.17229496898690558,
+      "eval_logits/chosen": -3.1633849143981934,
+      "eval_logits/rejected": -3.157712459564209,
+      "eval_logps/chosen": -58.692779541015625,
+      "eval_logps/rejected": -63.163307189941406,
+      "eval_loss": 0.6931362748146057,
+      "eval_rewards/accuracies": 0.5097583532333374,
+      "eval_rewards/chosen": 0.0001911700383061543,
+      "eval_rewards/margins": 2.2999165594228543e-05,
+      "eval_rewards/rejected": 0.00016817086725495756,
+      "eval_runtime": 383.5761,
+      "eval_samples_per_second": 11.221,
+      "eval_steps_per_second": 1.403,
+      "step": 500
+    },
+    {
+      "epoch": 0.17574086836664368,
+      "grad_norm": 1.721936583518982,
+      "learning_rate": 1.7566016073478757e-08,
+      "logits/chosen": -3.0158679485321045,
+      "logits/rejected": -3.0033555030822754,
+      "logps/chosen": -55.452674865722656,
+      "logps/rejected": -56.09540557861328,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.00022453709971159697,
+      "rewards/margins": 0.00014329221448861063,
+      "rewards/rejected": -0.0003678292559925467,
+      "step": 510
+    },
+    {
+      "epoch": 0.17918676774638181,
+      "grad_norm": 1.5276626348495483,
+      "learning_rate": 1.7910447761194026e-08,
+      "logits/chosen": -3.1147525310516357,
+      "logits/rejected": -3.0960052013397217,
+      "logps/chosen": -53.568603515625,
+      "logps/rejected": -53.7443733215332,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.00010024486255133525,
+      "rewards/margins": 9.326756844529882e-05,
+      "rewards/rejected": -0.0001935124455485493,
+      "step": 520
+    },
+    {
+      "epoch": 0.18263266712611992,
+      "grad_norm": 1.6583023071289062,
+      "learning_rate": 1.8254879448909295e-08,
+      "logits/chosen": -2.998107433319092,
+      "logits/rejected": -2.970668315887451,
+      "logps/chosen": -56.82636642456055,
+      "logps/rejected": -52.79780960083008,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -4.209352482575923e-05,
+      "rewards/margins": 0.00018835824448615313,
+      "rewards/rejected": -0.00023045176931191236,
+      "step": 530
+    },
+    {
+      "epoch": 0.18607856650585802,
+      "grad_norm": 1.5804274082183838,
+      "learning_rate": 1.8599311136624565e-08,
+      "logits/chosen": -3.1351001262664795,
+      "logits/rejected": -3.1064293384552,
+      "logps/chosen": -55.8760986328125,
+      "logps/rejected": -52.004486083984375,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.00017582179862074554,
+      "rewards/margins": -2.520221096347086e-05,
+      "rewards/rejected": -0.0001506195985712111,
+      "step": 540
+    },
+    {
+      "epoch": 0.18952446588559613,
+      "grad_norm": 1.579467535018921,
+      "learning_rate": 1.894374282433984e-08,
+      "logits/chosen": -3.034126043319702,
+      "logits/rejected": -3.0284879207611084,
+      "logps/chosen": -51.593223571777344,
+      "logps/rejected": -53.631202697753906,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4781250059604645,
+      "rewards/chosen": -0.00023668438370805234,
+      "rewards/margins": -0.0001321500021731481,
+      "rewards/rejected": -0.00010453439608681947,
+      "step": 550
+    },
+    {
+      "epoch": 0.19297036526533426,
+      "grad_norm": 1.6588585376739502,
+      "learning_rate": 1.928817451205511e-08,
+      "logits/chosen": -3.0799546241760254,
+      "logits/rejected": -3.07519268989563,
+      "logps/chosen": -54.6545295715332,
+      "logps/rejected": -55.11370849609375,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4593749940395355,
+      "rewards/chosen": -0.00021095783449709415,
+      "rewards/margins": -5.425519702839665e-05,
+      "rewards/rejected": -0.00015670261927880347,
+      "step": 560
+    },
+    {
+      "epoch": 0.19641626464507236,
+      "grad_norm": 1.748795747756958,
+      "learning_rate": 1.963260619977038e-08,
+      "logits/chosen": -3.1151251792907715,
+      "logits/rejected": -3.0859477519989014,
+      "logps/chosen": -54.47711181640625,
+      "logps/rejected": -53.61101531982422,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.00020242284517735243,
+      "rewards/margins": 2.193655018345453e-05,
+      "rewards/rejected": -0.0002243594208266586,
+      "step": 570
+    },
+    {
+      "epoch": 0.19986216402481047,
+      "grad_norm": 1.532230257987976,
+      "learning_rate": 1.997703788748565e-08,
+      "logits/chosen": -3.0565433502197266,
+      "logits/rejected": -3.0425939559936523,
+      "logps/chosen": -53.6988410949707,
+      "logps/rejected": -54.561500549316406,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -7.838986493879929e-05,
+      "rewards/margins": 0.0003687096177600324,
+      "rewards/rejected": -0.0004470994754228741,
+      "step": 580
+    },
+    {
+      "epoch": 0.2033080634045486,
+      "grad_norm": 1.7507727146148682,
+      "learning_rate": 2.0321469575200918e-08,
+      "logits/chosen": -2.963016986846924,
+      "logits/rejected": -2.9410290718078613,
+      "logps/chosen": -52.462425231933594,
+      "logps/rejected": -52.87834930419922,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.000218748435145244,
+      "rewards/margins": 0.00018759837257675827,
+      "rewards/rejected": -0.0004063468368258327,
+      "step": 590
+    },
+    {
+      "epoch": 0.2067539627842867,
+      "grad_norm": 1.4773313999176025,
+      "learning_rate": 2.0665901262916187e-08,
+      "logits/chosen": -3.0627598762512207,
+      "logits/rejected": -3.037601947784424,
+      "logps/chosen": -55.9506950378418,
+      "logps/rejected": -50.8673210144043,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.528124988079071,
+      "rewards/chosen": -0.00018720950174611062,
+      "rewards/margins": 0.0001278236450161785,
+      "rewards/rejected": -0.0003150331322103739,
+      "step": 600
+    },
+    {
+      "epoch": 0.2067539627842867,
+      "eval_logits/chosen": -3.163013219833374,
+      "eval_logits/rejected": -3.15741229057312,
+      "eval_logps/chosen": -58.692039489746094,
+      "eval_logps/rejected": -63.159584045410156,
+      "eval_loss": 0.6931512355804443,
+      "eval_rewards/accuracies": 0.4937267601490021,
+      "eval_rewards/chosen": 0.00019856891594827175,
+      "eval_rewards/margins": -6.843654318799963e-06,
+      "eval_rewards/rejected": 0.00020541257981676608,
+      "eval_runtime": 383.516,
+      "eval_samples_per_second": 11.222,
+      "eval_steps_per_second": 1.403,
+      "step": 600
+    },
+    {
+      "epoch": 0.2101998621640248,
+      "grad_norm": 1.5543370246887207,
+      "learning_rate": 2.1010332950631457e-08,
+      "logits/chosen": -3.0838775634765625,
+      "logits/rejected": -3.053241491317749,
+      "logps/chosen": -53.58643341064453,
+      "logps/rejected": -52.67365646362305,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5531250238418579,
+      "rewards/chosen": -1.6549254723940976e-05,
+      "rewards/margins": 0.00022414434351958334,
+      "rewards/rejected": -0.00024069359642453492,
+      "step": 610
+    },
+    {
+      "epoch": 0.2136457615437629,
+      "grad_norm": 1.6253598928451538,
+      "learning_rate": 2.1354764638346726e-08,
+      "logits/chosen": -3.090259075164795,
+      "logits/rejected": -3.060715675354004,
+      "logps/chosen": -54.1126594543457,
+      "logps/rejected": -52.56208038330078,
+      "loss": 0.693,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.00018682744121178985,
+      "rewards/margins": 0.0002093940565828234,
+      "rewards/rejected": -0.00039622155600227416,
+      "step": 620
+    },
+    {
+      "epoch": 0.21709166092350105,
+      "grad_norm": 1.5586414337158203,
+      "learning_rate": 2.1699196326062e-08,
+      "logits/chosen": -3.065364122390747,
+      "logits/rejected": -3.031407356262207,
+      "logps/chosen": -53.37366485595703,
+      "logps/rejected": -50.76912307739258,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.00033979618456214666,
+      "rewards/margins": -7.258246569108451e-06,
+      "rewards/rejected": -0.00033253798028454185,
+      "step": 630
+    },
+    {
+      "epoch": 0.22053756030323915,
+      "grad_norm": 1.6641521453857422,
+      "learning_rate": 2.2043628013777268e-08,
+      "logits/chosen": -3.0864169597625732,
+      "logits/rejected": -3.086726665496826,
+      "logps/chosen": -51.543548583984375,
+      "logps/rejected": -57.350486755371094,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.00019868407980538905,
+      "rewards/margins": 0.00021513670799322426,
+      "rewards/rejected": -0.0004138207877986133,
+      "step": 640
+    },
+    {
+      "epoch": 0.22398345968297725,
+      "grad_norm": 1.507374882698059,
+      "learning_rate": 2.2388059701492537e-08,
+      "logits/chosen": -3.062448024749756,
+      "logits/rejected": -3.040160655975342,
+      "logps/chosen": -52.4200439453125,
+      "logps/rejected": -52.10943603515625,
+      "loss": 0.693,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.00012118128506699577,
+      "rewards/margins": 0.00026257545687258244,
+      "rewards/rejected": -0.0003837567346636206,
+      "step": 650
+    },
+    {
+      "epoch": 0.22742935906271536,
+      "grad_norm": 1.7674659490585327,
+      "learning_rate": 2.2732491389207807e-08,
+      "logits/chosen": -3.0894970893859863,
+      "logits/rejected": -3.058967113494873,
+      "logps/chosen": -54.398155212402344,
+      "logps/rejected": -50.13960647583008,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.00017992423090618104,
+      "rewards/margins": 0.0004350285162217915,
+      "rewards/rejected": -0.0006149526452645659,
+      "step": 660
+    },
+    {
+      "epoch": 0.2308752584424535,
+      "grad_norm": 1.6007637977600098,
+      "learning_rate": 2.3076923076923076e-08,
+      "logits/chosen": -3.076066493988037,
+      "logits/rejected": -3.0499377250671387,
+      "logps/chosen": -55.18376541137695,
+      "logps/rejected": -51.55232620239258,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -5.257116936263628e-05,
+      "rewards/margins": 0.00043723214184865355,
+      "rewards/rejected": -0.0004898033803328872,
+      "step": 670
+    },
+    {
+      "epoch": 0.2343211578221916,
+      "grad_norm": 1.632431983947754,
+      "learning_rate": 2.3421354764638345e-08,
+      "logits/chosen": -3.0412890911102295,
+      "logits/rejected": -3.022753953933716,
+      "logps/chosen": -54.66859817504883,
+      "logps/rejected": -54.66973876953125,
+      "loss": 0.693,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.00016700258129276335,
+      "rewards/margins": 0.00023606112517882138,
+      "rewards/rejected": -0.0004030637501273304,
+      "step": 680
+    },
+    {
+      "epoch": 0.2377670572019297,
+      "grad_norm": 1.5880471467971802,
+      "learning_rate": 2.3765786452353615e-08,
+      "logits/chosen": -3.080070972442627,
+      "logits/rejected": -3.061021327972412,
+      "logps/chosen": -53.522193908691406,
+      "logps/rejected": -52.60344696044922,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.00018376897787675261,
+      "rewards/margins": 0.000327289046254009,
+      "rewards/rejected": -0.0005110580241307616,
+      "step": 690
+    },
+    {
+      "epoch": 0.2412129565816678,
+      "grad_norm": 1.5821027755737305,
+      "learning_rate": 2.4110218140068884e-08,
+      "logits/chosen": -3.085906505584717,
+      "logits/rejected": -3.0593483448028564,
+      "logps/chosen": -56.36487579345703,
+      "logps/rejected": -53.1934814453125,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.00013561693776864558,
+      "rewards/margins": 0.0004910177667625248,
+      "rewards/rejected": -0.0006266346899792552,
+      "step": 700
+    },
+    {
+      "epoch": 0.2412129565816678,
+      "eval_logits/chosen": -3.162858724594116,
+      "eval_logits/rejected": -3.157215118408203,
+      "eval_logps/chosen": -58.681705474853516,
+      "eval_logps/rejected": -63.158203125,
+      "eval_loss": 0.6931067109107971,
+      "eval_rewards/accuracies": 0.49047398567199707,
+      "eval_rewards/chosen": 0.00030189091921783984,
+      "eval_rewards/margins": 8.26695904834196e-05,
+      "eval_rewards/rejected": 0.000219221314182505,
+      "eval_runtime": 383.2187,
+      "eval_samples_per_second": 11.231,
+      "eval_steps_per_second": 1.404,
+      "step": 700
+    },
+    {
+      "epoch": 0.24465885596140594,
+      "grad_norm": 1.489160418510437,
+      "learning_rate": 2.4454649827784154e-08,
+      "logits/chosen": -3.019388437271118,
+      "logits/rejected": -3.0163187980651855,
+      "logps/chosen": -52.59061813354492,
+      "logps/rejected": -53.9920768737793,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.000267073221039027,
+      "rewards/margins": 0.0002355240285396576,
+      "rewards/rejected": -0.0005025971913710237,
+      "step": 710
+    },
+    {
+      "epoch": 0.24810475534114404,
+      "grad_norm": 1.5831902027130127,
+      "learning_rate": 2.4799081515499426e-08,
+      "logits/chosen": -3.0615711212158203,
+      "logits/rejected": -3.0400683879852295,
+      "logps/chosen": -52.0587272644043,
+      "logps/rejected": -52.489646911621094,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.0003516732540447265,
+      "rewards/margins": 0.00027333511388860643,
+      "rewards/rejected": -0.0006250083679333329,
+      "step": 720
+    },
+    {
+      "epoch": 0.25155065472088217,
+      "grad_norm": 1.6529788970947266,
+      "learning_rate": 2.5143513203214696e-08,
+      "logits/chosen": -3.0406594276428223,
+      "logits/rejected": -3.005689859390259,
+      "logps/chosen": -53.62286376953125,
+      "logps/rejected": -51.2553825378418,
+      "loss": 0.693,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0003550280525814742,
+      "rewards/margins": 0.0002727129904087633,
+      "rewards/rejected": -0.0006277411011978984,
+      "step": 730
+    },
+    {
+      "epoch": 0.2549965541006203,
+      "grad_norm": 1.4943519830703735,
+      "learning_rate": 2.5487944890929965e-08,
+      "logits/chosen": -3.1910765171051025,
+      "logits/rejected": -3.1548802852630615,
+      "logps/chosen": -54.192893981933594,
+      "logps/rejected": -53.274024963378906,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.0002570011420175433,
+      "rewards/margins": 0.0003994024300482124,
+      "rewards/rejected": -0.0006564036011695862,
+      "step": 740
+    },
+    {
+      "epoch": 0.2584424534803584,
+      "grad_norm": 1.6463762521743774,
+      "learning_rate": 2.5832376578645234e-08,
+      "logits/chosen": -3.060882568359375,
+      "logits/rejected": -3.047121047973633,
+      "logps/chosen": -53.72297286987305,
+      "logps/rejected": -53.23944091796875,
+      "loss": 0.693,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.00036774121690541506,
+      "rewards/margins": 0.0003451154916547239,
+      "rewards/rejected": -0.000712856650352478,
+      "step": 750
+    },
+    {
+      "epoch": 0.2618883528600965,
+      "grad_norm": 1.5722792148590088,
+      "learning_rate": 2.6176808266360504e-08,
+      "logits/chosen": -3.0213165283203125,
+      "logits/rejected": -2.999450206756592,
+      "logps/chosen": -55.4737663269043,
+      "logps/rejected": -52.230079650878906,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.00045472499914467335,
+      "rewards/margins": 0.00022367588826455176,
+      "rewards/rejected": -0.0006784008583053946,
+      "step": 760
+    },
+    {
+      "epoch": 0.2653342522398346,
+      "grad_norm": 1.6122773885726929,
+      "learning_rate": 2.6521239954075773e-08,
+      "logits/chosen": -3.029160737991333,
+      "logits/rejected": -3.004692792892456,
+      "logps/chosen": -53.8378791809082,
+      "logps/rejected": -50.965538024902344,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.000408306805184111,
+      "rewards/margins": 0.00045906248851679265,
+      "rewards/rejected": -0.0008673692937009037,
+      "step": 770
+    },
+    {
+      "epoch": 0.2687801516195727,
+      "grad_norm": 1.6387323141098022,
+      "learning_rate": 2.6865671641791042e-08,
+      "logits/chosen": -3.019998073577881,
+      "logits/rejected": -3.0012447834014893,
+      "logps/chosen": -52.1688117980957,
+      "logps/rejected": -50.70772171020508,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.0006026190822012722,
+      "rewards/margins": 0.0004704033490270376,
+      "rewards/rejected": -0.0010730224894359708,
+      "step": 780
+    },
+    {
+      "epoch": 0.2722260509993108,
+      "grad_norm": 1.7423993349075317,
+      "learning_rate": 2.721010332950631e-08,
+      "logits/chosen": -3.0469167232513428,
+      "logits/rejected": -3.042785167694092,
+      "logps/chosen": -52.47370529174805,
+      "logps/rejected": -55.14872360229492,
+      "loss": 0.693,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.000580360705498606,
+      "rewards/margins": 0.000302892760373652,
+      "rewards/rejected": -0.000883253465872258,
+      "step": 790
+    },
+    {
+      "epoch": 0.27567195037904896,
+      "grad_norm": 1.5481418371200562,
+      "learning_rate": 2.7554535017221584e-08,
+      "logits/chosen": -3.05647611618042,
+      "logits/rejected": -3.029874086380005,
+      "logps/chosen": -53.20207595825195,
+      "logps/rejected": -52.88569259643555,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.0004723466990981251,
+      "rewards/margins": 0.0004238039255142212,
+      "rewards/rejected": -0.0008961505955085158,
+      "step": 800
+    },
+    {
+      "epoch": 0.27567195037904896,
+      "eval_logits/chosen": -3.16223406791687,
+      "eval_logits/rejected": -3.156554937362671,
+      "eval_logps/chosen": -58.67033386230469,
+      "eval_logps/rejected": -63.14847946166992,
+      "eval_loss": 0.6930984258651733,
+      "eval_rewards/accuracies": 0.5236988663673401,
+      "eval_rewards/chosen": 0.0004156152717769146,
+      "eval_rewards/margins": 9.918153955368325e-05,
+      "eval_rewards/rejected": 0.0003164336958434433,
+      "eval_runtime": 383.5282,
+      "eval_samples_per_second": 11.222,
+      "eval_steps_per_second": 1.403,
+      "step": 800
+    },
+    {
+      "epoch": 0.27911784975878706,
+      "grad_norm": 1.651687502861023,
+      "learning_rate": 2.7898966704936854e-08,
+      "logits/chosen": -3.052823066711426,
+      "logits/rejected": -3.029231548309326,
+      "logps/chosen": -53.7202033996582,
+      "logps/rejected": -55.988372802734375,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.0005311970598995686,
+      "rewards/margins": 0.0005335467285476625,
+      "rewards/rejected": -0.001064743846654892,
+      "step": 810
+    },
+    {
+      "epoch": 0.28256374913852517,
+      "grad_norm": 1.730443000793457,
+      "learning_rate": 2.8243398392652123e-08,
+      "logits/chosen": -3.1058716773986816,
+      "logits/rejected": -3.073948383331299,
+      "logps/chosen": -56.261619567871094,
+      "logps/rejected": -51.645240783691406,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.0002955960517283529,
+      "rewards/margins": 0.000810113619081676,
+      "rewards/rejected": -0.0011057096999138594,
+      "step": 820
+    },
+    {
+      "epoch": 0.28600964851826327,
+      "grad_norm": 1.6980032920837402,
+      "learning_rate": 2.8587830080367392e-08,
+      "logits/chosen": -3.114914655685425,
+      "logits/rejected": -3.0774927139282227,
+      "logps/chosen": -56.127952575683594,
+      "logps/rejected": -52.4610595703125,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.0002395716728642583,
+      "rewards/margins": 0.0008646426722407341,
+      "rewards/rejected": -0.0011042144615203142,
+      "step": 830
+    },
+    {
+      "epoch": 0.2894555478980014,
+      "grad_norm": 1.6411075592041016,
+      "learning_rate": 2.8932261768082662e-08,
+      "logits/chosen": -3.099675178527832,
+      "logits/rejected": -3.0755584239959717,
+      "logps/chosen": -55.094947814941406,
+      "logps/rejected": -54.92778778076172,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.0003347202728036791,
+      "rewards/margins": 0.0008746219682507217,
+      "rewards/rejected": -0.0012093421537429094,
+      "step": 840
+    },
+    {
+      "epoch": 0.2929014472777395,
+      "grad_norm": 1.7345929145812988,
+      "learning_rate": 2.927669345579793e-08,
+      "logits/chosen": -3.0120089054107666,
+      "logits/rejected": -3.0066404342651367,
+      "logps/chosen": -52.63999557495117,
+      "logps/rejected": -53.9082145690918,
+      "loss": 0.693,
+      "rewards/accuracies": 0.546875,
+      "rewards/chosen": -0.0008582788286730647,
+      "rewards/margins": 0.00029315095162019134,
+      "rewards/rejected": -0.0011514297220855951,
+      "step": 850
+    },
+    {
+      "epoch": 0.2963473466574776,
+      "grad_norm": 1.7958312034606934,
+      "learning_rate": 2.96211251435132e-08,
+      "logits/chosen": -3.103553295135498,
+      "logits/rejected": -3.0684285163879395,
+      "logps/chosen": -57.461708068847656,
+      "logps/rejected": -52.887550354003906,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.00043874848051927984,
+      "rewards/margins": 0.0007400623289868236,
+      "rewards/rejected": -0.001178810722194612,
+      "step": 860
+    },
+    {
+      "epoch": 0.2997932460372157,
+      "grad_norm": 1.6387503147125244,
+      "learning_rate": 2.996555683122847e-08,
+      "logits/chosen": -3.0778491497039795,
+      "logits/rejected": -3.0640883445739746,
+      "logps/chosen": -53.093963623046875,
+      "logps/rejected": -53.3946533203125,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.0007616410148330033,
+      "rewards/margins": 0.000349641777575016,
+      "rewards/rejected": -0.0011112827342003584,
+      "step": 870
+    },
+    {
+      "epoch": 0.30323914541695385,
+      "grad_norm": 1.612123727798462,
+      "learning_rate": 2.999990232856809e-08,
+      "logits/chosen": -3.0907082557678223,
+      "logits/rejected": -3.0665841102600098,
+      "logps/chosen": -55.66045379638672,
+      "logps/rejected": -54.62714767456055,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.0005586418556049466,
+      "rewards/margins": 0.0008368236012756824,
+      "rewards/rejected": -0.0013954653404653072,
+      "step": 880
+    },
+    {
+      "epoch": 0.30668504479669195,
+      "grad_norm": 1.735127568244934,
+      "learning_rate": 2.999956470055992e-08,
+      "logits/chosen": -3.0690016746520996,
+      "logits/rejected": -3.0514986515045166,
+      "logps/chosen": -54.8330192565918,
+      "logps/rejected": -53.8523063659668,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0008289706893265247,
+      "rewards/margins": 0.0006711410242132843,
+      "rewards/rejected": -0.00150011177174747,
+      "step": 890
+    },
+    {
+      "epoch": 0.31013094417643006,
+      "grad_norm": 1.6623343229293823,
+      "learning_rate": 2.99989859155823e-08,
+      "logits/chosen": -3.1553189754486084,
+      "logits/rejected": -3.132119655609131,
+      "logps/chosen": -51.97255325317383,
+      "logps/rejected": -52.4308967590332,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.00073774199699983,
+      "rewards/margins": 0.0009643202647566795,
+      "rewards/rejected": -0.0017020622035488486,
+      "step": 900
+    },
+    {
+      "epoch": 0.31013094417643006,
+      "eval_logits/chosen": -3.1620285511016846,
+      "eval_logits/rejected": -3.1563503742218018,
+      "eval_logps/chosen": -58.655914306640625,
+      "eval_logps/rejected": -63.13775634765625,
+      "eval_loss": 0.6930801868438721,
+      "eval_rewards/accuracies": 0.5185873508453369,
+      "eval_rewards/chosen": 0.0005597746931016445,
+      "eval_rewards/margins": 0.00013609326560981572,
+      "eval_rewards/rejected": 0.00042368145659565926,
+      "eval_runtime": 382.9837,
+      "eval_samples_per_second": 11.238,
+      "eval_steps_per_second": 1.405,
+      "step": 900
+    },
+    {
+      "epoch": 0.31357684355616816,
+      "grad_norm": 1.5862501859664917,
+      "learning_rate": 2.9998165982940705e-08,
+      "logits/chosen": -3.1258630752563477,
+      "logits/rejected": -3.090562343597412,
+      "logps/chosen": -55.947715759277344,
+      "logps/rejected": -52.314125061035156,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0005809779395349324,
+      "rewards/margins": 0.0006968885427340865,
+      "rewards/rejected": -0.0012778665404766798,
+      "step": 910
+    },
+    {
+      "epoch": 0.31702274293590627,
+      "grad_norm": 1.646518588066101,
+      "learning_rate": 2.999710491581768e-08,
+      "logits/chosen": -3.110973834991455,
+      "logits/rejected": -3.101970672607422,
+      "logps/chosen": -55.26581573486328,
+      "logps/rejected": -55.03619384765625,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.000841100700199604,
+      "rewards/margins": 0.0006255352636799216,
+      "rewards/rejected": -0.0014666361967101693,
+      "step": 920
+    },
+    {
+      "epoch": 0.32046864231564437,
+      "grad_norm": 1.7034251689910889,
+      "learning_rate": 2.999580273127263e-08,
+      "logits/chosen": -3.0472726821899414,
+      "logits/rejected": -3.029177188873291,
+      "logps/chosen": -55.30072784423828,
+      "logps/rejected": -54.390342712402344,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0004989964072592556,
+      "rewards/margins": 0.001189586939290166,
+      "rewards/rejected": -0.0016885834047570825,
+      "step": 930
+    },
+    {
+      "epoch": 0.3239145416953825,
+      "grad_norm": 1.6476922035217285,
+      "learning_rate": 2.9994259450241564e-08,
+      "logits/chosen": -3.083702325820923,
+      "logits/rejected": -3.0517983436584473,
+      "logps/chosen": -54.396766662597656,
+      "logps/rejected": -52.858238220214844,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0006399950361810625,
+      "rewards/margins": 0.0016470185946673155,
+      "rewards/rejected": -0.002287013689056039,
+      "step": 940
+    },
+    {
+      "epoch": 0.32736044107512063,
+      "grad_norm": 1.7700389623641968,
+      "learning_rate": 2.999247509753673e-08,
+      "logits/chosen": -3.096156120300293,
+      "logits/rejected": -3.07670259475708,
+      "logps/chosen": -55.316322326660156,
+      "logps/rejected": -53.52649688720703,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.0006754841888323426,
+      "rewards/margins": 0.0010412309784442186,
+      "rewards/rejected": -0.001716715283691883,
+      "step": 950
+    },
+    {
+      "epoch": 0.33080634045485874,
+      "grad_norm": 1.6826844215393066,
+      "learning_rate": 2.999044970184623e-08,
+      "logits/chosen": -3.018376111984253,
+      "logits/rejected": -2.9868927001953125,
+      "logps/chosen": -55.01654052734375,
+      "logps/rejected": -53.73602294921875,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.000983581761829555,
+      "rewards/margins": 0.001379077322781086,
+      "rewards/rejected": -0.002362658968195319,
+      "step": 960
+    },
+    {
+      "epoch": 0.33425223983459684,
+      "grad_norm": 1.5582091808319092,
+      "learning_rate": 2.998818329573357e-08,
+      "logits/chosen": -3.0365097522735596,
+      "logits/rejected": -3.008584976196289,
+      "logps/chosen": -52.53923416137695,
+      "logps/rejected": -52.17295455932617,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0012402713764458895,
+      "rewards/margins": 0.0011178593849763274,
+      "rewards/rejected": -0.002358130645006895,
+      "step": 970
+    },
+    {
+      "epoch": 0.33769813921433495,
+      "grad_norm": 1.4904838800430298,
+      "learning_rate": 2.9985675915637104e-08,
+      "logits/chosen": -3.0578620433807373,
+      "logits/rejected": -3.047434091567993,
+      "logps/chosen": -53.54816818237305,
+      "logps/rejected": -54.849029541015625,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": -0.0011458449298515916,
+      "rewards/margins": 0.0009728182922117412,
+      "rewards/rejected": -0.002118663163855672,
+      "step": 980
+    },
+    {
+      "epoch": 0.34114403859407305,
+      "grad_norm": 1.5820915699005127,
+      "learning_rate": 2.998292760186949e-08,
+      "logits/chosen": -3.059300184249878,
+      "logits/rejected": -3.0417773723602295,
+      "logps/chosen": -55.967498779296875,
+      "logps/rejected": -53.123863220214844,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.0010029891273006797,
+      "rewards/margins": 0.0008108352194540203,
+      "rewards/rejected": -0.0018138240557163954,
+      "step": 990
+    },
+    {
+      "epoch": 0.34458993797381116,
+      "grad_norm": 1.6625953912734985,
+      "learning_rate": 2.997993839861701e-08,
+      "logits/chosen": -3.102475643157959,
+      "logits/rejected": -3.088132858276367,
+      "logps/chosen": -50.54848098754883,
+      "logps/rejected": -53.16508102416992,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.001508292625658214,
+      "rewards/margins": 0.0012326488504186273,
+      "rewards/rejected": -0.002740941708907485,
+      "step": 1000
+    },
+    {
+      "epoch": 0.34458993797381116,
+      "eval_logits/chosen": -3.1609771251678467,
+      "eval_logits/rejected": -3.155362844467163,
+      "eval_logps/chosen": -58.63610076904297,
+      "eval_logps/rejected": -63.137489318847656,
+      "eval_loss": 0.692983090877533,
+      "eval_rewards/accuracies": 0.5278810262680054,
+      "eval_rewards/chosen": 0.0007579621160402894,
+      "eval_rewards/margins": 0.0003316086367703974,
+      "eval_rewards/rejected": 0.0004263535374775529,
+      "eval_runtime": 383.0791,
+      "eval_samples_per_second": 11.235,
+      "eval_steps_per_second": 1.404,
+      "step": 1000
+    },
+    {
+      "epoch": 0.34803583735354926,
+      "grad_norm": 1.815322995185852,
+      "learning_rate": 2.997670835393887e-08,
+      "logits/chosen": -3.0633022785186768,
+      "logits/rejected": -3.0380825996398926,
+      "logps/chosen": -52.82757568359375,
+      "logps/rejected": -53.5170783996582,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.001119806431233883,
+      "rewards/margins": 0.0016460234764963388,
+      "rewards/rejected": -0.002765829674899578,
+      "step": 1010
+    },
+    {
+      "epoch": 0.35148173673328736,
+      "grad_norm": 1.671033501625061,
+      "learning_rate": 2.997323751976643e-08,
+      "logits/chosen": -3.0768179893493652,
+      "logits/rejected": -3.059640884399414,
+      "logps/chosen": -55.023902893066406,
+      "logps/rejected": -54.8066291809082,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0013978948118165135,
+      "rewards/margins": 0.00099339394364506,
+      "rewards/rejected": -0.0023912887554615736,
+      "step": 1020
+    },
+    {
+      "epoch": 0.3549276361130255,
+      "grad_norm": 1.5725306272506714,
+      "learning_rate": 2.996952595190236e-08,
+      "logits/chosen": -3.120556354522705,
+      "logits/rejected": -3.088679552078247,
+      "logps/chosen": -53.35212326049805,
+      "logps/rejected": -53.22279739379883,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.0007638428942300379,
+      "rewards/margins": 0.0015006158500909805,
+      "rewards/rejected": -0.0022644586861133575,
+      "step": 1030
+    },
+    {
+      "epoch": 0.35837353549276363,
+      "grad_norm": 1.5925283432006836,
+      "learning_rate": 2.9965573710019774e-08,
+      "logits/chosen": -3.102203845977783,
+      "logits/rejected": -3.075490951538086,
+      "logps/chosen": -53.72309112548828,
+      "logps/rejected": -52.9486083984375,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.559374988079071,
+      "rewards/chosen": -0.0014259221497923136,
+      "rewards/margins": 0.000993490219116211,
+      "rewards/rejected": -0.002419412601739168,
+      "step": 1040
+    },
+    {
+      "epoch": 0.36181943487250173,
+      "grad_norm": 1.78251051902771,
+      "learning_rate": 2.99613808576612e-08,
+      "logits/chosen": -3.0715041160583496,
+      "logits/rejected": -3.0586888790130615,
+      "logps/chosen": -55.00690841674805,
+      "logps/rejected": -53.188499450683594,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.0013630136381834745,
+      "rewards/margins": 0.0009841513819992542,
+      "rewards/rejected": -0.0023471650201827288,
+      "step": 1050
+    },
+    {
+      "epoch": 0.36526533425223984,
+      "grad_norm": 1.728184700012207,
+      "learning_rate": 2.995694746223766e-08,
+      "logits/chosen": -3.0347065925598145,
+      "logits/rejected": -3.0225110054016113,
+      "logps/chosen": -51.3608512878418,
+      "logps/rejected": -53.31982421875,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.0012322448892518878,
+      "rewards/margins": 0.0013714361703023314,
+      "rewards/rejected": -0.002603681292384863,
+      "step": 1060
+    },
+    {
+      "epoch": 0.36871123363197794,
+      "grad_norm": 1.6160223484039307,
+      "learning_rate": 2.9952273595027483e-08,
+      "logits/chosen": -2.9934301376342773,
+      "logits/rejected": -2.9800262451171875,
+      "logps/chosen": -51.78477096557617,
+      "logps/rejected": -53.38568115234375,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": -0.0022202604450285435,
+      "rewards/margins": 0.001190343638882041,
+      "rewards/rejected": -0.003410604316741228,
+      "step": 1070
+    },
+    {
+      "epoch": 0.37215713301171605,
+      "grad_norm": 1.4740856885910034,
+      "learning_rate": 2.994735933117524e-08,
+      "logits/chosen": -3.017335891723633,
+      "logits/rejected": -2.9865212440490723,
+      "logps/chosen": -54.83051681518555,
+      "logps/rejected": -49.07413864135742,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0010770931839942932,
+      "rewards/margins": 0.0018510533263906837,
+      "rewards/rejected": -0.002928146393969655,
+      "step": 1080
+    },
+    {
+      "epoch": 0.37560303239145415,
+      "grad_norm": 1.9188686609268188,
+      "learning_rate": 2.994220474969048e-08,
+      "logits/chosen": -3.1487276554107666,
+      "logits/rejected": -3.1216890811920166,
+      "logps/chosen": -56.34492111206055,
+      "logps/rejected": -52.4360466003418,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0016724407905712724,
+      "rewards/margins": 0.0016059385379776359,
+      "rewards/rejected": -0.0032783790957182646,
+      "step": 1090
+    },
+    {
+      "epoch": 0.37904893177119225,
+      "grad_norm": 1.9188597202301025,
+      "learning_rate": 2.9936809933446495e-08,
+      "logits/chosen": -3.151737689971924,
+      "logits/rejected": -3.1368980407714844,
+      "logps/chosen": -52.7198600769043,
+      "logps/rejected": -54.62260818481445,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.001445785746909678,
+      "rewards/margins": 0.0015354168135672808,
+      "rewards/rejected": -0.0029812022112309933,
+      "step": 1100
+    },
+    {
+      "epoch": 0.37904893177119225,
+      "eval_logits/chosen": -3.1604361534118652,
+      "eval_logits/rejected": -3.1548290252685547,
+      "eval_logps/chosen": -58.62195587158203,
+      "eval_logps/rejected": -63.12851333618164,
+      "eval_loss": 0.6929577589035034,
+      "eval_rewards/accuracies": 0.55599445104599,
+      "eval_rewards/chosen": 0.0008994179079309106,
+      "eval_rewards/margins": 0.00038330251118168235,
+      "eval_rewards/rejected": 0.0005161153385415673,
+      "eval_runtime": 382.8058,
+      "eval_samples_per_second": 11.243,
+      "eval_steps_per_second": 1.405,
+      "step": 1100
+    },
+    {
+      "epoch": 0.3824948311509304,
+      "grad_norm": 1.732284665107727,
+      "learning_rate": 2.993117496917897e-08,
+      "logits/chosen": -3.0373644828796387,
+      "logits/rejected": -3.025477647781372,
+      "logps/chosen": -54.39824295043945,
+      "logps/rejected": -54.050621032714844,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.0009606629610061646,
+      "rewards/margins": 0.001761974417604506,
+      "rewards/rejected": -0.0027226374950259924,
+      "step": 1110
+    },
+    {
+      "epoch": 0.3859407305306685,
+      "grad_norm": 1.7853158712387085,
+      "learning_rate": 2.9925299947484594e-08,
+      "logits/chosen": -3.085400342941284,
+      "logits/rejected": -3.0512118339538574,
+      "logps/chosen": -55.43015670776367,
+      "logps/rejected": -52.572792053222656,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.0018654648447409272,
+      "rewards/margins": 0.0021848485339432955,
+      "rewards/rejected": -0.0040503134950995445,
+      "step": 1120
+    },
+    {
+      "epoch": 0.3893866299104066,
+      "grad_norm": 1.6215202808380127,
+      "learning_rate": 2.99191849628196e-08,
+      "logits/chosen": -2.9989771842956543,
+      "logits/rejected": -2.9685416221618652,
+      "logps/chosen": -58.2257194519043,
+      "logps/rejected": -55.724830627441406,
+      "loss": 0.692,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.0014696985017508268,
+      "rewards/margins": 0.002226046519353986,
+      "rewards/rejected": -0.0036957454867661,
+      "step": 1130
+    },
+    {
+      "epoch": 0.3928325292901447,
+      "grad_norm": 1.6273730993270874,
+      "learning_rate": 2.991283011349826e-08,
+      "logits/chosen": -3.0022459030151367,
+      "logits/rejected": -2.9804234504699707,
+      "logps/chosen": -53.947410583496094,
+      "logps/rejected": -54.000038146972656,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.0019494646694511175,
+      "rewards/margins": 0.0018178420141339302,
+      "rewards/rejected": -0.0037673066835850477,
+      "step": 1140
+    },
+    {
+      "epoch": 0.39627842866988283,
+      "grad_norm": 1.7841063737869263,
+      "learning_rate": 2.9906235501691284e-08,
+      "logits/chosen": -3.029003858566284,
+      "logits/rejected": -2.9975883960723877,
+      "logps/chosen": -54.552734375,
+      "logps/rejected": -54.092750549316406,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.0013875309377908707,
+      "rewards/margins": 0.0024590925313532352,
+      "rewards/rejected": -0.0038466237019747496,
+      "step": 1150
+    },
+    {
+      "epoch": 0.39972432804962094,
+      "grad_norm": 1.6211848258972168,
+      "learning_rate": 2.989940123342419e-08,
+      "logits/chosen": -3.0719547271728516,
+      "logits/rejected": -3.0432779788970947,
+      "logps/chosen": -54.452796936035156,
+      "logps/rejected": -51.71295166015625,
+      "loss": 0.692,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.001930365338921547,
+      "rewards/margins": 0.002290282165631652,
+      "rewards/rejected": -0.004220647271722555,
+      "step": 1160
+    },
+    {
+      "epoch": 0.40317022742935904,
+      "grad_norm": 1.5567917823791504,
+      "learning_rate": 2.989232741857559e-08,
+      "logits/chosen": -3.0519187450408936,
+      "logits/rejected": -3.034728765487671,
+      "logps/chosen": -54.27106475830078,
+      "logps/rejected": -54.246337890625,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.0021559789311140776,
+      "rewards/margins": 0.0021426836028695107,
+      "rewards/rejected": -0.0042986623011529446,
+      "step": 1170
+    },
+    {
+      "epoch": 0.4066161268090972,
+      "grad_norm": 1.6385148763656616,
+      "learning_rate": 2.988501417087543e-08,
+      "logits/chosen": -3.067915201187134,
+      "logits/rejected": -3.049938440322876,
+      "logps/chosen": -54.49457550048828,
+      "logps/rejected": -53.63732147216797,
+      "loss": 0.692,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0018288461724296212,
+      "rewards/margins": 0.002315042307600379,
+      "rewards/rejected": -0.004143889062106609,
+      "step": 1180
+    },
+    {
+      "epoch": 0.4100620261888353,
+      "grad_norm": 1.6005433797836304,
+      "learning_rate": 2.987746160790317e-08,
+      "logits/chosen": -3.05593204498291,
+      "logits/rejected": -3.0402281284332275,
+      "logps/chosen": -54.26140213012695,
+      "logps/rejected": -53.785247802734375,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.0020933211781084538,
+      "rewards/margins": 0.002112672198563814,
+      "rewards/rejected": -0.004205993376672268,
+      "step": 1190
+    },
+    {
+      "epoch": 0.4135079255685734,
+      "grad_norm": 1.603008508682251,
+      "learning_rate": 2.986966985108589e-08,
+      "logits/chosen": -3.1287004947662354,
+      "logits/rejected": -3.104531764984131,
+      "logps/chosen": -54.51715087890625,
+      "logps/rejected": -53.65093231201172,
+      "loss": 0.692,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.0024987785145640373,
+      "rewards/margins": 0.0022224222775548697,
+      "rewards/rejected": -0.004721201024949551,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4135079255685734,
+      "eval_logits/chosen": -3.1595113277435303,
+      "eval_logits/rejected": -3.153871536254883,
+      "eval_logps/chosen": -58.597347259521484,
+      "eval_logps/rejected": -63.12055587768555,
+      "eval_loss": 0.6928762197494507,
+      "eval_rewards/accuracies": 0.5406598448753357,
+      "eval_rewards/chosen": 0.0011454792693257332,
+      "eval_rewards/margins": 0.0005498203681781888,
+      "eval_rewards/rejected": 0.0005956589011475444,
+      "eval_runtime": 383.3037,
+      "eval_samples_per_second": 11.229,
+      "eval_steps_per_second": 1.404,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4169538249483115,
+      "grad_norm": 1.624894380569458,
+      "learning_rate": 2.986163902569632e-08,
+      "logits/chosen": -3.1258113384246826,
+      "logits/rejected": -3.0944161415100098,
+      "logps/chosen": -55.167015075683594,
+      "logps/rejected": -52.52492141723633,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.0015079013537615538,
+      "rewards/margins": 0.0028875190764665604,
+      "rewards/rejected": -0.0043954201973974705,
+      "step": 1210
+    },
+    {
+      "epoch": 0.4203997243280496,
+      "grad_norm": 1.6836867332458496,
+      "learning_rate": 2.985336926085083e-08,
+      "logits/chosen": -3.084153413772583,
+      "logits/rejected": -3.0661087036132812,
+      "logps/chosen": -55.137840270996094,
+      "logps/rejected": -53.53403854370117,
+      "loss": 0.692,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.001879109418950975,
+      "rewards/margins": 0.0022940735798329115,
+      "rewards/rejected": -0.004173182882368565,
+      "step": 1220
+    },
+    {
+      "epoch": 0.4238456237077877,
+      "grad_norm": 1.8374478816986084,
+      "learning_rate": 2.984486068950738e-08,
+      "logits/chosen": -3.029895782470703,
+      "logits/rejected": -3.0017082691192627,
+      "logps/chosen": -54.8785285949707,
+      "logps/rejected": -52.86114501953125,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0027864398434758186,
+      "rewards/margins": 0.0018703562673181295,
+      "rewards/rejected": -0.004656796809285879,
+      "step": 1230
+    },
+    {
+      "epoch": 0.4272915230875258,
+      "grad_norm": 1.766595721244812,
+      "learning_rate": 2.983611344846336e-08,
+      "logits/chosen": -3.1316726207733154,
+      "logits/rejected": -3.1052050590515137,
+      "logps/chosen": -54.941551208496094,
+      "logps/rejected": -52.024017333984375,
+      "loss": 0.692,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.0023036033380776644,
+      "rewards/margins": 0.002266914816573262,
+      "rewards/rejected": -0.004570518620312214,
+      "step": 1240
+    },
+    {
+      "epoch": 0.43073742246726393,
+      "grad_norm": 1.650951862335205,
+      "learning_rate": 2.9827127678353394e-08,
+      "logits/chosen": -3.1023736000061035,
+      "logits/rejected": -3.0858802795410156,
+      "logps/chosen": -53.763511657714844,
+      "logps/rejected": -54.49433517456055,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.559374988079071,
+      "rewards/chosen": -0.002987388987094164,
+      "rewards/margins": 0.0011331771966069937,
+      "rewards/rejected": -0.004120565950870514,
+      "step": 1250
+    },
+    {
+      "epoch": 0.4341833218470021,
+      "grad_norm": 1.6005022525787354,
+      "learning_rate": 2.981790352364707e-08,
+      "logits/chosen": -2.9852304458618164,
+      "logits/rejected": -2.9662821292877197,
+      "logps/chosen": -52.70601272583008,
+      "logps/rejected": -50.93710708618164,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.002790682716295123,
+      "rewards/margins": 0.0016632797196507454,
+      "rewards/rejected": -0.004453962203115225,
+      "step": 1260
+    },
+    {
+      "epoch": 0.4376292212267402,
+      "grad_norm": 1.6014958620071411,
+      "learning_rate": 2.980844113264666e-08,
+      "logits/chosen": -3.0891928672790527,
+      "logits/rejected": -3.0815272331237793,
+      "logps/chosen": -52.714141845703125,
+      "logps/rejected": -54.518341064453125,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.003591346787288785,
+      "rewards/margins": 0.0015908284112811089,
+      "rewards/rejected": -0.0051821754314005375,
+      "step": 1270
+    },
+    {
+      "epoch": 0.4410751206064783,
+      "grad_norm": 1.7393684387207031,
+      "learning_rate": 2.979874065748466e-08,
+      "logits/chosen": -3.097439765930176,
+      "logits/rejected": -3.0686800479888916,
+      "logps/chosen": -57.12340545654297,
+      "logps/rejected": -54.4604377746582,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.0015168075915426016,
+      "rewards/margins": 0.003128941636532545,
+      "rewards/rejected": -0.004645749926567078,
+      "step": 1280
+    },
+    {
+      "epoch": 0.4445210199862164,
+      "grad_norm": 1.7047237157821655,
+      "learning_rate": 2.9788802254121418e-08,
+      "logits/chosen": -3.0883235931396484,
+      "logits/rejected": -3.069504499435425,
+      "logps/chosen": -56.2879753112793,
+      "logps/rejected": -55.285804748535156,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.002942117629572749,
+      "rewards/margins": 0.001484010135754943,
+      "rewards/rejected": -0.004426128230988979,
+      "step": 1290
+    },
+    {
+      "epoch": 0.4479669193659545,
+      "grad_norm": 1.6297000646591187,
+      "learning_rate": 2.977862608234259e-08,
+      "logits/chosen": -3.0652670860290527,
+      "logits/rejected": -3.0222158432006836,
+      "logps/chosen": -57.147308349609375,
+      "logps/rejected": -50.46062088012695,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.0025200347881764174,
+      "rewards/margins": 0.0035328716039657593,
+      "rewards/rejected": -0.006052906159311533,
+      "step": 1300
+    },
+    {
+      "epoch": 0.4479669193659545,
+      "eval_logits/chosen": -3.1584436893463135,
+      "eval_logits/rejected": -3.152804136276245,
+      "eval_logps/chosen": -58.58187484741211,
+      "eval_logps/rejected": -63.11204147338867,
+      "eval_loss": 0.692842960357666,
+      "eval_rewards/accuracies": 0.538336455821991,
+      "eval_rewards/chosen": 0.0013002099003642797,
+      "eval_rewards/margins": 0.0006193388835527003,
+      "eval_rewards/rejected": 0.0006808710168115795,
+      "eval_runtime": 383.128,
+      "eval_samples_per_second": 11.234,
+      "eval_steps_per_second": 1.404,
+      "step": 1300
+    },
+    {
+      "epoch": 0.4514128187456926,
+      "grad_norm": 1.5527344942092896,
+      "learning_rate": 2.9768212305756564e-08,
+      "logits/chosen": -3.076319932937622,
+      "logits/rejected": -3.0410382747650146,
+      "logps/chosen": -55.12217330932617,
+      "logps/rejected": -51.68041229248047,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.0029966712463647127,
+      "rewards/margins": 0.003207107540220022,
+      "rewards/rejected": -0.006203779019415379,
+      "step": 1310
+    },
+    {
+      "epoch": 0.4548587181254307,
+      "grad_norm": 1.700046420097351,
+      "learning_rate": 2.975756109179186e-08,
+      "logits/chosen": -3.0147175788879395,
+      "logits/rejected": -2.9876627922058105,
+      "logps/chosen": -52.7305793762207,
+      "logps/rejected": -51.671630859375,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.002761057810857892,
+      "rewards/margins": 0.003240293590351939,
+      "rewards/rejected": -0.0060013518668711185,
+      "step": 1320
+    },
+    {
+      "epoch": 0.4583046175051689,
+      "grad_norm": 1.7509071826934814,
+      "learning_rate": 2.9746672611694415e-08,
+      "logits/chosen": -3.0124611854553223,
+      "logits/rejected": -2.982241153717041,
+      "logps/chosen": -55.62883377075195,
+      "logps/rejected": -53.884681701660156,
+      "loss": 0.692,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.002685070037841797,
+      "rewards/margins": 0.0023676182609051466,
+      "rewards/rejected": -0.005052688531577587,
+      "step": 1330
+    },
+    {
+      "epoch": 0.461750516884907,
+      "grad_norm": 1.8595162630081177,
+      "learning_rate": 2.9735547040524823e-08,
+      "logits/chosen": -3.022301197052002,
+      "logits/rejected": -2.994101047515869,
+      "logps/chosen": -56.34235382080078,
+      "logps/rejected": -53.45936965942383,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.002455942565575242,
+      "rewards/margins": 0.003516028868034482,
+      "rewards/rejected": -0.005971971899271011,
+      "step": 1340
+    },
+    {
+      "epoch": 0.4651964162646451,
+      "grad_norm": 1.6195266246795654,
+      "learning_rate": 2.9724184557155553e-08,
+      "logits/chosen": -3.0337252616882324,
+      "logits/rejected": -3.008955478668213,
+      "logps/chosen": -52.2797737121582,
+      "logps/rejected": -52.3746223449707,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.0034384247846901417,
+      "rewards/margins": 0.003493095515295863,
+      "rewards/rejected": -0.006931520998477936,
+      "step": 1350
+    },
+    {
+      "epoch": 0.4686423156443832,
+      "grad_norm": 1.71376371383667,
+      "learning_rate": 2.9712585344268038e-08,
+      "logits/chosen": -3.0666913986206055,
+      "logits/rejected": -3.0587944984436035,
+      "logps/chosen": -56.04633712768555,
+      "logps/rejected": -55.0241584777832,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.00299390172585845,
+      "rewards/margins": 0.0016801670426502824,
+      "rewards/rejected": -0.0046740686520934105,
+      "step": 1360
+    },
+    {
+      "epoch": 0.4720882150241213,
+      "grad_norm": 1.7956026792526245,
+      "learning_rate": 2.970074958834976e-08,
+      "logits/chosen": -3.056245803833008,
+      "logits/rejected": -3.035292863845825,
+      "logps/chosen": -55.98676681518555,
+      "logps/rejected": -52.84367752075195,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.002737827366217971,
+      "rewards/margins": 0.0026592675130814314,
+      "rewards/rejected": -0.005397094879299402,
+      "step": 1370
+    },
+    {
+      "epoch": 0.4755341144038594,
+      "grad_norm": 1.7535145282745361,
+      "learning_rate": 2.968867747969124e-08,
+      "logits/chosen": -3.0069825649261475,
+      "logits/rejected": -2.9955132007598877,
+      "logps/chosen": -52.11106491088867,
+      "logps/rejected": -54.7510871887207,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.0034170071594417095,
+      "rewards/margins": 0.0029301538597792387,
+      "rewards/rejected": -0.006347161717712879,
+      "step": 1380
+    },
+    {
+      "epoch": 0.4789800137835975,
+      "grad_norm": 1.647588849067688,
+      "learning_rate": 2.9676369212382992e-08,
+      "logits/chosen": -3.0744569301605225,
+      "logits/rejected": -3.0442047119140625,
+      "logps/chosen": -54.375022888183594,
+      "logps/rejected": -52.406288146972656,
+      "loss": 0.692,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.0035322539042681456,
+      "rewards/margins": 0.002261875430122018,
+      "rewards/rejected": -0.005794128868728876,
+      "step": 1390
+    },
+    {
+      "epoch": 0.4824259131633356,
+      "grad_norm": 1.7980599403381348,
+      "learning_rate": 2.966382498431239e-08,
+      "logits/chosen": -3.0923869609832764,
+      "logits/rejected": -3.0627763271331787,
+      "logps/chosen": -56.16261672973633,
+      "logps/rejected": -54.02740478515625,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.002646544249728322,
+      "rewards/margins": 0.0028753154911100864,
+      "rewards/rejected": -0.005521859508007765,
+      "step": 1400
+    },
+    {
+      "epoch": 0.4824259131633356,
+      "eval_logits/chosen": -3.1573994159698486,
+      "eval_logits/rejected": -3.151780605316162,
+      "eval_logps/chosen": -58.55329132080078,
+      "eval_logps/rejected": -63.11597442626953,
+      "eval_loss": 0.6926826238632202,
+      "eval_rewards/accuracies": 0.5648234486579895,
+      "eval_rewards/chosen": 0.0015860300045460463,
+      "eval_rewards/margins": 0.0009445542236790061,
+      "eval_rewards/rejected": 0.0006414757808670402,
+      "eval_runtime": 382.8463,
+      "eval_samples_per_second": 11.242,
+      "eval_steps_per_second": 1.405,
+      "step": 1400
+    },
+    {
+      "epoch": 0.48587181254307377,
+      "grad_norm": 1.6935856342315674,
+      "learning_rate": 2.9651044997160484e-08,
+      "logits/chosen": -3.0481629371643066,
+      "logits/rejected": -3.029620885848999,
+      "logps/chosen": -55.1282844543457,
+      "logps/rejected": -54.97698974609375,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.002967803506180644,
+      "rewards/margins": 0.0028852166142314672,
+      "rewards/rejected": -0.005853020586073399,
+      "step": 1410
+    },
+    {
+      "epoch": 0.48931771192281187,
+      "grad_norm": 1.54836106300354,
+      "learning_rate": 2.9638029456398766e-08,
+      "logits/chosen": -3.0113720893859863,
+      "logits/rejected": -2.997924327850342,
+      "logps/chosen": -54.432891845703125,
+      "logps/rejected": -53.80595016479492,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.0036449008621275425,
+      "rewards/margins": 0.002691942732781172,
+      "rewards/rejected": -0.00633684266358614,
+      "step": 1420
+    },
+    {
+      "epoch": 0.49276361130255,
+      "grad_norm": 1.589377999305725,
+      "learning_rate": 2.962477857128587e-08,
+      "logits/chosen": -3.0500454902648926,
+      "logits/rejected": -3.0280795097351074,
+      "logps/chosen": -53.9366455078125,
+      "logps/rejected": -54.18046951293945,
+      "loss": 0.691,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.003311690641567111,
+      "rewards/margins": 0.004361622966825962,
+      "rewards/rejected": -0.007673312909901142,
+      "step": 1430
+    },
+    {
+      "epoch": 0.4962095106822881,
+      "grad_norm": 1.8061046600341797,
+      "learning_rate": 2.9611292554864195e-08,
+      "logits/chosen": -3.119410753250122,
+      "logits/rejected": -3.094177007675171,
+      "logps/chosen": -56.072303771972656,
+      "logps/rejected": -52.75751876831055,
+      "loss": 0.691,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0034040685277432203,
+      "rewards/margins": 0.004354691132903099,
+      "rewards/rejected": -0.007758758962154388,
+      "step": 1440
+    },
+    {
+      "epoch": 0.4996554100620262,
+      "grad_norm": 1.6284228563308716,
+      "learning_rate": 2.9597571623956494e-08,
+      "logits/chosen": -3.1263468265533447,
+      "logits/rejected": -3.11881685256958,
+      "logps/chosen": -53.06935501098633,
+      "logps/rejected": -52.56400680541992,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.0034648750443011522,
+      "rewards/margins": 0.0019054736476391554,
+      "rewards/rejected": -0.005370349157601595,
+      "step": 1450
+    },
+    {
+      "epoch": 0.5031013094417643,
+      "grad_norm": 1.6312525272369385,
+      "learning_rate": 2.9583615999162376e-08,
+      "logits/chosen": -3.0971179008483887,
+      "logits/rejected": -3.089125156402588,
+      "logps/chosen": -55.19788360595703,
+      "logps/rejected": -56.41972732543945,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0035923414397984743,
+      "rewards/margins": 0.0035721107851713896,
+      "rewards/rejected": -0.007164451293647289,
+      "step": 1460
+    },
+    {
+      "epoch": 0.5065472088215024,
+      "grad_norm": 1.5604609251022339,
+      "learning_rate": 2.9569425904854763e-08,
+      "logits/chosen": -3.0576207637786865,
+      "logits/rejected": -3.0247445106506348,
+      "logps/chosen": -52.80244827270508,
+      "logps/rejected": -51.93998336791992,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0034666378051042557,
+      "rewards/margins": 0.003667470533400774,
+      "rewards/rejected": -0.00713410833850503,
+      "step": 1470
+    },
+    {
+      "epoch": 0.5099931082012406,
+      "grad_norm": 1.8163104057312012,
+      "learning_rate": 2.9555001569176282e-08,
+      "logits/chosen": -3.0879034996032715,
+      "logits/rejected": -3.0647711753845215,
+      "logps/chosen": -54.4842529296875,
+      "logps/rejected": -53.94389724731445,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.0034971176646649837,
+      "rewards/margins": 0.003796276170760393,
+      "rewards/rejected": -0.007293394301086664,
+      "step": 1480
+    },
+    {
+      "epoch": 0.5134390075809786,
+      "grad_norm": 1.641088843345642,
+      "learning_rate": 2.9540343224035607e-08,
+      "logits/chosen": -2.99851131439209,
+      "logits/rejected": -2.9712367057800293,
+      "logps/chosen": -54.9056282043457,
+      "logps/rejected": -53.13238525390625,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.0031726867891848087,
+      "rewards/margins": 0.0046774414367973804,
+      "rewards/rejected": -0.00785012822598219,
+      "step": 1490
+    },
+    {
+      "epoch": 0.5168849069607168,
+      "grad_norm": 1.536830186843872,
+      "learning_rate": 2.9525451105103716e-08,
+      "logits/chosen": -3.049912929534912,
+      "logits/rejected": -3.0329270362854004,
+      "logps/chosen": -54.653175354003906,
+      "logps/rejected": -54.951560974121094,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.0034207901917397976,
+      "rewards/margins": 0.00354943610727787,
+      "rewards/rejected": -0.006970226764678955,
+      "step": 1500
+    },
+    {
+      "epoch": 0.5168849069607168,
+      "eval_logits/chosen": -3.1560568809509277,
+      "eval_logits/rejected": -3.150465488433838,
+      "eval_logps/chosen": -58.55392837524414,
+      "eval_logps/rejected": -63.124332427978516,
+      "eval_loss": 0.6926473379135132,
+      "eval_rewards/accuracies": 0.5573884844779968,
+      "eval_rewards/chosen": 0.0015796381048858166,
+      "eval_rewards/margins": 0.0010217450326308608,
+      "eval_rewards/rejected": 0.0005578931886702776,
+      "eval_runtime": 382.8662,
+      "eval_samples_per_second": 11.242,
+      "eval_steps_per_second": 1.405,
+      "step": 1500
+    },
+    {
+      "epoch": 0.5203308063404548,
+      "grad_norm": 1.7428698539733887,
+      "learning_rate": 2.95103254518101e-08,
+      "logits/chosen": -3.125021457672119,
+      "logits/rejected": -3.097287654876709,
+      "logps/chosen": -55.09740447998047,
+      "logps/rejected": -52.48225021362305,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.003843106795102358,
+      "rewards/margins": 0.003928641323000193,
+      "rewards/rejected": -0.0077717481181025505,
+      "step": 1510
+    },
+    {
+      "epoch": 0.523776705720193,
+      "grad_norm": 1.5329238176345825,
+      "learning_rate": 2.9494966507338943e-08,
+      "logits/chosen": -3.120823621749878,
+      "logits/rejected": -3.1067392826080322,
+      "logps/chosen": -53.9554328918457,
+      "logps/rejected": -55.38352584838867,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.005240868777036667,
+      "rewards/margins": 0.0025690165348351,
+      "rewards/rejected": -0.007809885777533054,
+      "step": 1520
+    },
+    {
+      "epoch": 0.5272226050999311,
+      "grad_norm": 1.6470295190811157,
+      "learning_rate": 2.9479374518625167e-08,
+      "logits/chosen": -3.0496158599853516,
+      "logits/rejected": -3.0335171222686768,
+      "logps/chosen": -55.030189514160156,
+      "logps/rejected": -55.5028076171875,
+      "loss": 0.692,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.004132495261728764,
+      "rewards/margins": 0.0022706554736942053,
+      "rewards/rejected": -0.006403150502592325,
+      "step": 1530
+    },
+    {
+      "epoch": 0.5306685044796692,
+      "grad_norm": 1.5335067510604858,
+      "learning_rate": 2.9463549736350504e-08,
+      "logits/chosen": -2.9831607341766357,
+      "logits/rejected": -2.961778163909912,
+      "logps/chosen": -55.00383377075195,
+      "logps/rejected": -55.2543830871582,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.006033133715391159,
+      "rewards/margins": 0.002813856117427349,
+      "rewards/rejected": -0.008846990764141083,
+      "step": 1540
+    },
+    {
+      "epoch": 0.5341144038594073,
+      "grad_norm": 1.7475658655166626,
+      "learning_rate": 2.944749241493944e-08,
+      "logits/chosen": -3.043959140777588,
+      "logits/rejected": -3.0194754600524902,
+      "logps/chosen": -54.183555603027344,
+      "logps/rejected": -53.47435760498047,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.004173078574240208,
+      "rewards/margins": 0.004195301793515682,
+      "rewards/rejected": -0.008368381299078465,
+      "step": 1550
+    },
+    {
+      "epoch": 0.5375603032391454,
+      "grad_norm": 1.7715306282043457,
+      "learning_rate": 2.943120281255513e-08,
+      "logits/chosen": -3.026848077774048,
+      "logits/rejected": -3.019639492034912,
+      "logps/chosen": -53.85759735107422,
+      "logps/rejected": -56.1359748840332,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.005459682084619999,
+      "rewards/margins": 0.0021346521098166704,
+      "rewards/rejected": -0.0075943348929286,
+      "step": 1560
+    },
+    {
+      "epoch": 0.5410062026188835,
+      "grad_norm": 1.6364390850067139,
+      "learning_rate": 2.941468119109524e-08,
+      "logits/chosen": -3.027050495147705,
+      "logits/rejected": -3.015225648880005,
+      "logps/chosen": -52.420814514160156,
+      "logps/rejected": -54.33024978637695,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.0053177643567323685,
+      "rewards/margins": 0.0025779330171644688,
+      "rewards/rejected": -0.007895697839558125,
+      "step": 1570
+    },
+    {
+      "epoch": 0.5444521019986216,
+      "grad_norm": 1.6407400369644165,
+      "learning_rate": 2.9397927816187772e-08,
+      "logits/chosen": -3.0577361583709717,
+      "logits/rejected": -3.0284156799316406,
+      "logps/chosen": -54.564796447753906,
+      "logps/rejected": -54.013710021972656,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.003071602899581194,
+      "rewards/margins": 0.005887563340365887,
+      "rewards/rejected": -0.008959166705608368,
+      "step": 1580
+    },
+    {
+      "epoch": 0.5478980013783598,
+      "grad_norm": 1.4859795570373535,
+      "learning_rate": 2.938094295718674e-08,
+      "logits/chosen": -3.0795650482177734,
+      "logits/rejected": -3.044229030609131,
+      "logps/chosen": -54.97173309326172,
+      "logps/rejected": -51.42909622192383,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": -0.003988319542258978,
+      "rewards/margins": 0.005732369609177113,
+      "rewards/rejected": -0.009720688685774803,
+      "step": 1590
+    },
+    {
+      "epoch": 0.5513439007580979,
+      "grad_norm": 1.7501975297927856,
+      "learning_rate": 2.9363726887167886e-08,
+      "logits/chosen": -3.067391872406006,
+      "logits/rejected": -3.0574698448181152,
+      "logps/chosen": -54.2914924621582,
+      "logps/rejected": -55.19401168823242,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.004783066920936108,
+      "rewards/margins": 0.0031693843193352222,
+      "rewards/rejected": -0.007952452637255192,
+      "step": 1600
+    },
+    {
+      "epoch": 0.5513439007580979,
+      "eval_logits/chosen": -3.1548821926116943,
+      "eval_logits/rejected": -3.149266481399536,
+      "eval_logps/chosen": -58.5287971496582,
+      "eval_logps/rejected": -63.11451721191406,
+      "eval_loss": 0.6925737857818604,
+      "eval_rewards/accuracies": 0.5576208233833313,
+      "eval_rewards/chosen": 0.001830965280532837,
+      "eval_rewards/margins": 0.0011748875258490443,
+      "eval_rewards/rejected": 0.0006560777546837926,
+      "eval_runtime": 383.585,
+      "eval_samples_per_second": 11.22,
+      "eval_steps_per_second": 1.403,
+      "step": 1600
+    },
+    {
+      "epoch": 0.554789800137836,
+      "grad_norm": 1.6718322038650513,
+      "learning_rate": 2.9346279882924254e-08,
+      "logits/chosen": -2.9646155834198,
+      "logits/rejected": -2.949723243713379,
+      "logps/chosen": -52.64680862426758,
+      "logps/rejected": -53.51043701171875,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.004991953261196613,
+      "rewards/margins": 0.0032116789370775223,
+      "rewards/rejected": -0.00820363312959671,
+      "step": 1610
+    },
+    {
+      "epoch": 0.5582356995175741,
+      "grad_norm": 1.6192212104797363,
+      "learning_rate": 2.9328602224961756e-08,
+      "logits/chosen": -3.098616361618042,
+      "logits/rejected": -3.07834792137146,
+      "logps/chosen": -55.74953079223633,
+      "logps/rejected": -55.364707946777344,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.005525125656276941,
+      "rewards/margins": 0.003510179463773966,
+      "rewards/rejected": -0.009035304188728333,
+      "step": 1620
+    },
+    {
+      "epoch": 0.5616815988973122,
+      "grad_norm": 1.6109540462493896,
+      "learning_rate": 2.931069419749468e-08,
+      "logits/chosen": -3.059250593185425,
+      "logits/rejected": -3.039900541305542,
+      "logps/chosen": -52.56306838989258,
+      "logps/rejected": -54.02900314331055,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.005090720020234585,
+      "rewards/margins": 0.004961690865457058,
+      "rewards/rejected": -0.010052410885691643,
+      "step": 1630
+    },
+    {
+      "epoch": 0.5651274982770503,
+      "grad_norm": 1.6446406841278076,
+      "learning_rate": 2.929255608844109e-08,
+      "logits/chosen": -3.1257262229919434,
+      "logits/rejected": -3.1184473037719727,
+      "logps/chosen": -52.92694091796875,
+      "logps/rejected": -56.12247848510742,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.006517951842397451,
+      "rewards/margins": 0.0032779318280518055,
+      "rewards/rejected": -0.009795883670449257,
+      "step": 1640
+    },
+    {
+      "epoch": 0.5685733976567884,
+      "grad_norm": 1.7818858623504639,
+      "learning_rate": 2.92741881894182e-08,
+      "logits/chosen": -3.1139843463897705,
+      "logits/rejected": -3.087250232696533,
+      "logps/chosen": -54.50800323486328,
+      "logps/rejected": -54.3870964050293,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.004666454158723354,
+      "rewards/margins": 0.004705200903117657,
+      "rewards/rejected": -0.009371655061841011,
+      "step": 1650
+    },
+    {
+      "epoch": 0.5720192970365265,
+      "grad_norm": 1.688673496246338,
+      "learning_rate": 2.925559079573771e-08,
+      "logits/chosen": -3.0219969749450684,
+      "logits/rejected": -2.989677667617798,
+      "logps/chosen": -53.84386444091797,
+      "logps/rejected": -50.862388610839844,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.004406494088470936,
+      "rewards/margins": 0.0059952340088784695,
+      "rewards/rejected": -0.010401727631688118,
+      "step": 1660
+    },
+    {
+      "epoch": 0.5754651964162646,
+      "grad_norm": 1.7014291286468506,
+      "learning_rate": 2.923676420640104e-08,
+      "logits/chosen": -3.119534969329834,
+      "logits/rejected": -3.094654083251953,
+      "logps/chosen": -55.472557067871094,
+      "logps/rejected": -54.810546875,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.004092915914952755,
+      "rewards/margins": 0.0057355985045433044,
+      "rewards/rejected": -0.009828515350818634,
+      "step": 1670
+    },
+    {
+      "epoch": 0.5789110957960028,
+      "grad_norm": 1.7532297372817993,
+      "learning_rate": 2.9217708724094516e-08,
+      "logits/chosen": -3.0433664321899414,
+      "logits/rejected": -3.0183422565460205,
+      "logps/chosen": -55.80550003051758,
+      "logps/rejected": -55.93865203857422,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0050854929722845554,
+      "rewards/margins": 0.006522328592836857,
+      "rewards/rejected": -0.011607821099460125,
+      "step": 1680
+    },
+    {
+      "epoch": 0.5823569951757409,
+      "grad_norm": 1.5913794040679932,
+      "learning_rate": 2.919842465518452e-08,
+      "logits/chosen": -3.085564136505127,
+      "logits/rejected": -3.0642447471618652,
+      "logps/chosen": -53.73085403442383,
+      "logps/rejected": -53.102813720703125,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.005227671004831791,
+      "rewards/margins": 0.005013564135879278,
+      "rewards/rejected": -0.010241234675049782,
+      "step": 1690
+    },
+    {
+      "epoch": 0.585802894555479,
+      "grad_norm": 1.6771739721298218,
+      "learning_rate": 2.9178912309712562e-08,
+      "logits/chosen": -3.0842106342315674,
+      "logits/rejected": -3.05106782913208,
+      "logps/chosen": -53.84814453125,
+      "logps/rejected": -52.485748291015625,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.005559450946748257,
+      "rewards/margins": 0.005207203794270754,
+      "rewards/rejected": -0.010766653344035149,
+      "step": 1700
+    },
+    {
+      "epoch": 0.585802894555479,
+      "eval_logits/chosen": -3.15273118019104,
+      "eval_logits/rejected": -3.14707350730896,
+      "eval_logps/chosen": -58.5250244140625,
+      "eval_logps/rejected": -63.13581085205078,
+      "eval_loss": 0.6924527287483215,
+      "eval_rewards/accuracies": 0.5625,
+      "eval_rewards/chosen": 0.0018686669645830989,
+      "eval_rewards/margins": 0.001425592228770256,
+      "eval_rewards/rejected": 0.0004430746485013515,
+      "eval_runtime": 383.333,
+      "eval_samples_per_second": 11.228,
+      "eval_steps_per_second": 1.403,
+      "step": 1700
+    },
+    {
+      "epoch": 0.5892487939352171,
+      "grad_norm": 1.8865458965301514,
+      "learning_rate": 2.9159172001390287e-08,
+      "logits/chosen": -3.0084052085876465,
+      "logits/rejected": -2.9815661907196045,
+      "logps/chosen": -51.73400115966797,
+      "logps/rejected": -51.31731033325195,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.005637140944600105,
+      "rewards/margins": 0.005700516514480114,
+      "rewards/rejected": -0.011337658390402794,
+      "step": 1710
+    },
+    {
+      "epoch": 0.5926946933149552,
+      "grad_norm": 1.7400603294372559,
+      "learning_rate": 2.9139204047594427e-08,
+      "logits/chosen": -3.0641531944274902,
+      "logits/rejected": -3.0458168983459473,
+      "logps/chosen": -52.66326904296875,
+      "logps/rejected": -54.883628845214844,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.0070638759061694145,
+      "rewards/margins": 0.004609099123626947,
+      "rewards/rejected": -0.01167297549545765,
+      "step": 1720
+    },
+    {
+      "epoch": 0.5961405926946933,
+      "grad_norm": 1.5579107999801636,
+      "learning_rate": 2.9119008769361717e-08,
+      "logits/chosen": -3.163053035736084,
+      "logits/rejected": -3.136101484298706,
+      "logps/chosen": -56.28434371948242,
+      "logps/rejected": -55.26015090942383,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.005049136467278004,
+      "rewards/margins": 0.004816731903702021,
+      "rewards/rejected": -0.009865867905318737,
+      "step": 1730
+    },
+    {
+      "epoch": 0.5995864920744314,
+      "grad_norm": 1.7325162887573242,
+      "learning_rate": 2.9098586491383723e-08,
+      "logits/chosen": -3.0940730571746826,
+      "logits/rejected": -3.070148468017578,
+      "logps/chosen": -55.9141960144043,
+      "logps/rejected": -54.17383575439453,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.004517878871411085,
+      "rewards/margins": 0.006125928368419409,
+      "rewards/rejected": -0.010643808171153069,
+      "step": 1740
+    },
+    {
+      "epoch": 0.6030323914541695,
+      "grad_norm": 1.5141555070877075,
+      "learning_rate": 2.9077937542001613e-08,
+      "logits/chosen": -3.0392913818359375,
+      "logits/rejected": -3.0206305980682373,
+      "logps/chosen": -53.762168884277344,
+      "logps/rejected": -55.75056076049805,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.006261381320655346,
+      "rewards/margins": 0.0053213476203382015,
+      "rewards/rejected": -0.011582729406654835,
+      "step": 1750
+    },
+    {
+      "epoch": 0.6064782908339077,
+      "grad_norm": 1.6287105083465576,
+      "learning_rate": 2.905706225320089e-08,
+      "logits/chosen": -2.9722933769226074,
+      "logits/rejected": -2.9494502544403076,
+      "logps/chosen": -55.24475860595703,
+      "logps/rejected": -52.7999153137207,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.00793982669711113,
+      "rewards/margins": 0.004009750671684742,
+      "rewards/rejected": -0.011949577368795872,
+      "step": 1760
+    },
+    {
+      "epoch": 0.6099241902136457,
+      "grad_norm": 1.8194224834442139,
+      "learning_rate": 2.903596096060606e-08,
+      "logits/chosen": -3.0681567192077637,
+      "logits/rejected": -3.03401517868042,
+      "logps/chosen": -55.460914611816406,
+      "logps/rejected": -53.46044921875,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.005258029326796532,
+      "rewards/margins": 0.00694118719547987,
+      "rewards/rejected": -0.012199217453598976,
+      "step": 1770
+    },
+    {
+      "epoch": 0.6133700895933839,
+      "grad_norm": 1.6226096153259277,
+      "learning_rate": 2.9014634003475212e-08,
+      "logits/chosen": -3.03596830368042,
+      "logits/rejected": -3.017085075378418,
+      "logps/chosen": -54.464073181152344,
+      "logps/rejected": -53.735618591308594,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.007325285580009222,
+      "rewards/margins": 0.00453385291621089,
+      "rewards/rejected": -0.011859138496220112,
+      "step": 1780
+    },
+    {
+      "epoch": 0.616815988973122,
+      "grad_norm": 1.7624455690383911,
+      "learning_rate": 2.8993081724694585e-08,
+      "logits/chosen": -3.1157326698303223,
+      "logits/rejected": -3.086496114730835,
+      "logps/chosen": -58.0665397644043,
+      "logps/rejected": -56.53605270385742,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.005327856633812189,
+      "rewards/margins": 0.006564105860888958,
+      "rewards/rejected": -0.011891962960362434,
+      "step": 1790
+    },
+    {
+      "epoch": 0.6202618883528601,
+      "grad_norm": 1.7398505210876465,
+      "learning_rate": 2.8971304470773063e-08,
+      "logits/chosen": -3.0596938133239746,
+      "logits/rejected": -3.0392985343933105,
+      "logps/chosen": -55.2796745300293,
+      "logps/rejected": -54.008445739746094,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.007436450570821762,
+      "rewards/margins": 0.004914063960313797,
+      "rewards/rejected": -0.012350514531135559,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6202618883528601,
+      "eval_logits/chosen": -3.1512882709503174,
+      "eval_logits/rejected": -3.1456427574157715,
+      "eval_logps/chosen": -58.51980972290039,
+      "eval_logps/rejected": -63.16015625,
+      "eval_loss": 0.6923107504844666,
+      "eval_rewards/accuracies": 0.5550650358200073,
+      "eval_rewards/chosen": 0.0019208698067814112,
+      "eval_rewards/margins": 0.0017212230013683438,
+      "eval_rewards/rejected": 0.00019964661623816937,
+      "eval_runtime": 383.2682,
+      "eval_samples_per_second": 11.23,
+      "eval_steps_per_second": 1.404,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6237077877325982,
+      "grad_norm": 1.6733312606811523,
+      "learning_rate": 2.8949302591836568e-08,
+      "logits/chosen": -2.99592924118042,
+      "logits/rejected": -2.9752960205078125,
+      "logps/chosen": -54.15668869018555,
+      "logps/rejected": -52.7047233581543,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.006561542861163616,
+      "rewards/margins": 0.005290547385811806,
+      "rewards/rejected": -0.011852090246975422,
+      "step": 1810
+    },
+    {
+      "epoch": 0.6271536871123363,
+      "grad_norm": 1.6915125846862793,
+      "learning_rate": 2.892707644162246e-08,
+      "logits/chosen": -3.107539653778076,
+      "logits/rejected": -3.096742868423462,
+      "logps/chosen": -53.7323112487793,
+      "logps/rejected": -57.218109130859375,
+      "loss": 0.69,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.006404039915651083,
+      "rewards/margins": 0.006453787442296743,
+      "rewards/rejected": -0.012857827357947826,
+      "step": 1820
+    },
+    {
+      "epoch": 0.6305995864920745,
+      "grad_norm": 1.7556084394454956,
+      "learning_rate": 2.890462637747385e-08,
+      "logits/chosen": -3.06276273727417,
+      "logits/rejected": -3.0504953861236572,
+      "logps/chosen": -53.11023712158203,
+      "logps/rejected": -55.1548957824707,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.006401040591299534,
+      "rewards/margins": 0.004786512814462185,
+      "rewards/rejected": -0.011187553405761719,
+      "step": 1830
+    },
+    {
+      "epoch": 0.6340454858718125,
+      "grad_norm": 1.5613744258880615,
+      "learning_rate": 2.8881952760333842e-08,
+      "logits/chosen": -3.0585200786590576,
+      "logits/rejected": -3.023749828338623,
+      "logps/chosen": -54.3676643371582,
+      "logps/rejected": -52.61095428466797,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0060415551997721195,
+      "rewards/margins": 0.006825410760939121,
+      "rewards/rejected": -0.012866966426372528,
+      "step": 1840
+    },
+    {
+      "epoch": 0.6374913852515507,
+      "grad_norm": 1.6058069467544556,
+      "learning_rate": 2.8859055954739726e-08,
+      "logits/chosen": -3.024113655090332,
+      "logits/rejected": -3.015065908432007,
+      "logps/chosen": -55.3964729309082,
+      "logps/rejected": -60.47266387939453,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.005508395843207836,
+      "rewards/margins": 0.006812488194555044,
+      "rewards/rejected": -0.012320883572101593,
+      "step": 1850
+    },
+    {
+      "epoch": 0.6409372846312887,
+      "grad_norm": 1.6921290159225464,
+      "learning_rate": 2.883593632881714e-08,
+      "logits/chosen": -3.087735414505005,
+      "logits/rejected": -3.0622169971466064,
+      "logps/chosen": -53.47019577026367,
+      "logps/rejected": -53.597084045410156,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.005874739959836006,
+      "rewards/margins": 0.008003123104572296,
+      "rewards/rejected": -0.013877863064408302,
+      "step": 1860
+    },
+    {
+      "epoch": 0.6443831840110269,
+      "grad_norm": 1.7478374242782593,
+      "learning_rate": 2.8812594254274137e-08,
+      "logits/chosen": -3.0546376705169678,
+      "logits/rejected": -3.0308868885040283,
+      "logps/chosen": -54.71067428588867,
+      "logps/rejected": -54.864295959472656,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.007343374192714691,
+      "rewards/margins": 0.005724083632230759,
+      "rewards/rejected": -0.013067456893622875,
+      "step": 1870
+    },
+    {
+      "epoch": 0.647829083390765,
+      "grad_norm": 1.6382542848587036,
+      "learning_rate": 2.87890301063952e-08,
+      "logits/chosen": -3.10436749458313,
+      "logits/rejected": -3.088474750518799,
+      "logps/chosen": -55.34270477294922,
+      "logps/rejected": -55.82938766479492,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.007465527858585119,
+      "rewards/margins": 0.005134102888405323,
+      "rewards/rejected": -0.012599630281329155,
+      "step": 1880
+    },
+    {
+      "epoch": 0.6512749827705031,
+      "grad_norm": 1.533689022064209,
+      "learning_rate": 2.8765244264035226e-08,
+      "logits/chosen": -3.0730273723602295,
+      "logits/rejected": -3.0429303646087646,
+      "logps/chosen": -54.48848342895508,
+      "logps/rejected": -53.28242111206055,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.006783283315598965,
+      "rewards/margins": 0.006285698153078556,
+      "rewards/rejected": -0.013068979606032372,
+      "step": 1890
+    },
+    {
+      "epoch": 0.6547208821502413,
+      "grad_norm": 1.919150471687317,
+      "learning_rate": 2.874123710961342e-08,
+      "logits/chosen": -3.0587804317474365,
+      "logits/rejected": -3.0278632640838623,
+      "logps/chosen": -56.523887634277344,
+      "logps/rejected": -53.204322814941406,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.006831943988800049,
+      "rewards/margins": 0.005919927731156349,
+      "rewards/rejected": -0.012751871719956398,
+      "step": 1900
+    },
+    {
+      "epoch": 0.6547208821502413,
+      "eval_logits/chosen": -3.1493473052978516,
+      "eval_logits/rejected": -3.1437480449676514,
+      "eval_logps/chosen": -58.52531814575195,
+      "eval_logps/rejected": -63.18954086303711,
+      "eval_loss": 0.6921970248222351,
+      "eval_rewards/accuracies": 0.5720260143280029,
+      "eval_rewards/chosen": 0.0018657483160495758,
+      "eval_rewards/margins": 0.0019599366933107376,
+      "eval_rewards/rejected": -9.418852278031409e-05,
+      "eval_runtime": 383.0569,
+      "eval_samples_per_second": 11.236,
+      "eval_steps_per_second": 1.404,
+      "step": 1900
+    },
+    {
+      "epoch": 0.6581667815299793,
+      "grad_norm": 1.6474887132644653,
+      "learning_rate": 2.8717009029107154e-08,
+      "logits/chosen": -3.0777201652526855,
+      "logits/rejected": -3.0466856956481934,
+      "logps/chosen": -56.33563232421875,
+      "logps/rejected": -53.8904914855957,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.005524791777133942,
+      "rewards/margins": 0.007622270379215479,
+      "rewards/rejected": -0.013147061690688133,
+      "step": 1910
+    },
+    {
+      "epoch": 0.6616126809097175,
+      "grad_norm": 1.602634072303772,
+      "learning_rate": 2.8692560412045768e-08,
+      "logits/chosen": -3.0690441131591797,
+      "logits/rejected": -3.039360761642456,
+      "logps/chosen": -56.79243087768555,
+      "logps/rejected": -56.49236297607422,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0041068424470722675,
+      "rewards/margins": 0.008856844156980515,
+      "rewards/rejected": -0.01296368520706892,
+      "step": 1920
+    },
+    {
+      "epoch": 0.6650585802894555,
+      "grad_norm": 1.874599814414978,
+      "learning_rate": 2.8667891651504296e-08,
+      "logits/chosen": -3.163945198059082,
+      "logits/rejected": -3.142845630645752,
+      "logps/chosen": -56.51337432861328,
+      "logps/rejected": -54.66511154174805,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.007726091891527176,
+      "rewards/margins": 0.00538170849904418,
+      "rewards/rejected": -0.013107800856232643,
+      "step": 1930
+    },
+    {
+      "epoch": 0.6685044796691937,
+      "grad_norm": 1.7147718667984009,
+      "learning_rate": 2.864300314409715e-08,
+      "logits/chosen": -3.0739340782165527,
+      "logits/rejected": -3.063815116882324,
+      "logps/chosen": -55.67998504638672,
+      "logps/rejected": -55.72857666015625,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.559374988079071,
+      "rewards/chosen": -0.008140677586197853,
+      "rewards/margins": 0.0033669457770884037,
+      "rewards/rejected": -0.011507624760270119,
+      "step": 1940
+    },
+    {
+      "epoch": 0.6719503790489317,
+      "grad_norm": 1.6882637739181519,
+      "learning_rate": 2.8617895289971742e-08,
+      "logits/chosen": -3.1226413249969482,
+      "logits/rejected": -3.080296039581299,
+      "logps/chosen": -56.050498962402344,
+      "logps/rejected": -52.656578063964844,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0054300702176988125,
+      "rewards/margins": 0.009088740684092045,
+      "rewards/rejected": -0.01451881229877472,
+      "step": 1950
+    },
+    {
+      "epoch": 0.6753962784286699,
+      "grad_norm": 1.6859049797058105,
+      "learning_rate": 2.859256849280205e-08,
+      "logits/chosen": -3.120871067047119,
+      "logits/rejected": -3.0894556045532227,
+      "logps/chosen": -57.80708694458008,
+      "logps/rejected": -54.033119201660156,
+      "loss": 0.689,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.006516996771097183,
+      "rewards/margins": 0.008544852957129478,
+      "rewards/rejected": -0.015061849728226662,
+      "step": 1960
+    },
+    {
+      "epoch": 0.6788421778084079,
+      "grad_norm": 1.8792203664779663,
+      "learning_rate": 2.8567023159782142e-08,
+      "logits/chosen": -3.068342685699463,
+      "logits/rejected": -3.0561721324920654,
+      "logps/chosen": -52.9785041809082,
+      "logps/rejected": -54.5283203125,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.005995939020067453,
+      "rewards/margins": 0.006886272691190243,
+      "rewards/rejected": -0.012882212176918983,
+      "step": 1970
+    },
+    {
+      "epoch": 0.6822880771881461,
+      "grad_norm": 1.8789464235305786,
+      "learning_rate": 2.8541259701619605e-08,
+      "logits/chosen": -3.0346813201904297,
+      "logits/rejected": -3.0116920471191406,
+      "logps/chosen": -54.796600341796875,
+      "logps/rejected": -56.61003875732422,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.00708343181759119,
+      "rewards/margins": 0.007308699190616608,
+      "rewards/rejected": -0.014392131939530373,
+      "step": 1980
+    },
+    {
+      "epoch": 0.6857339765678843,
+      "grad_norm": 1.5827641487121582,
+      "learning_rate": 2.8515278532528962e-08,
+      "logits/chosen": -3.03825044631958,
+      "logits/rejected": -3.016231060028076,
+      "logps/chosen": -54.58489227294922,
+      "logps/rejected": -54.53600311279297,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.008786956779658794,
+      "rewards/margins": 0.0055664218962192535,
+      "rewards/rejected": -0.014353379607200623,
+      "step": 1990
+    },
+    {
+      "epoch": 0.6891798759476223,
+      "grad_norm": 1.7519923448562622,
+      "learning_rate": 2.8489080070224994e-08,
+      "logits/chosen": -3.124659776687622,
+      "logits/rejected": -3.0998432636260986,
+      "logps/chosen": -55.90535354614258,
+      "logps/rejected": -55.45905685424805,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.006949001457542181,
+      "rewards/margins": 0.007495499216020107,
+      "rewards/rejected": -0.014444500207901001,
+      "step": 2000
+    },
+    {
+      "epoch": 0.6891798759476223,
+      "eval_logits/chosen": -3.147533416748047,
+      "eval_logits/rejected": -3.141845226287842,
+      "eval_logps/chosen": -58.547149658203125,
+      "eval_logps/rejected": -63.25021743774414,
+      "eval_loss": 0.692008376121521,
+      "eval_rewards/accuracies": 0.5794609785079956,
+      "eval_rewards/chosen": 0.00164742860943079,
+      "eval_rewards/margins": 0.002348378300666809,
+      "eval_rewards/rejected": -0.0007009496912360191,
+      "eval_runtime": 383.1439,
+      "eval_samples_per_second": 11.233,
+      "eval_steps_per_second": 1.404,
+      "step": 2000
+    },
+    {
+      "epoch": 0.6926257753273605,
+      "grad_norm": 1.7864019870758057,
+      "learning_rate": 2.8462664735916055e-08,
+      "logits/chosen": -3.046966075897217,
+      "logits/rejected": -3.035414934158325,
+      "logps/chosen": -50.98776626586914,
+      "logps/rejected": -54.61168670654297,
+      "loss": 0.69,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.008853059262037277,
+      "rewards/margins": 0.006547768600285053,
+      "rewards/rejected": -0.015400826930999756,
+      "step": 2010
+    },
+    {
+      "epoch": 0.6960716747070985,
+      "grad_norm": 1.801180124282837,
+      "learning_rate": 2.843603295429726e-08,
+      "logits/chosen": -3.0936148166656494,
+      "logits/rejected": -3.076673984527588,
+      "logps/chosen": -56.879241943359375,
+      "logps/rejected": -56.220489501953125,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.007561421487480402,
+      "rewards/margins": 0.0054386211559176445,
+      "rewards/rejected": -0.013000041246414185,
+      "step": 2020
+    },
+    {
+      "epoch": 0.6995175740868367,
+      "grad_norm": 1.6343004703521729,
+      "learning_rate": 2.8409185153543683e-08,
+      "logits/chosen": -3.0453267097473145,
+      "logits/rejected": -3.020216464996338,
+      "logps/chosen": -54.402381896972656,
+      "logps/rejected": -54.2779655456543,
+      "loss": 0.689,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.009398328140377998,
+      "rewards/margins": 0.008563392795622349,
+      "rewards/rejected": -0.017961720004677773,
+      "step": 2030
+    },
+    {
+      "epoch": 0.7029634734665747,
+      "grad_norm": 1.905030608177185,
+      "learning_rate": 2.8382121765303474e-08,
+      "logits/chosen": -3.039381742477417,
+      "logits/rejected": -3.024355411529541,
+      "logps/chosen": -53.85132598876953,
+      "logps/rejected": -55.04918670654297,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.009854688309133053,
+      "rewards/margins": 0.006136411800980568,
+      "rewards/rejected": -0.015991101041436195,
+      "step": 2040
+    },
+    {
+      "epoch": 0.7064093728463129,
+      "grad_norm": 1.568685531616211,
+      "learning_rate": 2.83548432246909e-08,
+      "logits/chosen": -3.083832263946533,
+      "logits/rejected": -3.053781509399414,
+      "logps/chosen": -54.89458084106445,
+      "logps/rejected": -53.5596923828125,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.00734227430075407,
+      "rewards/margins": 0.009402116760611534,
+      "rewards/rejected": -0.01674439199268818,
+      "step": 2050
+    },
+    {
+      "epoch": 0.709855272226051,
+      "grad_norm": 1.8152940273284912,
+      "learning_rate": 2.8327349970279373e-08,
+      "logits/chosen": -3.0100879669189453,
+      "logits/rejected": -3.005795955657959,
+      "logps/chosen": -54.128334045410156,
+      "logps/rejected": -54.788246154785156,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.011146822944283485,
+      "rewards/margins": 0.005282334052026272,
+      "rewards/rejected": -0.016429157927632332,
+      "step": 2060
+    },
+    {
+      "epoch": 0.7133011716057891,
+      "grad_norm": 1.6905978918075562,
+      "learning_rate": 2.8299642444094375e-08,
+      "logits/chosen": -3.0924534797668457,
+      "logits/rejected": -3.066502332687378,
+      "logps/chosen": -55.53911209106445,
+      "logps/rejected": -53.181304931640625,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": -0.007923191413283348,
+      "rewards/margins": 0.007709300611168146,
+      "rewards/rejected": -0.015632491558790207,
+      "step": 2070
+    },
+    {
+      "epoch": 0.7167470709855273,
+      "grad_norm": 1.5745396614074707,
+      "learning_rate": 2.8271721091606375e-08,
+      "logits/chosen": -3.061793804168701,
+      "logits/rejected": -3.0323164463043213,
+      "logps/chosen": -55.7205810546875,
+      "logps/rejected": -52.4936637878418,
+      "loss": 0.688,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.00960648525506258,
+      "rewards/margins": 0.010651672258973122,
+      "rewards/rejected": -0.020258156582713127,
+      "step": 2080
+    },
+    {
+      "epoch": 0.7201929703652653,
+      "grad_norm": 1.6268614530563354,
+      "learning_rate": 2.8243586361723648e-08,
+      "logits/chosen": -3.060872793197632,
+      "logits/rejected": -3.0494332313537598,
+      "logps/chosen": -54.28404998779297,
+      "logps/rejected": -56.351348876953125,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.00913246814161539,
+      "rewards/margins": 0.005009587854146957,
+      "rewards/rejected": -0.014142056927084923,
+      "step": 2090
+    },
+    {
+      "epoch": 0.7236388697450035,
+      "grad_norm": 1.8515605926513672,
+      "learning_rate": 2.8215238706785065e-08,
+      "logits/chosen": -3.0667924880981445,
+      "logits/rejected": -3.0514025688171387,
+      "logps/chosen": -54.150611877441406,
+      "logps/rejected": -54.157676696777344,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.007633053697645664,
+      "rewards/margins": 0.008397385478019714,
+      "rewards/rejected": -0.016030440106987953,
+      "step": 2100
+    },
+    {
+      "epoch": 0.7236388697450035,
+      "eval_logits/chosen": -3.1450421810150146,
+      "eval_logits/rejected": -3.139359951019287,
+      "eval_logps/chosen": -58.54225540161133,
+      "eval_logps/rejected": -63.27001190185547,
+      "eval_loss": 0.691891610622406,
+      "eval_rewards/accuracies": 0.5817843675613403,
+      "eval_rewards/chosen": 0.001696433057077229,
+      "eval_rewards/margins": 0.002595358993858099,
+      "eval_rewards/rejected": -0.0008989257621578872,
+      "eval_runtime": 383.0825,
+      "eval_samples_per_second": 11.235,
+      "eval_steps_per_second": 1.404,
+      "step": 2100
+    },
+    {
+      "epoch": 0.7270847691247415,
+      "grad_norm": 1.6765116453170776,
+      "learning_rate": 2.818667858255283e-08,
+      "logits/chosen": -3.000230312347412,
+      "logits/rejected": -2.9789202213287354,
+      "logps/chosen": -54.828392028808594,
+      "logps/rejected": -54.66155242919922,
+      "loss": 0.689,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.010706872679293156,
+      "rewards/margins": 0.008508158847689629,
+      "rewards/rejected": -0.01921503059566021,
+      "step": 2110
+    },
+    {
+      "epoch": 0.7305306685044797,
+      "grad_norm": 1.6759587526321411,
+      "learning_rate": 2.815790644820513e-08,
+      "logits/chosen": -3.06356143951416,
+      "logits/rejected": -3.0408058166503906,
+      "logps/chosen": -55.93558883666992,
+      "logps/rejected": -56.97571563720703,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.009105021134018898,
+      "rewards/margins": 0.010383954271674156,
+      "rewards/rejected": -0.019488973543047905,
+      "step": 2120
+    },
+    {
+      "epoch": 0.7339765678842178,
+      "grad_norm": 1.6780146360397339,
+      "learning_rate": 2.812892276632877e-08,
+      "logits/chosen": -3.0177605152130127,
+      "logits/rejected": -3.001660108566284,
+      "logps/chosen": -55.7459716796875,
+      "logps/rejected": -55.48844528198242,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0095748919993639,
+      "rewards/margins": 0.007698676083236933,
+      "rewards/rejected": -0.017273565754294395,
+      "step": 2130
+    },
+    {
+      "epoch": 0.7374224672639559,
+      "grad_norm": 1.6097902059555054,
+      "learning_rate": 2.809972800291174e-08,
+      "logits/chosen": -3.005211114883423,
+      "logits/rejected": -2.978876829147339,
+      "logps/chosen": -52.527076721191406,
+      "logps/rejected": -53.1186637878418,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.008640892803668976,
+      "rewards/margins": 0.009533102624118328,
+      "rewards/rejected": -0.01817399449646473,
+      "step": 2140
+    },
+    {
+      "epoch": 0.740868366643694,
+      "grad_norm": 1.632975459098816,
+      "learning_rate": 2.8070322627335695e-08,
+      "logits/chosen": -3.052849531173706,
+      "logits/rejected": -3.0351340770721436,
+      "logps/chosen": -54.9369010925293,
+      "logps/rejected": -54.140380859375,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.010388058610260487,
+      "rewards/margins": 0.007618204690515995,
+      "rewards/rejected": -0.018006261438131332,
+      "step": 2150
+    },
+    {
+      "epoch": 0.7443142660234321,
+      "grad_norm": 1.5164223909378052,
+      "learning_rate": 2.8040707112368445e-08,
+      "logits/chosen": -3.0478174686431885,
+      "logits/rejected": -3.0239357948303223,
+      "logps/chosen": -57.22887420654297,
+      "logps/rejected": -54.8706169128418,
+      "loss": 0.689,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.008819004520773888,
+      "rewards/margins": 0.00856473296880722,
+      "rewards/rejected": -0.017383737489581108,
+      "step": 2160
+    },
+    {
+      "epoch": 0.7477601654031703,
+      "grad_norm": 1.519369125366211,
+      "learning_rate": 2.8010881934156328e-08,
+      "logits/chosen": -2.9943318367004395,
+      "logits/rejected": -2.9687514305114746,
+      "logps/chosen": -53.63547897338867,
+      "logps/rejected": -52.30000686645508,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.013919174671173096,
+      "rewards/margins": 0.007626353297382593,
+      "rewards/rejected": -0.0215455275028944,
+      "step": 2170
+    },
+    {
+      "epoch": 0.7512060647829083,
+      "grad_norm": 1.690244197845459,
+      "learning_rate": 2.7980847572216564e-08,
+      "logits/chosen": -3.0349280834198,
+      "logits/rejected": -3.0168070793151855,
+      "logps/chosen": -53.4122314453125,
+      "logps/rejected": -54.22038650512695,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.010714937932789326,
+      "rewards/margins": 0.007258834782987833,
+      "rewards/rejected": -0.017973775044083595,
+      "step": 2180
+    },
+    {
+      "epoch": 0.7546519641626465,
+      "grad_norm": 1.5385255813598633,
+      "learning_rate": 2.795060450942955e-08,
+      "logits/chosen": -3.0832526683807373,
+      "logits/rejected": -3.063964366912842,
+      "logps/chosen": -55.5803337097168,
+      "logps/rejected": -55.1922492980957,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.009492478333413601,
+      "rewards/margins": 0.007539073470979929,
+      "rewards/rejected": -0.017031552270054817,
+      "step": 2190
+    },
+    {
+      "epoch": 0.7580978635423845,
+      "grad_norm": 1.5867794752120972,
+      "learning_rate": 2.792015323203109e-08,
+      "logits/chosen": -3.0289359092712402,
+      "logits/rejected": -3.0132575035095215,
+      "logps/chosen": -56.235137939453125,
+      "logps/rejected": -56.1661262512207,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.011641815304756165,
+      "rewards/margins": 0.005428203381597996,
+      "rewards/rejected": -0.017070019617676735,
+      "step": 2200
+    },
+    {
+      "epoch": 0.7580978635423845,
+      "eval_logits/chosen": -3.143202543258667,
+      "eval_logits/rejected": -3.137552261352539,
+      "eval_logps/chosen": -58.58650207519531,
+      "eval_logps/rejected": -63.33800506591797,
+      "eval_loss": 0.6917803883552551,
+      "eval_rewards/accuracies": 0.5736523866653442,
+      "eval_rewards/chosen": 0.0012538820737972856,
+      "eval_rewards/margins": 0.0028327065519988537,
+      "eval_rewards/rejected": -0.0015788244782015681,
+      "eval_runtime": 383.2069,
+      "eval_samples_per_second": 11.232,
+      "eval_steps_per_second": 1.404,
+      "step": 2200
+    },
+    {
+      "epoch": 0.7615437629221227,
+      "grad_norm": 1.5717720985412598,
+      "learning_rate": 2.7889494229604574e-08,
+      "logits/chosen": -3.0646491050720215,
+      "logits/rejected": -3.0340261459350586,
+      "logps/chosen": -55.80480194091797,
+      "logps/rejected": -54.85691452026367,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.009509737603366375,
+      "rewards/margins": 0.011556681245565414,
+      "rewards/rejected": -0.021066417917609215,
+      "step": 2210
+    },
+    {
+      "epoch": 0.7649896623018608,
+      "grad_norm": 1.705826997756958,
+      "learning_rate": 2.785862799507311e-08,
+      "logits/chosen": -3.0503382682800293,
+      "logits/rejected": -3.018540620803833,
+      "logps/chosen": -57.812110900878906,
+      "logps/rejected": -55.095420837402344,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.012197548523545265,
+      "rewards/margins": 0.007037202827632427,
+      "rewards/rejected": -0.019234752282500267,
+      "step": 2220
+    },
+    {
+      "epoch": 0.7684355616815989,
+      "grad_norm": 1.6724340915679932,
+      "learning_rate": 2.7827555024691606e-08,
+      "logits/chosen": -3.0132381916046143,
+      "logits/rejected": -3.0029046535491943,
+      "logps/chosen": -55.5567512512207,
+      "logps/rejected": -55.328269958496094,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0133312763646245,
+      "rewards/margins": 0.005717568565160036,
+      "rewards/rejected": -0.019048845395445824,
+      "step": 2230
+    },
+    {
+      "epoch": 0.771881461061337,
+      "grad_norm": 1.5314977169036865,
+      "learning_rate": 2.779627581803879e-08,
+      "logits/chosen": -3.119142770767212,
+      "logits/rejected": -3.1006906032562256,
+      "logps/chosen": -54.72248077392578,
+      "logps/rejected": -53.366493225097656,
+      "loss": 0.689,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.012560924515128136,
+      "rewards/margins": 0.008677699603140354,
+      "rewards/rejected": -0.021238626912236214,
+      "step": 2240
+    },
+    {
+      "epoch": 0.7753273604410751,
+      "grad_norm": 1.8275576829910278,
+      "learning_rate": 2.7764790878009167e-08,
+      "logits/chosen": -3.0306878089904785,
+      "logits/rejected": -3.0095317363739014,
+      "logps/chosen": -54.93732452392578,
+      "logps/rejected": -53.3547248840332,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.011992533691227436,
+      "rewards/margins": 0.007551254238933325,
+      "rewards/rejected": -0.019543787464499474,
+      "step": 2250
+    },
+    {
+      "epoch": 0.7787732598208132,
+      "grad_norm": 1.6525906324386597,
+      "learning_rate": 2.7733100710804933e-08,
+      "logits/chosen": -3.1169586181640625,
+      "logits/rejected": -3.092015504837036,
+      "logps/chosen": -54.20165252685547,
+      "logps/rejected": -54.83784103393555,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.011864202097058296,
+      "rewards/margins": 0.008022557944059372,
+      "rewards/rejected": -0.019886760041117668,
+      "step": 2260
+    },
+    {
+      "epoch": 0.7822191592005513,
+      "grad_norm": 1.820604681968689,
+      "learning_rate": 2.7701205825927858e-08,
+      "logits/chosen": -3.0310654640197754,
+      "logits/rejected": -3.0206122398376465,
+      "logps/chosen": -54.566078186035156,
+      "logps/rejected": -54.23729705810547,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.013129832223057747,
+      "rewards/margins": 0.007158802356570959,
+      "rewards/rejected": -0.020288635045289993,
+      "step": 2270
+    },
+    {
+      "epoch": 0.7856650585802895,
+      "grad_norm": 1.762580156326294,
+      "learning_rate": 2.7669106736171083e-08,
+      "logits/chosen": -3.039483070373535,
+      "logits/rejected": -3.008972644805908,
+      "logps/chosen": -57.1434211730957,
+      "logps/rejected": -55.221717834472656,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.008556559681892395,
+      "rewards/margins": 0.011257710866630077,
+      "rewards/rejected": -0.019814269617199898,
+      "step": 2280
+    },
+    {
+      "epoch": 0.7891109579600276,
+      "grad_norm": 1.692659854888916,
+      "learning_rate": 2.7636803957610853e-08,
+      "logits/chosen": -3.002232074737549,
+      "logits/rejected": -2.9761745929718018,
+      "logps/chosen": -53.7590446472168,
+      "logps/rejected": -53.743690490722656,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.010840626433491707,
+      "rewards/margins": 0.009021765552461147,
+      "rewards/rejected": -0.01986239291727543,
+      "step": 2290
+    },
+    {
+      "epoch": 0.7925568573397657,
+      "grad_norm": 1.5440070629119873,
+      "learning_rate": 2.7604298009598256e-08,
+      "logits/chosen": -3.0259952545166016,
+      "logits/rejected": -3.001887798309326,
+      "logps/chosen": -55.53232955932617,
+      "logps/rejected": -55.49774169921875,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.014076339080929756,
+      "rewards/margins": 0.00811396911740303,
+      "rewards/rejected": -0.022190308198332787,
+      "step": 2300
+    },
+    {
+      "epoch": 0.7925568573397657,
+      "eval_logits/chosen": -3.1414272785186768,
+      "eval_logits/rejected": -3.1357762813568115,
+      "eval_logps/chosen": -58.60091018676758,
+      "eval_logps/rejected": -63.376060485839844,
+      "eval_loss": 0.6916689872741699,
+      "eval_rewards/accuracies": 0.5729553699493408,
+      "eval_rewards/chosen": 0.001109847566112876,
+      "eval_rewards/margins": 0.00306925131008029,
+      "eval_rewards/rejected": -0.0019594039767980576,
+      "eval_runtime": 383.467,
+      "eval_samples_per_second": 11.224,
+      "eval_steps_per_second": 1.403,
+      "step": 2300
+    },
+    {
+      "epoch": 0.7960027567195038,
+      "grad_norm": 1.7879085540771484,
+      "learning_rate": 2.7571589414750856e-08,
+      "logits/chosen": -3.013345241546631,
+      "logits/rejected": -2.9962551593780518,
+      "logps/chosen": -54.762611389160156,
+      "logps/rejected": -55.41656494140625,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.01334276795387268,
+      "rewards/margins": 0.008525204844772816,
+      "rewards/rejected": -0.021867971867322922,
+      "step": 2310
+    },
+    {
+      "epoch": 0.7994486560992419,
+      "grad_norm": 1.718151569366455,
+      "learning_rate": 2.753867869894428e-08,
+      "logits/chosen": -3.0735602378845215,
+      "logits/rejected": -3.0488388538360596,
+      "logps/chosen": -57.336463928222656,
+      "logps/rejected": -58.2114372253418,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.013173140585422516,
+      "rewards/margins": 0.008210359141230583,
+      "rewards/rejected": -0.0213834997266531,
+      "step": 2320
+    },
+    {
+      "epoch": 0.80289455547898,
+      "grad_norm": 1.6745678186416626,
+      "learning_rate": 2.750556639130379e-08,
+      "logits/chosen": -3.067460060119629,
+      "logits/rejected": -3.0413782596588135,
+      "logps/chosen": -55.30506134033203,
+      "logps/rejected": -54.623779296875,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.01076031755656004,
+      "rewards/margins": 0.010189579799771309,
+      "rewards/rejected": -0.020949898287653923,
+      "step": 2330
+    },
+    {
+      "epoch": 0.8063404548587181,
+      "grad_norm": 1.546802282333374,
+      "learning_rate": 2.747225302419575e-08,
+      "logits/chosen": -3.0492866039276123,
+      "logits/rejected": -3.020920753479004,
+      "logps/chosen": -53.70014572143555,
+      "logps/rejected": -52.5570182800293,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.012411813251674175,
+      "rewards/margins": 0.009221012704074383,
+      "rewards/rejected": -0.021632825955748558,
+      "step": 2340
+    },
+    {
+      "epoch": 0.8097863542384562,
+      "grad_norm": 1.7089766263961792,
+      "learning_rate": 2.7438739133219083e-08,
+      "logits/chosen": -3.0722365379333496,
+      "logits/rejected": -3.057375907897949,
+      "logps/chosen": -55.09632110595703,
+      "logps/rejected": -57.072547912597656,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.013475869782269001,
+      "rewards/margins": 0.007299723569303751,
+      "rewards/rejected": -0.02077559381723404,
+      "step": 2350
+    },
+    {
+      "epoch": 0.8132322536181944,
+      "grad_norm": 1.676021933555603,
+      "learning_rate": 2.7405025257196645e-08,
+      "logits/chosen": -3.0481390953063965,
+      "logits/rejected": -3.0131804943084717,
+      "logps/chosen": -58.44108200073242,
+      "logps/rejected": -54.808509826660156,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.010662969201803207,
+      "rewards/margins": 0.013338451273739338,
+      "rewards/rejected": -0.02400142326951027,
+      "step": 2360
+    },
+    {
+      "epoch": 0.8166781529979324,
+      "grad_norm": 1.8042562007904053,
+      "learning_rate": 2.7371111938166578e-08,
+      "logits/chosen": -3.0210914611816406,
+      "logits/rejected": -3.002652883529663,
+      "logps/chosen": -55.331512451171875,
+      "logps/rejected": -54.533470153808594,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.012376179918646812,
+      "rewards/margins": 0.007489274255931377,
+      "rewards/rejected": -0.019865456968545914,
+      "step": 2370
+    },
+    {
+      "epoch": 0.8201240523776706,
+      "grad_norm": 1.8038052320480347,
+      "learning_rate": 2.7336999721373596e-08,
+      "logits/chosen": -3.0341835021972656,
+      "logits/rejected": -3.0135598182678223,
+      "logps/chosen": -55.100624084472656,
+      "logps/rejected": -55.18485641479492,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.011685562320053577,
+      "rewards/margins": 0.009405437856912613,
+      "rewards/rejected": -0.021090997382998466,
+      "step": 2380
+    },
+    {
+      "epoch": 0.8235699517574087,
+      "grad_norm": 1.809814214706421,
+      "learning_rate": 2.7302689155260193e-08,
+      "logits/chosen": -3.034025192260742,
+      "logits/rejected": -3.0140433311462402,
+      "logps/chosen": -56.813377380371094,
+      "logps/rejected": -56.7078742980957,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.012832743115723133,
+      "rewards/margins": 0.008531561121344566,
+      "rewards/rejected": -0.021364305168390274,
+      "step": 2390
+    },
+    {
+      "epoch": 0.8270158511371468,
+      "grad_norm": 1.7898904085159302,
+      "learning_rate": 2.7268180791457856e-08,
+      "logits/chosen": -3.03529691696167,
+      "logits/rejected": -3.0144174098968506,
+      "logps/chosen": -56.322471618652344,
+      "logps/rejected": -53.683677673339844,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.014290323480963707,
+      "rewards/margins": 0.006967701017856598,
+      "rewards/rejected": -0.021258026361465454,
+      "step": 2400
+    },
+    {
+      "epoch": 0.8270158511371468,
+      "eval_logits/chosen": -3.1394083499908447,
+      "eval_logits/rejected": -3.133765459060669,
+      "eval_logps/chosen": -58.653778076171875,
+      "eval_logps/rejected": -63.45909881591797,
+      "eval_loss": 0.6915270686149597,
+      "eval_rewards/accuracies": 0.5764405131340027,
+      "eval_rewards/chosen": 0.000581158499699086,
+      "eval_rewards/margins": 0.0033709816634655,
+      "eval_rewards/rejected": -0.0027898226398974657,
+      "eval_runtime": 383.0617,
+      "eval_samples_per_second": 11.236,
+      "eval_steps_per_second": 1.404,
+      "step": 2400
+    },
+    {
+      "epoch": 0.8304617505168849,
+      "grad_norm": 1.732565999031067,
+      "learning_rate": 2.7233475184778185e-08,
+      "logits/chosen": -3.026383399963379,
+      "logits/rejected": -3.003863573074341,
+      "logps/chosen": -55.51924514770508,
+      "logps/rejected": -54.477210998535156,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.013324117287993431,
+      "rewards/margins": 0.010489655658602715,
+      "rewards/rejected": -0.023813774809241295,
+      "step": 2410
+    },
+    {
+      "epoch": 0.833907649896623,
+      "grad_norm": 1.879536747932434,
+      "learning_rate": 2.7198572893203962e-08,
+      "logits/chosen": -3.032336711883545,
+      "logits/rejected": -3.0056231021881104,
+      "logps/chosen": -56.296226501464844,
+      "logps/rejected": -54.810699462890625,
+      "loss": 0.689,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.014226064085960388,
+      "rewards/margins": 0.00867706723511219,
+      "rewards/rejected": -0.02290312945842743,
+      "step": 2420
+    },
+    {
+      "epoch": 0.8373535492763611,
+      "grad_norm": 1.7612248659133911,
+      "learning_rate": 2.7163474477880205e-08,
+      "logits/chosen": -3.0923616886138916,
+      "logits/rejected": -3.0665535926818848,
+      "logps/chosen": -57.85490798950195,
+      "logps/rejected": -54.48235321044922,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.015593056567013264,
+      "rewards/margins": 0.008288629353046417,
+      "rewards/rejected": -0.023881684988737106,
+      "step": 2430
+    },
+    {
+      "epoch": 0.8407994486560992,
+      "grad_norm": 1.5588469505310059,
+      "learning_rate": 2.7128180503105116e-08,
+      "logits/chosen": -3.0545125007629395,
+      "logits/rejected": -3.023500919342041,
+      "logps/chosen": -56.5146598815918,
+      "logps/rejected": -53.4724006652832,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.01189456693828106,
+      "rewards/margins": 0.012264418415725231,
+      "rewards/rejected": -0.024158984422683716,
+      "step": 2440
+    },
+    {
+      "epoch": 0.8442453480358374,
+      "grad_norm": 1.5571277141571045,
+      "learning_rate": 2.7092691536321027e-08,
+      "logits/chosen": -3.0364913940429688,
+      "logits/rejected": -3.0087835788726807,
+      "logps/chosen": -55.713035583496094,
+      "logps/rejected": -54.43060302734375,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.014015443623065948,
+      "rewards/margins": 0.011403219774365425,
+      "rewards/rejected": -0.025418663397431374,
+      "step": 2450
+    },
+    {
+      "epoch": 0.8476912474155754,
+      "grad_norm": 1.794663667678833,
+      "learning_rate": 2.7057008148105274e-08,
+      "logits/chosen": -3.07753324508667,
+      "logits/rejected": -3.0524916648864746,
+      "logps/chosen": -56.4511604309082,
+      "logps/rejected": -55.62166213989258,
+      "loss": 0.6868,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.013616129755973816,
+      "rewards/margins": 0.013101039454340935,
+      "rewards/rejected": -0.0267171673476696,
+      "step": 2460
+    },
+    {
+      "epoch": 0.8511371467953136,
+      "grad_norm": 1.7949426174163818,
+      "learning_rate": 2.7021130912161025e-08,
+      "logits/chosen": -3.13236403465271,
+      "logits/rejected": -3.1115479469299316,
+      "logps/chosen": -56.294769287109375,
+      "logps/rejected": -58.44313430786133,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.01762278750538826,
+      "rewards/margins": 0.010658183135092258,
+      "rewards/rejected": -0.028280969709157944,
+      "step": 2470
+    },
+    {
+      "epoch": 0.8545830461750517,
+      "grad_norm": 1.6377383470535278,
+      "learning_rate": 2.6985060405308054e-08,
+      "logits/chosen": -2.95481538772583,
+      "logits/rejected": -2.9481287002563477,
+      "logps/chosen": -55.100929260253906,
+      "logps/rejected": -57.38971710205078,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.01560432929545641,
+      "rewards/margins": 0.00560566782951355,
+      "rewards/rejected": -0.021209994331002235,
+      "step": 2480
+    },
+    {
+      "epoch": 0.8580289455547898,
+      "grad_norm": 1.7566173076629639,
+      "learning_rate": 2.694879720747346e-08,
+      "logits/chosen": -3.0400028228759766,
+      "logits/rejected": -3.031888961791992,
+      "logps/chosen": -53.79430389404297,
+      "logps/rejected": -55.64051055908203,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.016788221895694733,
+      "rewards/margins": 0.005911668296903372,
+      "rewards/rejected": -0.022699888795614243,
+      "step": 2490
+    },
+    {
+      "epoch": 0.8614748449345279,
+      "grad_norm": 1.7388852834701538,
+      "learning_rate": 2.6912341901682364e-08,
+      "logits/chosen": -3.0010955333709717,
+      "logits/rejected": -2.987089157104492,
+      "logps/chosen": -55.31623458862305,
+      "logps/rejected": -56.1048469543457,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.015220506116747856,
+      "rewards/margins": 0.007865633815526962,
+      "rewards/rejected": -0.023086141794919968,
+      "step": 2500
+    },
+    {
+      "epoch": 0.8614748449345279,
+      "eval_logits/chosen": -3.1371681690216064,
+      "eval_logits/rejected": -3.1314775943756104,
+      "eval_logps/chosen": -58.693355560302734,
+      "eval_logps/rejected": -63.524539947509766,
+      "eval_loss": 0.6914054751396179,
+      "eval_rewards/accuracies": 0.5743494629859924,
+      "eval_rewards/chosen": 0.00018539321899879724,
+      "eval_rewards/margins": 0.003629608778283,
+      "eval_rewards/rejected": -0.0034442150499671698,
+      "eval_runtime": 383.2768,
+      "eval_samples_per_second": 11.229,
+      "eval_steps_per_second": 1.404,
+      "step": 2500
+    },
+    {
+      "epoch": 0.864920744314266,
+      "grad_norm": 1.7079205513000488,
+      "learning_rate": 2.6875695074048508e-08,
+      "logits/chosen": -3.0127339363098145,
+      "logits/rejected": -2.986274242401123,
+      "logps/chosen": -54.8475456237793,
+      "logps/rejected": -54.299415588378906,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.015724454075098038,
+      "rewards/margins": 0.011717638000845909,
+      "rewards/rejected": -0.027442093938589096,
+      "step": 2510
+    },
+    {
+      "epoch": 0.8683666436940042,
+      "grad_norm": 1.6540602445602417,
+      "learning_rate": 2.6838857313764847e-08,
+      "logits/chosen": -3.0741376876831055,
+      "logits/rejected": -3.0483016967773438,
+      "logps/chosen": -56.42127227783203,
+      "logps/rejected": -54.64488983154297,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.015339541248977184,
+      "rewards/margins": 0.009082010947167873,
+      "rewards/rejected": -0.024421552196145058,
+      "step": 2520
+    },
+    {
+      "epoch": 0.8718125430737422,
+      "grad_norm": 1.539483904838562,
+      "learning_rate": 2.6801829213094076e-08,
+      "logits/chosen": -2.988391637802124,
+      "logits/rejected": -2.973419189453125,
+      "logps/chosen": -54.72621536254883,
+      "logps/rejected": -55.950599670410156,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.01688106544315815,
+      "rewards/margins": 0.008914967067539692,
+      "rewards/rejected": -0.025796031579375267,
+      "step": 2530
+    },
+    {
+      "epoch": 0.8752584424534804,
+      "grad_norm": 1.777274489402771,
+      "learning_rate": 2.6764611367359113e-08,
+      "logits/chosen": -3.1266283988952637,
+      "logits/rejected": -3.094198703765869,
+      "logps/chosen": -56.490806579589844,
+      "logps/rejected": -52.0720329284668,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.013644588179886341,
+      "rewards/margins": 0.011979375965893269,
+      "rewards/rejected": -0.02562396600842476,
+      "step": 2540
+    },
+    {
+      "epoch": 0.8787043418332184,
+      "grad_norm": 1.679472804069519,
+      "learning_rate": 2.6727204374933516e-08,
+      "logits/chosen": -3.006377935409546,
+      "logits/rejected": -2.972329616546631,
+      "logps/chosen": -56.1802864074707,
+      "logps/rejected": -54.93523025512695,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.015014847740530968,
+      "rewards/margins": 0.015222616493701935,
+      "rewards/rejected": -0.030237462371587753,
+      "step": 2550
+    },
+    {
+      "epoch": 0.8821502412129566,
+      "grad_norm": 1.6933256387710571,
+      "learning_rate": 2.668960883723186e-08,
+      "logits/chosen": -3.097569227218628,
+      "logits/rejected": -3.06351900100708,
+      "logps/chosen": -57.41898727416992,
+      "logps/rejected": -54.62512969970703,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.01545868068933487,
+      "rewards/margins": 0.01016304362565279,
+      "rewards/rejected": -0.025621721521019936,
+      "step": 2560
+    },
+    {
+      "epoch": 0.8855961405926946,
+      "grad_norm": 1.771246075630188,
+      "learning_rate": 2.6651825358700086e-08,
+      "logits/chosen": -3.099323272705078,
+      "logits/rejected": -3.0635647773742676,
+      "logps/chosen": -56.979820251464844,
+      "logps/rejected": -54.21015548706055,
+      "loss": 0.6861,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.012876259163022041,
+      "rewards/margins": 0.014484544284641743,
+      "rewards/rejected": -0.027360806241631508,
+      "step": 2570
+    },
+    {
+      "epoch": 0.8890420399724328,
+      "grad_norm": 1.7435131072998047,
+      "learning_rate": 2.6613854546805768e-08,
+      "logits/chosen": -3.080430746078491,
+      "logits/rejected": -3.057739734649658,
+      "logps/chosen": -54.0610466003418,
+      "logps/rejected": -54.35429763793945,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.017877351492643356,
+      "rewards/margins": 0.011237652972340584,
+      "rewards/rejected": -0.02911500260233879,
+      "step": 2580
+    },
+    {
+      "epoch": 0.892487939352171,
+      "grad_norm": 1.7916028499603271,
+      "learning_rate": 2.657569701202835e-08,
+      "logits/chosen": -3.1027426719665527,
+      "logits/rejected": -3.086071252822876,
+      "logps/chosen": -56.8061408996582,
+      "logps/rejected": -55.5204963684082,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.014563458040356636,
+      "rewards/margins": 0.00886179506778717,
+      "rewards/rejected": -0.023425253108143806,
+      "step": 2590
+    },
+    {
+      "epoch": 0.895933838731909,
+      "grad_norm": 1.7147051095962524,
+      "learning_rate": 2.6537353367849334e-08,
+      "logits/chosen": -3.0566537380218506,
+      "logits/rejected": -3.039797306060791,
+      "logps/chosen": -54.290252685546875,
+      "logps/rejected": -56.83160400390625,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.01830853894352913,
+      "rewards/margins": 0.010240554809570312,
+      "rewards/rejected": -0.02854909561574459,
+      "step": 2600
+    },
+    {
+      "epoch": 0.895933838731909,
+      "eval_logits/chosen": -3.1353888511657715,
+      "eval_logits/rejected": -3.1297316551208496,
+      "eval_logps/chosen": -58.74568557739258,
+      "eval_logps/rejected": -63.61227035522461,
+      "eval_loss": 0.6912384629249573,
+      "eval_rewards/accuracies": 0.5764405131340027,
+      "eval_rewards/chosen": -0.00033788851578719914,
+      "eval_rewards/margins": 0.003983610309660435,
+      "eval_rewards/rejected": -0.004321497865021229,
+      "eval_runtime": 383.4296,
+      "eval_samples_per_second": 11.225,
+      "eval_steps_per_second": 1.403,
+      "step": 2600
+    },
+    {
+      "epoch": 0.8993797381116472,
+      "grad_norm": 1.9461175203323364,
+      "learning_rate": 2.6498824230742407e-08,
+      "logits/chosen": -3.067842483520508,
+      "logits/rejected": -3.0541365146636963,
+      "logps/chosen": -53.832603454589844,
+      "logps/rejected": -55.621337890625,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.017074428498744965,
+      "rewards/margins": 0.009231159463524818,
+      "rewards/rejected": -0.026305589824914932,
+      "step": 2610
+    },
+    {
+      "epoch": 0.9028256374913852,
+      "grad_norm": 1.697290062904358,
+      "learning_rate": 2.6460110220163545e-08,
+      "logits/chosen": -3.0223097801208496,
+      "logits/rejected": -3.005430221557617,
+      "logps/chosen": -55.441749572753906,
+      "logps/rejected": -57.4183349609375,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.014238406904041767,
+      "rewards/margins": 0.012645892798900604,
+      "rewards/rejected": -0.026884302496910095,
+      "step": 2620
+    },
+    {
+      "epoch": 0.9062715368711234,
+      "grad_norm": 1.8861323595046997,
+      "learning_rate": 2.642121195854104e-08,
+      "logits/chosen": -3.0131638050079346,
+      "logits/rejected": -2.9852371215820312,
+      "logps/chosen": -55.69242477416992,
+      "logps/rejected": -53.9802360534668,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.01810877025127411,
+      "rewards/margins": 0.009742341935634613,
+      "rewards/rejected": -0.027851110324263573,
+      "step": 2630
+    },
+    {
+      "epoch": 0.9097174362508614,
+      "grad_norm": 1.7388513088226318,
+      "learning_rate": 2.63821300712655e-08,
+      "logits/chosen": -3.1097218990325928,
+      "logits/rejected": -3.0865743160247803,
+      "logps/chosen": -55.02528762817383,
+      "logps/rejected": -54.090065002441406,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.020634543150663376,
+      "rewards/margins": 0.011185268871486187,
+      "rewards/rejected": -0.03181981295347214,
+      "step": 2640
+    },
+    {
+      "epoch": 0.9131633356305996,
+      "grad_norm": 1.7347242832183838,
+      "learning_rate": 2.6342865186679795e-08,
+      "logits/chosen": -3.0091593265533447,
+      "logits/rejected": -2.9795501232147217,
+      "logps/chosen": -56.85670852661133,
+      "logps/rejected": -56.33771514892578,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.01379752904176712,
+      "rewards/margins": 0.013645144179463387,
+      "rewards/rejected": -0.027442673221230507,
+      "step": 2650
+    },
+    {
+      "epoch": 0.9166092350103378,
+      "grad_norm": 1.601115345954895,
+      "learning_rate": 2.6303417936068952e-08,
+      "logits/chosen": -3.1014761924743652,
+      "logits/rejected": -3.0682764053344727,
+      "logps/chosen": -59.339454650878906,
+      "logps/rejected": -54.093170166015625,
+      "loss": 0.687,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.014956450089812279,
+      "rewards/margins": 0.012698328122496605,
+      "rewards/rejected": -0.027654778212308884,
+      "step": 2660
+    },
+    {
+      "epoch": 0.9200551343900758,
+      "grad_norm": 1.8243662118911743,
+      "learning_rate": 2.6263788953649995e-08,
+      "logits/chosen": -3.014390707015991,
+      "logits/rejected": -2.989600419998169,
+      "logps/chosen": -55.28759002685547,
+      "logps/rejected": -55.3654899597168,
+      "loss": 0.687,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.016797643154859543,
+      "rewards/margins": 0.012854975648224354,
+      "rewards/rejected": -0.02965261973440647,
+      "step": 2670
+    },
+    {
+      "epoch": 0.923501033769814,
+      "grad_norm": 1.7097259759902954,
+      "learning_rate": 2.622397887656177e-08,
+      "logits/chosen": -3.0448696613311768,
+      "logits/rejected": -3.0230023860931396,
+      "logps/chosen": -54.8726692199707,
+      "logps/rejected": -57.26301956176758,
+      "loss": 0.688,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.01944391056895256,
+      "rewards/margins": 0.010921550914645195,
+      "rewards/rejected": -0.030365461483597755,
+      "step": 2680
+    },
+    {
+      "epoch": 0.926946933149552,
+      "grad_norm": 1.6671255826950073,
+      "learning_rate": 2.6183988344854695e-08,
+      "logits/chosen": -3.057912588119507,
+      "logits/rejected": -3.0290675163269043,
+      "logps/chosen": -56.00664138793945,
+      "logps/rejected": -55.487571716308594,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.013452179729938507,
+      "rewards/margins": 0.016251515597105026,
+      "rewards/rejected": -0.029703695327043533,
+      "step": 2690
+    },
+    {
+      "epoch": 0.9303928325292902,
+      "grad_norm": 1.6534792184829712,
+      "learning_rate": 2.614381800148045e-08,
+      "logits/chosen": -3.043503522872925,
+      "logits/rejected": -3.013298511505127,
+      "logps/chosen": -56.7213020324707,
+      "logps/rejected": -55.827125549316406,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.01884482242166996,
+      "rewards/margins": 0.011901089921593666,
+      "rewards/rejected": -0.030745912343263626,
+      "step": 2700
+    },
+    {
+      "epoch": 0.9303928325292902,
+      "eval_logits/chosen": -3.1338086128234863,
+      "eval_logits/rejected": -3.128178596496582,
+      "eval_logps/chosen": -58.81418991088867,
+      "eval_logps/rejected": -63.709659576416016,
+      "eval_loss": 0.6911041736602783,
+      "eval_rewards/accuracies": 0.578066885471344,
+      "eval_rewards/chosen": -0.0010229477193206549,
+      "eval_rewards/margins": 0.004272367339581251,
+      "eval_rewards/rejected": -0.00529531529173255,
+      "eval_runtime": 383.4926,
+      "eval_samples_per_second": 11.223,
+      "eval_steps_per_second": 1.403,
+      "step": 2700
+    },
+    {
+      "epoch": 0.9338387319090282,
+      "grad_norm": 1.699280858039856,
+      "learning_rate": 2.610346849228166e-08,
+      "logits/chosen": -3.095877170562744,
+      "logits/rejected": -3.078768014907837,
+      "logps/chosen": -56.982749938964844,
+      "logps/rejected": -56.65929412841797,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.01803940162062645,
+      "rewards/margins": 0.009354648180305958,
+      "rewards/rejected": -0.027394050732254982,
+      "step": 2710
+    },
+    {
+      "epoch": 0.9372846312887664,
+      "grad_norm": 1.7073124647140503,
+      "learning_rate": 2.6062940465981518e-08,
+      "logits/chosen": -3.047394275665283,
+      "logits/rejected": -3.0333359241485596,
+      "logps/chosen": -57.0693473815918,
+      "logps/rejected": -56.85771942138672,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.021695932373404503,
+      "rewards/margins": 0.011905960738658905,
+      "rewards/rejected": -0.03360189497470856,
+      "step": 2720
+    },
+    {
+      "epoch": 0.9407305306685044,
+      "grad_norm": 1.777449131011963,
+      "learning_rate": 2.6022234574173322e-08,
+      "logits/chosen": -3.0252881050109863,
+      "logits/rejected": -3.011453151702881,
+      "logps/chosen": -56.63299560546875,
+      "logps/rejected": -55.708038330078125,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.019217390567064285,
+      "rewards/margins": 0.010420707985758781,
+      "rewards/rejected": -0.029638096690177917,
+      "step": 2730
+    },
+    {
+      "epoch": 0.9441764300482426,
+      "grad_norm": 1.820221185684204,
+      "learning_rate": 2.5981351471310035e-08,
+      "logits/chosen": -3.0347018241882324,
+      "logits/rejected": -3.0120925903320312,
+      "logps/chosen": -54.34062957763672,
+      "logps/rejected": -54.43853759765625,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.021897148340940475,
+      "rewards/margins": 0.011848915368318558,
+      "rewards/rejected": -0.03374606743454933,
+      "step": 2740
+    },
+    {
+      "epoch": 0.9476223294279807,
+      "grad_norm": 1.577182650566101,
+      "learning_rate": 2.5940291814693755e-08,
+      "logits/chosen": -3.0675253868103027,
+      "logits/rejected": -3.0455355644226074,
+      "logps/chosen": -56.61127471923828,
+      "logps/rejected": -56.39185333251953,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.019232813268899918,
+      "rewards/margins": 0.010296612977981567,
+      "rewards/rejected": -0.029529426246881485,
+      "step": 2750
+    },
+    {
+      "epoch": 0.9510682288077188,
+      "grad_norm": 1.8613027334213257,
+      "learning_rate": 2.5899056264465114e-08,
+      "logits/chosen": -3.009302854537964,
+      "logits/rejected": -2.9880309104919434,
+      "logps/chosen": -56.20166778564453,
+      "logps/rejected": -54.97979736328125,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.01675734855234623,
+      "rewards/margins": 0.011660220101475716,
+      "rewards/rejected": -0.028417566791176796,
+      "step": 2760
+    },
+    {
+      "epoch": 0.954514128187457,
+      "grad_norm": 1.672033667564392,
+      "learning_rate": 2.585764548359271e-08,
+      "logits/chosen": -3.0462260246276855,
+      "logits/rejected": -3.031433582305908,
+      "logps/chosen": -57.62470626831055,
+      "logps/rejected": -57.719459533691406,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.014977472834289074,
+      "rewards/margins": 0.013343590311706066,
+      "rewards/rejected": -0.02832106314599514,
+      "step": 2770
+    },
+    {
+      "epoch": 0.957960027567195,
+      "grad_norm": 1.8654658794403076,
+      "learning_rate": 2.5816060137862434e-08,
+      "logits/chosen": -2.9718852043151855,
+      "logits/rejected": -2.950334072113037,
+      "logps/chosen": -54.41728591918945,
+      "logps/rejected": -55.060752868652344,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.021649938076734543,
+      "rewards/margins": 0.011123741045594215,
+      "rewards/rejected": -0.03277368098497391,
+      "step": 2780
+    },
+    {
+      "epoch": 0.9614059269469332,
+      "grad_norm": 1.5528416633605957,
+      "learning_rate": 2.5774300895866742e-08,
+      "logits/chosen": -3.0153703689575195,
+      "logits/rejected": -3.003854751586914,
+      "logps/chosen": -54.09788131713867,
+      "logps/rejected": -58.43914794921875,
+      "loss": 0.689,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0217584315687418,
+      "rewards/margins": 0.009019347839057446,
+      "rewards/rejected": -0.030777782201766968,
+      "step": 2790
+    },
+    {
+      "epoch": 0.9648518263266712,
+      "grad_norm": 1.6665146350860596,
+      "learning_rate": 2.573236842899394e-08,
+      "logits/chosen": -3.105117082595825,
+      "logits/rejected": -3.081604242324829,
+      "logps/chosen": -56.500526428222656,
+      "logps/rejected": -59.34136199951172,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.018329959362745285,
+      "rewards/margins": 0.012782074511051178,
+      "rewards/rejected": -0.031112033873796463,
+      "step": 2800
+    },
+    {
+      "epoch": 0.9648518263266712,
+      "eval_logits/chosen": -3.1317389011383057,
+      "eval_logits/rejected": -3.1260881423950195,
+      "eval_logps/chosen": -58.87005615234375,
+      "eval_logps/rejected": -63.78676986694336,
+      "eval_loss": 0.6910077333450317,
+      "eval_rewards/accuracies": 0.5759758353233337,
+      "eval_rewards/chosen": -0.001581624150276184,
+      "eval_rewards/margins": 0.0044848122633993626,
+      "eval_rewards/rejected": -0.006066435948014259,
+      "eval_runtime": 383.4692,
+      "eval_samples_per_second": 11.224,
+      "eval_steps_per_second": 1.403,
+      "step": 2800
+    },
+    {
+      "epoch": 0.9682977257064094,
+      "grad_norm": 1.802642822265625,
+      "learning_rate": 2.5690263411417367e-08,
+      "logits/chosen": -3.0609283447265625,
+      "logits/rejected": -3.040710926055908,
+      "logps/chosen": -55.67799758911133,
+      "logps/rejected": -56.90732955932617,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.019678916782140732,
+      "rewards/margins": 0.013977396301925182,
+      "rewards/rejected": -0.03365631029009819,
+      "step": 2810
+    },
+    {
+      "epoch": 0.9717436250861475,
+      "grad_norm": 1.886015772819519,
+      "learning_rate": 2.564798652008456e-08,
+      "logits/chosen": -3.0623276233673096,
+      "logits/rejected": -3.0405044555664062,
+      "logps/chosen": -56.510520935058594,
+      "logps/rejected": -56.427879333496094,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.015837090089917183,
+      "rewards/margins": 0.012199919670820236,
+      "rewards/rejected": -0.02803700603544712,
+      "step": 2820
+    },
+    {
+      "epoch": 0.9751895244658856,
+      "grad_norm": 1.751362681388855,
+      "learning_rate": 2.5605538434706384e-08,
+      "logits/chosen": -2.960659980773926,
+      "logits/rejected": -2.9560775756835938,
+      "logps/chosen": -54.369422912597656,
+      "logps/rejected": -54.67145919799805,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.024322327226400375,
+      "rewards/margins": 0.006370414048433304,
+      "rewards/rejected": -0.03069274127483368,
+      "step": 2830
+    },
+    {
+      "epoch": 0.9786354238456237,
+      "grad_norm": 1.7003294229507446,
+      "learning_rate": 2.5562919837746077e-08,
+      "logits/chosen": -3.0403616428375244,
+      "logits/rejected": -3.0088396072387695,
+      "logps/chosen": -56.176788330078125,
+      "logps/rejected": -55.7294807434082,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.019663270562887192,
+      "rewards/margins": 0.015215925872325897,
+      "rewards/rejected": -0.03487919643521309,
+      "step": 2840
+    },
+    {
+      "epoch": 0.9820813232253618,
+      "grad_norm": 1.7429810762405396,
+      "learning_rate": 2.5520131414408313e-08,
+      "logits/chosen": -2.9951140880584717,
+      "logits/rejected": -2.970262289047241,
+      "logps/chosen": -59.49003219604492,
+      "logps/rejected": -55.706764221191406,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.01597842015326023,
+      "rewards/margins": 0.01622283086180687,
+      "rewards/rejected": -0.03220125287771225,
+      "step": 2850
+    },
+    {
+      "epoch": 0.9855272226051,
+      "grad_norm": 1.7891939878463745,
+      "learning_rate": 2.547717385262815e-08,
+      "logits/chosen": -3.070168972015381,
+      "logits/rejected": -3.0436317920684814,
+      "logps/chosen": -55.60955810546875,
+      "logps/rejected": -54.9880256652832,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.020469585433602333,
+      "rewards/margins": 0.01731836050748825,
+      "rewards/rejected": -0.037787944078445435,
+      "step": 2860
+    },
+    {
+      "epoch": 0.988973121984838,
+      "grad_norm": 1.7842190265655518,
+      "learning_rate": 2.543404784305999e-08,
+      "logits/chosen": -3.0611140727996826,
+      "logits/rejected": -3.0393478870391846,
+      "logps/chosen": -57.559226989746094,
+      "logps/rejected": -56.61458206176758,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.020926518365740776,
+      "rewards/margins": 0.013585202395915985,
+      "rewards/rejected": -0.03451172262430191,
+      "step": 2870
+    },
+    {
+      "epoch": 0.9924190213645762,
+      "grad_norm": 1.844835877418518,
+      "learning_rate": 2.539075407906647e-08,
+      "logits/chosen": -3.0824265480041504,
+      "logits/rejected": -3.0539908409118652,
+      "logps/chosen": -58.49979782104492,
+      "logps/rejected": -56.52234649658203,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.01962786167860031,
+      "rewards/margins": 0.01639539934694767,
+      "rewards/rejected": -0.03602326288819313,
+      "step": 2880
+    },
+    {
+      "epoch": 0.9958649207443143,
+      "grad_norm": 1.82573664188385,
+      "learning_rate": 2.534729325670733e-08,
+      "logits/chosen": -2.9983744621276855,
+      "logits/rejected": -2.9789533615112305,
+      "logps/chosen": -54.04158401489258,
+      "logps/rejected": -56.86323928833008,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.01882469281554222,
+      "rewards/margins": 0.009732010774314404,
+      "rewards/rejected": -0.0285567045211792,
+      "step": 2890
+    },
+    {
+      "epoch": 0.9993108201240524,
+      "grad_norm": 1.7199468612670898,
+      "learning_rate": 2.530366607472818e-08,
+      "logits/chosen": -2.9952845573425293,
+      "logits/rejected": -2.9637436866760254,
+      "logps/chosen": -53.29069900512695,
+      "logps/rejected": -56.52406692504883,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.021782074123620987,
+      "rewards/margins": 0.012855974026024342,
+      "rewards/rejected": -0.034638047218322754,
+      "step": 2900
+    },
+    {
+      "epoch": 0.9993108201240524,
+      "eval_logits/chosen": -3.128720760345459,
+      "eval_logits/rejected": -3.1230790615081787,
+      "eval_logps/chosen": -58.94960403442383,
+      "eval_logps/rejected": -63.897159576416016,
+      "eval_loss": 0.6908628940582275,
+      "eval_rewards/accuracies": 0.5762081742286682,
+      "eval_rewards/chosen": -0.0023771135602146387,
+      "eval_rewards/margins": 0.004793298430740833,
+      "eval_rewards/rejected": -0.007170410826802254,
+      "eval_runtime": 383.4845,
+      "eval_samples_per_second": 11.223,
+      "eval_steps_per_second": 1.403,
+      "step": 2900
+    },
+    {
+      "epoch": 1.0027567195037905,
+      "grad_norm": 1.7115106582641602,
+      "learning_rate": 2.5259873234549318e-08,
+      "logits/chosen": -3.100159168243408,
+      "logits/rejected": -3.075038433074951,
+      "logps/chosen": -55.7207145690918,
+      "logps/rejected": -55.09505081176758,
+      "loss": 0.6861,
+      "rewards/accuracies": 0.6781250238418579,
+      "rewards/chosen": -0.021721744909882545,
+      "rewards/margins": 0.014589185826480389,
+      "rewards/rejected": -0.03631092607975006,
+      "step": 2910
+    },
+    {
+      "epoch": 1.0062026188835287,
+      "grad_norm": 1.7290997505187988,
+      "learning_rate": 2.5215915440254415e-08,
+      "logits/chosen": -3.0028934478759766,
+      "logits/rejected": -2.9760749340057373,
+      "logps/chosen": -54.931121826171875,
+      "logps/rejected": -57.378997802734375,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.019888240844011307,
+      "rewards/margins": 0.01672576740384102,
+      "rewards/rejected": -0.036614008247852325,
+      "step": 2920
+    },
+    {
+      "epoch": 1.0096485182632666,
+      "grad_norm": 1.9308395385742188,
+      "learning_rate": 2.517179339857921e-08,
+      "logits/chosen": -3.019427537918091,
+      "logits/rejected": -3.0041069984436035,
+      "logps/chosen": -53.46974563598633,
+      "logps/rejected": -57.57658767700195,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.021256783977150917,
+      "rewards/margins": 0.015841824933886528,
+      "rewards/rejected": -0.037098608911037445,
+      "step": 2930
+    },
+    {
+      "epoch": 1.0130944176430048,
+      "grad_norm": 1.7856531143188477,
+      "learning_rate": 2.512750781890015e-08,
+      "logits/chosen": -3.0234858989715576,
+      "logits/rejected": -3.0063087940216064,
+      "logps/chosen": -57.436058044433594,
+      "logps/rejected": -57.569984436035156,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.019746603444218636,
+      "rewards/margins": 0.015167878940701485,
+      "rewards/rejected": -0.03491448238492012,
+      "step": 2940
+    },
+    {
+      "epoch": 1.016540317022743,
+      "grad_norm": 1.7923604249954224,
+      "learning_rate": 2.508305941322297e-08,
+      "logits/chosen": -3.040229558944702,
+      "logits/rejected": -3.0229856967926025,
+      "logps/chosen": -57.29484939575195,
+      "logps/rejected": -59.875938415527344,
+      "loss": 0.6857,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.018791478127241135,
+      "rewards/margins": 0.01557534746825695,
+      "rewards/rejected": -0.034366823732852936,
+      "step": 2950
+    },
+    {
+      "epoch": 1.019986216402481,
+      "grad_norm": 1.9251741170883179,
+      "learning_rate": 2.5038448896171267e-08,
+      "logits/chosen": -3.0152406692504883,
+      "logits/rejected": -3.007958173751831,
+      "logps/chosen": -54.25360870361328,
+      "logps/rejected": -58.87769317626953,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.022555235773324966,
+      "rewards/margins": 0.010101383551955223,
+      "rewards/rejected": -0.03265661746263504,
+      "step": 2960
+    },
+    {
+      "epoch": 1.0234321157822193,
+      "grad_norm": 1.679025411605835,
+      "learning_rate": 2.4993676984974994e-08,
+      "logits/chosen": -3.0465149879455566,
+      "logits/rejected": -3.0227606296539307,
+      "logps/chosen": -54.546287536621094,
+      "logps/rejected": -55.989967346191406,
+      "loss": 0.6857,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.02313600480556488,
+      "rewards/margins": 0.015595885924994946,
+      "rewards/rejected": -0.0387318916618824,
+      "step": 2970
+    },
+    {
+      "epoch": 1.0268780151619572,
+      "grad_norm": 1.7610876560211182,
+      "learning_rate": 2.4948744399458936e-08,
+      "logits/chosen": -2.9443602561950684,
+      "logits/rejected": -2.9163906574249268,
+      "logps/chosen": -57.74031448364258,
+      "logps/rejected": -56.23357391357422,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.017260603606700897,
+      "rewards/margins": 0.016214249655604362,
+      "rewards/rejected": -0.03347485512495041,
+      "step": 2980
+    },
+    {
+      "epoch": 1.0303239145416954,
+      "grad_norm": 1.695494532585144,
+      "learning_rate": 2.4903651862031125e-08,
+      "logits/chosen": -2.971776247024536,
+      "logits/rejected": -2.9627065658569336,
+      "logps/chosen": -54.761146545410156,
+      "logps/rejected": -56.9130744934082,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.025274667888879776,
+      "rewards/margins": 0.011709393002092838,
+      "rewards/rejected": -0.03698405995965004,
+      "step": 2990
+    },
+    {
+      "epoch": 1.0337698139214335,
+      "grad_norm": 1.542134165763855,
+      "learning_rate": 2.4858400097671262e-08,
+      "logits/chosen": -3.034815549850464,
+      "logits/rejected": -3.0088250637054443,
+      "logps/chosen": -56.11560821533203,
+      "logps/rejected": -55.88212966918945,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.025271693244576454,
+      "rewards/margins": 0.0122052151709795,
+      "rewards/rejected": -0.037476904690265656,
+      "step": 3000
+    },
+    {
+      "epoch": 1.0337698139214335,
+      "eval_logits/chosen": -3.1268460750579834,
+      "eval_logits/rejected": -3.1212127208709717,
+      "eval_logps/chosen": -59.03478240966797,
+      "eval_logps/rejected": -64.01641082763672,
+      "eval_loss": 0.6907060146331787,
+      "eval_rewards/accuracies": 0.5834107995033264,
+      "eval_rewards/chosen": -0.00322886579670012,
+      "eval_rewards/margins": 0.005133986007422209,
+      "eval_rewards/rejected": -0.008362852036952972,
+      "eval_runtime": 383.3478,
+      "eval_samples_per_second": 11.227,
+      "eval_steps_per_second": 1.403,
+      "step": 3000
+    },
+    {
+      "epoch": 1.0372157133011717,
+      "grad_norm": 1.759706735610962,
+      "learning_rate": 2.4812989833919003e-08,
+      "logits/chosen": -3.110006809234619,
+      "logits/rejected": -3.084453582763672,
+      "logps/chosen": -55.930686950683594,
+      "logps/rejected": -54.365562438964844,
+      "loss": 0.6861,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.022194284945726395,
+      "rewards/margins": 0.014933859929442406,
+      "rewards/rejected": -0.03712814301252365,
+      "step": 3010
+    },
+    {
+      "epoch": 1.0406616126809096,
+      "grad_norm": 1.513440489768982,
+      "learning_rate": 2.4767421800862314e-08,
+      "logits/chosen": -2.947720766067505,
+      "logits/rejected": -2.932685136795044,
+      "logps/chosen": -54.595481872558594,
+      "logps/rejected": -54.91960906982422,
+      "loss": 0.688,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.02822362445294857,
+      "rewards/margins": 0.011009271256625652,
+      "rewards/rejected": -0.039232898503541946,
+      "step": 3020
+    },
+    {
+      "epoch": 1.0441075120606478,
+      "grad_norm": 1.5499457120895386,
+      "learning_rate": 2.47216967311257e-08,
+      "logits/chosen": -2.981621265411377,
+      "logits/rejected": -2.9563755989074707,
+      "logps/chosen": -58.820709228515625,
+      "logps/rejected": -56.66492462158203,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.02402259036898613,
+      "rewards/margins": 0.014306925237178802,
+      "rewards/rejected": -0.03832951560616493,
+      "step": 3030
+    },
+    {
+      "epoch": 1.047553411440386,
+      "grad_norm": 1.789083480834961,
+      "learning_rate": 2.4675815359858456e-08,
+      "logits/chosen": -3.0098814964294434,
+      "logits/rejected": -2.9883923530578613,
+      "logps/chosen": -58.740211486816406,
+      "logps/rejected": -56.543556213378906,
+      "loss": 0.6852,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.022312412038445473,
+      "rewards/margins": 0.016624290496110916,
+      "rewards/rejected": -0.03893670439720154,
+      "step": 3040
+    },
+    {
+      "epoch": 1.050999310820124,
+      "grad_norm": 1.7752656936645508,
+      "learning_rate": 2.4629778424722802e-08,
+      "logits/chosen": -3.0468223094940186,
+      "logits/rejected": -3.0363242626190186,
+      "logps/chosen": -56.31238555908203,
+      "logps/rejected": -56.50005340576172,
+      "loss": 0.6852,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.022167257964611053,
+      "rewards/margins": 0.01672268845140934,
+      "rewards/rejected": -0.03888994827866554,
+      "step": 3050
+    },
+    {
+      "epoch": 1.0544452101998623,
+      "grad_norm": 1.8021841049194336,
+      "learning_rate": 2.458358666588208e-08,
+      "logits/chosen": -3.0902514457702637,
+      "logits/rejected": -3.0584259033203125,
+      "logps/chosen": -54.912513732910156,
+      "logps/rejected": -57.120758056640625,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.01902184821665287,
+      "rewards/margins": 0.01735542342066765,
+      "rewards/rejected": -0.03637726977467537,
+      "step": 3060
+    },
+    {
+      "epoch": 1.0578911095796002,
+      "grad_norm": 1.8034168481826782,
+      "learning_rate": 2.4537240825988818e-08,
+      "logits/chosen": -3.057995319366455,
+      "logits/rejected": -3.039032459259033,
+      "logps/chosen": -56.22139358520508,
+      "logps/rejected": -56.99552536010742,
+      "loss": 0.687,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.02486089989542961,
+      "rewards/margins": 0.01296932715922594,
+      "rewards/rejected": -0.037830229848623276,
+      "step": 3070
+    },
+    {
+      "epoch": 1.0613370089593384,
+      "grad_norm": 1.6976583003997803,
+      "learning_rate": 2.4490741650172783e-08,
+      "logits/chosen": -2.9569573402404785,
+      "logits/rejected": -2.9440407752990723,
+      "logps/chosen": -54.35454177856445,
+      "logps/rejected": -55.02104568481445,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.02362692728638649,
+      "rewards/margins": 0.013894198462367058,
+      "rewards/rejected": -0.0375211201608181,
+      "step": 3080
+    },
+    {
+      "epoch": 1.0647829083390765,
+      "grad_norm": 1.7445013523101807,
+      "learning_rate": 2.444408988602903e-08,
+      "logits/chosen": -3.0692954063415527,
+      "logits/rejected": -3.0486438274383545,
+      "logps/chosen": -60.94700241088867,
+      "logps/rejected": -58.622825622558594,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.021391045302152634,
+      "rewards/margins": 0.017580123618245125,
+      "rewards/rejected": -0.03897116705775261,
+      "step": 3090
+    },
+    {
+      "epoch": 1.0682288077188147,
+      "grad_norm": 1.7617013454437256,
+      "learning_rate": 2.4397286283605856e-08,
+      "logits/chosen": -2.9837212562561035,
+      "logits/rejected": -2.9614932537078857,
+      "logps/chosen": -58.716285705566406,
+      "logps/rejected": -58.024391174316406,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.023710574954748154,
+      "rewards/margins": 0.015334436669945717,
+      "rewards/rejected": -0.03904501348733902,
+      "step": 3100
+    },
+    {
+      "epoch": 1.0682288077188147,
+      "eval_logits/chosen": -3.124697208404541,
+      "eval_logits/rejected": -3.1189982891082764,
+      "eval_logps/chosen": -59.13443374633789,
+      "eval_logps/rejected": -64.13980102539062,
+      "eval_loss": 0.6906017661094666,
+      "eval_rewards/accuracies": 0.580622673034668,
+      "eval_rewards/chosen": -0.004225407727062702,
+      "eval_rewards/margins": 0.005371410865336657,
+      "eval_rewards/rejected": -0.009596818126738071,
+      "eval_runtime": 383.4141,
+      "eval_samples_per_second": 11.225,
+      "eval_steps_per_second": 1.403,
+      "step": 3100
+    },
+    {
+      "epoch": 1.0716747070985528,
+      "grad_norm": 1.5646659135818481,
+      "learning_rate": 2.4350331595392772e-08,
+      "logits/chosen": -3.0381569862365723,
+      "logits/rejected": -3.0126564502716064,
+      "logps/chosen": -55.83879470825195,
+      "logps/rejected": -56.8257942199707,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.023566443473100662,
+      "rewards/margins": 0.01747654564678669,
+      "rewards/rejected": -0.0410429909825325,
+      "step": 3110
+    },
+    {
+      "epoch": 1.0751206064782908,
+      "grad_norm": 1.8210567235946655,
+      "learning_rate": 2.430322657630836e-08,
+      "logits/chosen": -3.0764999389648438,
+      "logits/rejected": -3.0509510040283203,
+      "logps/chosen": -59.753662109375,
+      "logps/rejected": -58.814422607421875,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.021439004689455032,
+      "rewards/margins": 0.015430884435772896,
+      "rewards/rejected": -0.03686989098787308,
+      "step": 3120
+    },
+    {
+      "epoch": 1.078566505858029,
+      "grad_norm": 1.9204891920089722,
+      "learning_rate": 2.425597198368818e-08,
+      "logits/chosen": -3.0081515312194824,
+      "logits/rejected": -2.986262321472168,
+      "logps/chosen": -57.300010681152344,
+      "logps/rejected": -56.1832389831543,
+      "loss": 0.6857,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.024739116430282593,
+      "rewards/margins": 0.01571592316031456,
+      "rewards/rejected": -0.04045503959059715,
+      "step": 3130
+    },
+    {
+      "epoch": 1.082012405237767,
+      "grad_norm": 1.72856867313385,
+      "learning_rate": 2.4208568577272568e-08,
+      "logits/chosen": -3.0539886951446533,
+      "logits/rejected": -3.0409510135650635,
+      "logps/chosen": -56.03534698486328,
+      "logps/rejected": -57.60711669921875,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.02378886379301548,
+      "rewards/margins": 0.01587529107928276,
+      "rewards/rejected": -0.03966415673494339,
+      "step": 3140
+    },
+    {
+      "epoch": 1.0854583046175053,
+      "grad_norm": 1.6887574195861816,
+      "learning_rate": 2.416101711919443e-08,
+      "logits/chosen": -3.01220703125,
+      "logits/rejected": -2.9825055599212646,
+      "logps/chosen": -54.468589782714844,
+      "logps/rejected": -56.66431427001953,
+      "loss": 0.6841,
+      "rewards/accuracies": 0.684374988079071,
+      "rewards/chosen": -0.025612574070692062,
+      "rewards/margins": 0.018923945724964142,
+      "rewards/rejected": -0.044536516070365906,
+      "step": 3150
+    },
+    {
+      "epoch": 1.0889042039972432,
+      "grad_norm": 1.6957333087921143,
+      "learning_rate": 2.4113318373966993e-08,
+      "logits/chosen": -3.008650541305542,
+      "logits/rejected": -2.98211407661438,
+      "logps/chosen": -55.06719207763672,
+      "logps/rejected": -56.2410774230957,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.024777565151453018,
+      "rewards/margins": 0.013255933299660683,
+      "rewards/rejected": -0.03803349658846855,
+      "step": 3160
+    },
+    {
+      "epoch": 1.0923501033769814,
+      "grad_norm": 1.9195588827133179,
+      "learning_rate": 2.40654731084715e-08,
+      "logits/chosen": -2.996953248977661,
+      "logits/rejected": -2.975588083267212,
+      "logps/chosen": -56.95857620239258,
+      "logps/rejected": -57.565834045410156,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.02519887126982212,
+      "rewards/margins": 0.018115365877747536,
+      "rewards/rejected": -0.043314240872859955,
+      "step": 3170
+    },
+    {
+      "epoch": 1.0957960027567195,
+      "grad_norm": 1.738138198852539,
+      "learning_rate": 2.4017482091944878e-08,
+      "logits/chosen": -2.998206377029419,
+      "logits/rejected": -2.993584632873535,
+      "logps/chosen": -57.191802978515625,
+      "logps/rejected": -59.15422439575195,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": -0.02700095809996128,
+      "rewards/margins": 0.010117514058947563,
+      "rewards/rejected": -0.037118472158908844,
+      "step": 3180
+    },
+    {
+      "epoch": 1.0992419021364577,
+      "grad_norm": 1.866339087486267,
+      "learning_rate": 2.3969346095967406e-08,
+      "logits/chosen": -3.0541040897369385,
+      "logits/rejected": -3.0481302738189697,
+      "logps/chosen": -55.76299285888672,
+      "logps/rejected": -59.13557052612305,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.02823568508028984,
+      "rewards/margins": 0.008307856507599354,
+      "rewards/rejected": -0.03654354065656662,
+      "step": 3190
+    },
+    {
+      "epoch": 1.1026878015161956,
+      "grad_norm": 1.881668210029602,
+      "learning_rate": 2.3921065894450262e-08,
+      "logits/chosen": -3.0740349292755127,
+      "logits/rejected": -3.043767213821411,
+      "logps/chosen": -59.0565071105957,
+      "logps/rejected": -57.806182861328125,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.02167920581996441,
+      "rewards/margins": 0.018673600628972054,
+      "rewards/rejected": -0.04035280644893646,
+      "step": 3200
+    },
+    {
+      "epoch": 1.1026878015161956,
+      "eval_logits/chosen": -3.121842384338379,
+      "eval_logits/rejected": -3.1161463260650635,
+      "eval_logps/chosen": -59.2255859375,
+      "eval_logps/rejected": -64.27249145507812,
+      "eval_loss": 0.690406084060669,
+      "eval_rewards/accuracies": 0.5838754773139954,
+      "eval_rewards/chosen": -0.005136927589774132,
+      "eval_rewards/margins": 0.005786766763776541,
+      "eval_rewards/rejected": -0.010923693887889385,
+      "eval_runtime": 383.5748,
+      "eval_samples_per_second": 11.221,
+      "eval_steps_per_second": 1.403,
+      "step": 3200
+    },
+    {
+      "epoch": 1.1061337008959338,
+      "grad_norm": 1.6875972747802734,
+      "learning_rate": 2.3872642263623113e-08,
+      "logits/chosen": -3.112612247467041,
+      "logits/rejected": -3.090404987335205,
+      "logps/chosen": -57.133636474609375,
+      "logps/rejected": -56.93791580200195,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.025561952963471413,
+      "rewards/margins": 0.015494232065975666,
+      "rewards/rejected": -0.04105618596076965,
+      "step": 3210
+    },
+    {
+      "epoch": 1.109579600275672,
+      "grad_norm": 1.7941416501998901,
+      "learning_rate": 2.3824075982021626e-08,
+      "logits/chosen": -2.990532875061035,
+      "logits/rejected": -2.974168300628662,
+      "logps/chosen": -54.56916046142578,
+      "logps/rejected": -54.53216552734375,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.027438456192612648,
+      "rewards/margins": 0.010743899270892143,
+      "rewards/rejected": -0.038182348012924194,
+      "step": 3220
+    },
+    {
+      "epoch": 1.11302549965541,
+      "grad_norm": 1.8264120817184448,
+      "learning_rate": 2.3775367830474943e-08,
+      "logits/chosen": -2.994661808013916,
+      "logits/rejected": -2.979541540145874,
+      "logps/chosen": -56.54296875,
+      "logps/rejected": -59.20990753173828,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.024427255615592003,
+      "rewards/margins": 0.016915548592805862,
+      "rewards/rejected": -0.04134280979633331,
+      "step": 3230
+    },
+    {
+      "epoch": 1.1164713990351482,
+      "grad_norm": 1.8102046251296997,
+      "learning_rate": 2.3726518592093136e-08,
+      "logits/chosen": -2.980262041091919,
+      "logits/rejected": -2.9502079486846924,
+      "logps/chosen": -56.95534133911133,
+      "logps/rejected": -55.209190368652344,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.02289319410920143,
+      "rewards/margins": 0.018111836165189743,
+      "rewards/rejected": -0.04100503399968147,
+      "step": 3240
+    },
+    {
+      "epoch": 1.1199172984148862,
+      "grad_norm": 1.7997123003005981,
+      "learning_rate": 2.367752905225462e-08,
+      "logits/chosen": -3.123244524002075,
+      "logits/rejected": -3.113422393798828,
+      "logps/chosen": -57.912010192871094,
+      "logps/rejected": -57.473915100097656,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.02998507022857666,
+      "rewards/margins": 0.01054734643548727,
+      "rewards/rejected": -0.040532417595386505,
+      "step": 3250
+    },
+    {
+      "epoch": 1.1233631977946243,
+      "grad_norm": 1.7041951417922974,
+      "learning_rate": 2.3628399998593515e-08,
+      "logits/chosen": -3.0340027809143066,
+      "logits/rejected": -2.997541904449463,
+      "logps/chosen": -57.29707717895508,
+      "logps/rejected": -56.401588439941406,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.020992079749703407,
+      "rewards/margins": 0.020691480487585068,
+      "rewards/rejected": -0.041683558374643326,
+      "step": 3260
+    },
+    {
+      "epoch": 1.1268090971743625,
+      "grad_norm": 1.901633620262146,
+      "learning_rate": 2.3579132220986993e-08,
+      "logits/chosen": -3.00722074508667,
+      "logits/rejected": -2.9789793491363525,
+      "logps/chosen": -57.99610137939453,
+      "logps/rejected": -57.89921951293945,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.024969404563307762,
+      "rewards/margins": 0.015277421101927757,
+      "rewards/rejected": -0.04024682566523552,
+      "step": 3270
+    },
+    {
+      "epoch": 1.1302549965541007,
+      "grad_norm": 1.6994380950927734,
+      "learning_rate": 2.352972651154256e-08,
+      "logits/chosen": -2.999063730239868,
+      "logits/rejected": -2.965944766998291,
+      "logps/chosen": -56.44902420043945,
+      "logps/rejected": -56.36620330810547,
+      "loss": 0.684,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.026739532127976418,
+      "rewards/margins": 0.019118938595056534,
+      "rewards/rejected": -0.0458584725856781,
+      "step": 3280
+    },
+    {
+      "epoch": 1.1337008959338388,
+      "grad_norm": 1.6563690900802612,
+      "learning_rate": 2.3480183664585354e-08,
+      "logits/chosen": -3.054525852203369,
+      "logits/rejected": -3.019784450531006,
+      "logps/chosen": -56.85039520263672,
+      "logps/rejected": -55.61071014404297,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.025542262941598892,
+      "rewards/margins": 0.017069051042199135,
+      "rewards/rejected": -0.042611315846443176,
+      "step": 3290
+    },
+    {
+      "epoch": 1.1371467953135768,
+      "grad_norm": 1.6233892440795898,
+      "learning_rate": 2.343050447664534e-08,
+      "logits/chosen": -3.056015968322754,
+      "logits/rejected": -3.0365700721740723,
+      "logps/chosen": -57.272193908691406,
+      "logps/rejected": -57.5992431640625,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.0303605105727911,
+      "rewards/margins": 0.010287837125360966,
+      "rewards/rejected": -0.04064834490418434,
+      "step": 3300
+    },
+    {
+      "epoch": 1.1371467953135768,
+      "eval_logits/chosen": -3.119631290435791,
+      "eval_logits/rejected": -3.1139142513275146,
+      "eval_logps/chosen": -59.37312698364258,
+      "eval_logps/rejected": -64.45064544677734,
+      "eval_loss": 0.6902690529823303,
+      "eval_rewards/accuracies": 0.5873606204986572,
+      "eval_rewards/chosen": -0.006612339057028294,
+      "eval_rewards/margins": 0.006092800293117762,
+      "eval_rewards/rejected": -0.012705138884484768,
+      "eval_runtime": 383.5133,
+      "eval_samples_per_second": 11.223,
+      "eval_steps_per_second": 1.403,
+      "step": 3300
+    },
+    {
+      "epoch": 1.140592694693315,
+      "grad_norm": 2.0969696044921875,
+      "learning_rate": 2.338068974644453e-08,
+      "logits/chosen": -3.0407307147979736,
+      "logits/rejected": -3.019397735595703,
+      "logps/chosen": -55.3810920715332,
+      "logps/rejected": -56.598304748535156,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.03202560171484947,
+      "rewards/margins": 0.01439299900084734,
+      "rewards/rejected": -0.04641861096024513,
+      "step": 3310
+    },
+    {
+      "epoch": 1.144038594073053,
+      "grad_norm": 1.5600014925003052,
+      "learning_rate": 2.333074027488412e-08,
+      "logits/chosen": -3.0307507514953613,
+      "logits/rejected": -2.99760103225708,
+      "logps/chosen": -56.03546905517578,
+      "logps/rejected": -54.2665901184082,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.028267959132790565,
+      "rewards/margins": 0.020445603877305984,
+      "rewards/rejected": -0.0487135574221611,
+      "step": 3320
+    },
+    {
+      "epoch": 1.1474844934527912,
+      "grad_norm": 1.7586891651153564,
+      "learning_rate": 2.3280656865031624e-08,
+      "logits/chosen": -3.105713367462158,
+      "logits/rejected": -3.0847623348236084,
+      "logps/chosen": -54.640533447265625,
+      "logps/rejected": -58.454856872558594,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.02703413926064968,
+      "rewards/margins": 0.019927233457565308,
+      "rewards/rejected": -0.04696137458086014,
+      "step": 3330
+    },
+    {
+      "epoch": 1.1509303928325294,
+      "grad_norm": 1.8528821468353271,
+      "learning_rate": 2.3230440322107973e-08,
+      "logits/chosen": -3.0146238803863525,
+      "logits/rejected": -3.0011608600616455,
+      "logps/chosen": -54.98958206176758,
+      "logps/rejected": -58.26300048828125,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.031235381960868835,
+      "rewards/margins": 0.014607800170779228,
+      "rewards/rejected": -0.04584318399429321,
+      "step": 3340
+    },
+    {
+      "epoch": 1.1543762922122673,
+      "grad_norm": 1.8970739841461182,
+      "learning_rate": 2.3180091453474546e-08,
+      "logits/chosen": -3.0196709632873535,
+      "logits/rejected": -3.003033399581909,
+      "logps/chosen": -59.24443435668945,
+      "logps/rejected": -60.420021057128906,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.026004821062088013,
+      "rewards/margins": 0.019533216953277588,
+      "rewards/rejected": -0.0455380380153656,
+      "step": 3350
+    },
+    {
+      "epoch": 1.1578221915920055,
+      "grad_norm": 1.7306209802627563,
+      "learning_rate": 2.3129611068620208e-08,
+      "logits/chosen": -3.0181641578674316,
+      "logits/rejected": -2.999312400817871,
+      "logps/chosen": -56.587852478027344,
+      "logps/rejected": -56.988136291503906,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.024178151041269302,
+      "rewards/margins": 0.020215744152665138,
+      "rewards/rejected": -0.04439389705657959,
+      "step": 3360
+    },
+    {
+      "epoch": 1.1612680909717437,
+      "grad_norm": 1.6228654384613037,
+      "learning_rate": 2.3078999979148287e-08,
+      "logits/chosen": -3.002633571624756,
+      "logits/rejected": -2.9905872344970703,
+      "logps/chosen": -58.37123489379883,
+      "logps/rejected": -59.83577346801758,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.028042176738381386,
+      "rewards/margins": 0.01347642857581377,
+      "rewards/rejected": -0.04151860624551773,
+      "step": 3370
+    },
+    {
+      "epoch": 1.1647139903514818,
+      "grad_norm": 1.697937250137329,
+      "learning_rate": 2.3028258998763526e-08,
+      "logits/chosen": -3.0513787269592285,
+      "logits/rejected": -3.0297999382019043,
+      "logps/chosen": -55.665870666503906,
+      "logps/rejected": -55.926300048828125,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.03410068899393082,
+      "rewards/margins": 0.013039866462349892,
+      "rewards/rejected": -0.04714054986834526,
+      "step": 3380
+    },
+    {
+      "epoch": 1.1681598897312198,
+      "grad_norm": 1.8130141496658325,
+      "learning_rate": 2.2977388943259006e-08,
+      "logits/chosen": -3.0361554622650146,
+      "logits/rejected": -3.017449378967285,
+      "logps/chosen": -58.13988494873047,
+      "logps/rejected": -58.97544479370117,
+      "loss": 0.686,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.02971479669213295,
+      "rewards/margins": 0.015255960635840893,
+      "rewards/rejected": -0.04497075825929642,
+      "step": 3390
+    },
+    {
+      "epoch": 1.171605789110958,
+      "grad_norm": 1.776076078414917,
+      "learning_rate": 2.2926390630503015e-08,
+      "logits/chosen": -3.0007052421569824,
+      "logits/rejected": -2.9792122840881348,
+      "logps/chosen": -54.945411682128906,
+      "logps/rejected": -56.64307403564453,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.03474254161119461,
+      "rewards/margins": 0.01561157125979662,
+      "rewards/rejected": -0.050354111939668655,
+      "step": 3400
+    },
+    {
+      "epoch": 1.171605789110958,
+      "eval_logits/chosen": -3.117290496826172,
+      "eval_logits/rejected": -3.1116089820861816,
+      "eval_logps/chosen": -59.50706481933594,
+      "eval_logps/rejected": -64.59645080566406,
+      "eval_loss": 0.6902266144752502,
+      "eval_rewards/accuracies": 0.5785316228866577,
+      "eval_rewards/chosen": -0.007951668463647366,
+      "eval_rewards/margins": 0.006211577914655209,
+      "eval_rewards/rejected": -0.014163246378302574,
+      "eval_runtime": 383.1931,
+      "eval_samples_per_second": 11.232,
+      "eval_steps_per_second": 1.404,
+      "step": 3400
+    },
+    {
+      "epoch": 1.175051688490696,
+      "grad_norm": 1.8506773710250854,
+      "learning_rate": 2.2875264880425924e-08,
+      "logits/chosen": -3.0618579387664795,
+      "logits/rejected": -3.0377535820007324,
+      "logps/chosen": -57.790199279785156,
+      "logps/rejected": -57.322357177734375,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.02426614984869957,
+      "rewards/margins": 0.0251364354044199,
+      "rewards/rejected": -0.04940258339047432,
+      "step": 3410
+    },
+    {
+      "epoch": 1.1784975878704342,
+      "grad_norm": 2.010289430618286,
+      "learning_rate": 2.282401251500698e-08,
+      "logits/chosen": -3.0993943214416504,
+      "logits/rejected": -3.083157777786255,
+      "logps/chosen": -58.3515510559082,
+      "logps/rejected": -58.183837890625,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.02934526838362217,
+      "rewards/margins": 0.010435683652758598,
+      "rewards/rejected": -0.03978095203638077,
+      "step": 3420
+    },
+    {
+      "epoch": 1.1819434872501722,
+      "grad_norm": 1.7674137353897095,
+      "learning_rate": 2.277263435826111e-08,
+      "logits/chosen": -2.981959581375122,
+      "logits/rejected": -2.947040319442749,
+      "logps/chosen": -59.366554260253906,
+      "logps/rejected": -58.10066604614258,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.027270367369055748,
+      "rewards/margins": 0.023923374712467194,
+      "rewards/rejected": -0.05119375139474869,
+      "step": 3430
+    },
+    {
+      "epoch": 1.1853893866299103,
+      "grad_norm": 1.7860718965530396,
+      "learning_rate": 2.272113123622565e-08,
+      "logits/chosen": -3.019498348236084,
+      "logits/rejected": -2.9986841678619385,
+      "logps/chosen": -56.122657775878906,
+      "logps/rejected": -56.411865234375,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.03448476269841194,
+      "rewards/margins": 0.014978880062699318,
+      "rewards/rejected": -0.04946363717317581,
+      "step": 3440
+    },
+    {
+      "epoch": 1.1888352860096485,
+      "grad_norm": 1.7381949424743652,
+      "learning_rate": 2.266950397694709e-08,
+      "logits/chosen": -3.018294334411621,
+      "logits/rejected": -2.994025230407715,
+      "logps/chosen": -55.75679397583008,
+      "logps/rejected": -57.55413818359375,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.028243333101272583,
+      "rewards/margins": 0.018168529495596886,
+      "rewards/rejected": -0.04641186445951462,
+      "step": 3450
+    },
+    {
+      "epoch": 1.1922811853893867,
+      "grad_norm": 1.8627835512161255,
+      "learning_rate": 2.2617753410467744e-08,
+      "logits/chosen": -3.1187243461608887,
+      "logits/rejected": -3.090602397918701,
+      "logps/chosen": -59.478240966796875,
+      "logps/rejected": -59.34978103637695,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.029832344502210617,
+      "rewards/margins": 0.018782097846269608,
+      "rewards/rejected": -0.04861444607377052,
+      "step": 3460
+    },
+    {
+      "epoch": 1.1957270847691248,
+      "grad_norm": 1.7428736686706543,
+      "learning_rate": 2.2565880368812406e-08,
+      "logits/chosen": -2.95066499710083,
+      "logits/rejected": -2.913435459136963,
+      "logps/chosen": -60.353004455566406,
+      "logps/rejected": -56.9808235168457,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.025545936077833176,
+      "rewards/margins": 0.02004757523536682,
+      "rewards/rejected": -0.0455935075879097,
+      "step": 3470
+    },
+    {
+      "epoch": 1.1991729841488628,
+      "grad_norm": 1.731405258178711,
+      "learning_rate": 2.2513885685974985e-08,
+      "logits/chosen": -3.0981078147888184,
+      "logits/rejected": -3.0685362815856934,
+      "logps/chosen": -57.7297477722168,
+      "logps/rejected": -55.466758728027344,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.031421415507793427,
+      "rewards/margins": 0.01965133473277092,
+      "rewards/rejected": -0.051072753965854645,
+      "step": 3480
+    },
+    {
+      "epoch": 1.202618883528601,
+      "grad_norm": 1.7326806783676147,
+      "learning_rate": 2.246177019790507e-08,
+      "logits/chosen": -3.0435478687286377,
+      "logits/rejected": -3.010909080505371,
+      "logps/chosen": -57.0941047668457,
+      "logps/rejected": -56.86127853393555,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.027105292305350304,
+      "rewards/margins": 0.0245223231613636,
+      "rewards/rejected": -0.051627613604068756,
+      "step": 3490
+    },
+    {
+      "epoch": 1.206064782908339,
+      "grad_norm": 1.771358609199524,
+      "learning_rate": 2.240953474249454e-08,
+      "logits/chosen": -3.0621421337127686,
+      "logits/rejected": -3.0434374809265137,
+      "logps/chosen": -60.080299377441406,
+      "logps/rejected": -59.4837532043457,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.03441409394145012,
+      "rewards/margins": 0.015658359974622726,
+      "rewards/rejected": -0.050072453916072845,
+      "step": 3500
+    },
+    {
+      "epoch": 1.206064782908339,
+      "eval_logits/chosen": -3.115753173828125,
+      "eval_logits/rejected": -3.1100549697875977,
+      "eval_logps/chosen": -59.70413589477539,
+      "eval_logps/rejected": -64.836181640625,
+      "eval_loss": 0.6900400519371033,
+      "eval_rewards/accuracies": 0.5831784605979919,
+      "eval_rewards/chosen": -0.009922372177243233,
+      "eval_rewards/margins": 0.006638327147811651,
+      "eval_rewards/rejected": -0.01656069979071617,
+      "eval_runtime": 383.3461,
+      "eval_samples_per_second": 11.227,
+      "eval_steps_per_second": 1.403,
+      "step": 3500
+    },
+    {
+      "epoch": 1.2095106822880772,
+      "grad_norm": 1.7746785879135132,
+      "learning_rate": 2.235718015956402e-08,
+      "logits/chosen": -3.015407085418701,
+      "logits/rejected": -2.9884328842163086,
+      "logps/chosen": -57.12171173095703,
+      "logps/rejected": -58.0843391418457,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.03241681680083275,
+      "rewards/margins": 0.022321563214063644,
+      "rewards/rejected": -0.05473838374018669,
+      "step": 3510
+    },
+    {
+      "epoch": 1.2129565816678154,
+      "grad_norm": 1.9246511459350586,
+      "learning_rate": 2.230470729084946e-08,
+      "logits/chosen": -3.0177712440490723,
+      "logits/rejected": -2.999504566192627,
+      "logps/chosen": -57.75422286987305,
+      "logps/rejected": -60.72907257080078,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.031657539308071136,
+      "rewards/margins": 0.0175764299929142,
+      "rewards/rejected": -0.049233973026275635,
+      "step": 3520
+    },
+    {
+      "epoch": 1.2164024810475533,
+      "grad_norm": 1.7924283742904663,
+      "learning_rate": 2.2252116979988536e-08,
+      "logits/chosen": -3.0198140144348145,
+      "logits/rejected": -3.001586437225342,
+      "logps/chosen": -55.954734802246094,
+      "logps/rejected": -58.61615753173828,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.03621712699532509,
+      "rewards/margins": 0.018489833921194077,
+      "rewards/rejected": -0.054706960916519165,
+      "step": 3530
+    },
+    {
+      "epoch": 1.2198483804272915,
+      "grad_norm": 1.9447262287139893,
+      "learning_rate": 2.2199410072507116e-08,
+      "logits/chosen": -3.07135009765625,
+      "logits/rejected": -3.042304754257202,
+      "logps/chosen": -57.32279586791992,
+      "logps/rejected": -58.6002311706543,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.03287208825349808,
+      "rewards/margins": 0.017065642401576042,
+      "rewards/rejected": -0.04993772879242897,
+      "step": 3540
+    },
+    {
+      "epoch": 1.2232942798070296,
+      "grad_norm": 1.7399325370788574,
+      "learning_rate": 2.2146587415805665e-08,
+      "logits/chosen": -3.0617799758911133,
+      "logits/rejected": -3.0456764698028564,
+      "logps/chosen": -58.91600799560547,
+      "logps/rejected": -59.912269592285156,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.03620436415076256,
+      "rewards/margins": 0.017626499757170677,
+      "rewards/rejected": -0.05383085459470749,
+      "step": 3550
+    },
+    {
+      "epoch": 1.2267401791867678,
+      "grad_norm": 1.9127978086471558,
+      "learning_rate": 2.209364985914561e-08,
+      "logits/chosen": -3.0418179035186768,
+      "logits/rejected": -3.000598669052124,
+      "logps/chosen": -60.716346740722656,
+      "logps/rejected": -56.84751510620117,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.029868166893720627,
+      "rewards/margins": 0.020894726738333702,
+      "rewards/rejected": -0.05076289176940918,
+      "step": 3560
+    },
+    {
+      "epoch": 1.230186078566506,
+      "grad_norm": 1.6954758167266846,
+      "learning_rate": 2.2040598253635707e-08,
+      "logits/chosen": -2.970766544342041,
+      "logits/rejected": -2.9549448490142822,
+      "logps/chosen": -57.183624267578125,
+      "logps/rejected": -57.490882873535156,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.033541709184646606,
+      "rewards/margins": 0.015752049162983894,
+      "rewards/rejected": -0.04929376393556595,
+      "step": 3570
+    },
+    {
+      "epoch": 1.233631977946244,
+      "grad_norm": 1.7661042213439941,
+      "learning_rate": 2.1987433452218333e-08,
+      "logits/chosen": -3.0575459003448486,
+      "logits/rejected": -3.0263772010803223,
+      "logps/chosen": -58.5184326171875,
+      "logps/rejected": -60.102455139160156,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.03493792563676834,
+      "rewards/margins": 0.020248733460903168,
+      "rewards/rejected": -0.05518665909767151,
+      "step": 3580
+    },
+    {
+      "epoch": 1.237077877325982,
+      "grad_norm": 1.7716708183288574,
+      "learning_rate": 2.1934156309655783e-08,
+      "logits/chosen": -3.0406601428985596,
+      "logits/rejected": -3.02017879486084,
+      "logps/chosen": -59.16892623901367,
+      "logps/rejected": -59.33143997192383,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.037346698343753815,
+      "rewards/margins": 0.014562156982719898,
+      "rewards/rejected": -0.05190885066986084,
+      "step": 3590
+    },
+    {
+      "epoch": 1.2405237767057202,
+      "grad_norm": 1.9056254625320435,
+      "learning_rate": 2.188076768251653e-08,
+      "logits/chosen": -2.9631223678588867,
+      "logits/rejected": -2.9422271251678467,
+      "logps/chosen": -56.6925163269043,
+      "logps/rejected": -60.45619583129883,
+      "loss": 0.685,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.03808574378490448,
+      "rewards/margins": 0.017497101798653603,
+      "rewards/rejected": -0.05558284372091293,
+      "step": 3600
+    },
+    {
+      "epoch": 1.2405237767057202,
+      "eval_logits/chosen": -3.1125693321228027,
+      "eval_logits/rejected": -3.106886386871338,
+      "eval_logps/chosen": -59.863739013671875,
+      "eval_logps/rejected": -65.02647399902344,
+      "eval_loss": 0.689907968044281,
+      "eval_rewards/accuracies": 0.5782992839813232,
+      "eval_rewards/chosen": -0.011518481187522411,
+      "eval_rewards/margins": 0.006945009808987379,
+      "eval_rewards/rejected": -0.018463490530848503,
+      "eval_runtime": 383.2105,
+      "eval_samples_per_second": 11.231,
+      "eval_steps_per_second": 1.404,
+      "step": 3600
+    },
+    {
+      "epoch": 1.2439696760854584,
+      "grad_norm": 1.788948655128479,
+      "learning_rate": 2.1827268429161462e-08,
+      "logits/chosen": -3.0237889289855957,
+      "logits/rejected": -2.9949862957000732,
+      "logps/chosen": -59.3255615234375,
+      "logps/rejected": -57.539268493652344,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.035462163388729095,
+      "rewards/margins": 0.016460830345749855,
+      "rewards/rejected": -0.0519229993224144,
+      "step": 3610
+    },
+    {
+      "epoch": 1.2474155754651963,
+      "grad_norm": 1.7611157894134521,
+      "learning_rate": 2.177365940973005e-08,
+      "logits/chosen": -3.007209062576294,
+      "logits/rejected": -2.9913573265075684,
+      "logps/chosen": -59.62353515625,
+      "logps/rejected": -59.09318923950195,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.037894636392593384,
+      "rewards/margins": 0.014278972521424294,
+      "rewards/rejected": -0.05217360705137253,
+      "step": 3620
+    },
+    {
+      "epoch": 1.2508614748449345,
+      "grad_norm": 1.800534963607788,
+      "learning_rate": 2.1719941486126555e-08,
+      "logits/chosen": -3.071748733520508,
+      "logits/rejected": -3.0391387939453125,
+      "logps/chosen": -57.59711837768555,
+      "logps/rejected": -58.18121337890625,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.034704096615314484,
+      "rewards/margins": 0.02097286470234394,
+      "rewards/rejected": -0.055676959455013275,
+      "step": 3630
+    },
+    {
+      "epoch": 1.2543073742246726,
+      "grad_norm": 1.8541029691696167,
+      "learning_rate": 2.1666115522006147e-08,
+      "logits/chosen": -3.0995118618011475,
+      "logits/rejected": -3.0664963722229004,
+      "logps/chosen": -58.81087112426758,
+      "logps/rejected": -58.59917449951172,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.03368169069290161,
+      "rewards/margins": 0.024582918733358383,
+      "rewards/rejected": -0.058264605700969696,
+      "step": 3640
+    },
+    {
+      "epoch": 1.2577532736044108,
+      "grad_norm": 1.8306084871292114,
+      "learning_rate": 2.1612182382761037e-08,
+      "logits/chosen": -2.875537872314453,
+      "logits/rejected": -2.860386848449707,
+      "logps/chosen": -56.48638916015625,
+      "logps/rejected": -58.9466552734375,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.039603520184755325,
+      "rewards/margins": 0.010510370135307312,
+      "rewards/rejected": -0.050113894045352936,
+      "step": 3650
+    },
+    {
+      "epoch": 1.2611991729841487,
+      "grad_norm": 1.7572910785675049,
+      "learning_rate": 2.1558142935506547e-08,
+      "logits/chosen": -3.06649112701416,
+      "logits/rejected": -3.0416479110717773,
+      "logps/chosen": -58.490684509277344,
+      "logps/rejected": -56.93430709838867,
+      "loss": 0.6857,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.03784402459859848,
+      "rewards/margins": 0.01629071682691574,
+      "rewards/rejected": -0.05413473770022392,
+      "step": 3660
+    },
+    {
+      "epoch": 1.264645072363887,
+      "grad_norm": 1.8614615201950073,
+      "learning_rate": 2.1503998049067184e-08,
+      "logits/chosen": -3.018354654312134,
+      "logits/rejected": -2.9911229610443115,
+      "logps/chosen": -57.60223388671875,
+      "logps/rejected": -58.49652862548828,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.03380398079752922,
+      "rewards/margins": 0.023725925013422966,
+      "rewards/rejected": -0.057529907673597336,
+      "step": 3670
+    },
+    {
+      "epoch": 1.268090971743625,
+      "grad_norm": 1.8270007371902466,
+      "learning_rate": 2.144974859396265e-08,
+      "logits/chosen": -3.0520198345184326,
+      "logits/rejected": -3.0269174575805664,
+      "logps/chosen": -59.025901794433594,
+      "logps/rejected": -57.426612854003906,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.03417273610830307,
+      "rewards/margins": 0.02032269909977913,
+      "rewards/rejected": -0.0544954352080822,
+      "step": 3680
+    },
+    {
+      "epoch": 1.2715368711233632,
+      "grad_norm": 1.785484790802002,
+      "learning_rate": 2.139539544239387e-08,
+      "logits/chosen": -2.9499237537384033,
+      "logits/rejected": -2.928621768951416,
+      "logps/chosen": -56.885337829589844,
+      "logps/rejected": -57.8686637878418,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.038981880992650986,
+      "rewards/margins": 0.015064500272274017,
+      "rewards/rejected": -0.0540463849902153,
+      "step": 3690
+    },
+    {
+      "epoch": 1.2749827705031014,
+      "grad_norm": 1.849421501159668,
+      "learning_rate": 2.1340939468228963e-08,
+      "logits/chosen": -2.973597526550293,
+      "logits/rejected": -2.9438486099243164,
+      "logps/chosen": -57.69378662109375,
+      "logps/rejected": -57.61130905151367,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.038613900542259216,
+      "rewards/margins": 0.019549783319234848,
+      "rewards/rejected": -0.058163683861494064,
+      "step": 3700
+    },
+    {
+      "epoch": 1.2749827705031014,
+      "eval_logits/chosen": -3.1105759143829346,
+      "eval_logits/rejected": -3.104912519454956,
+      "eval_logps/chosen": -60.00641632080078,
+      "eval_logps/rejected": -65.19781494140625,
+      "eval_loss": 0.6897803544998169,
+      "eval_rewards/accuracies": 0.5820167064666748,
+      "eval_rewards/chosen": -0.012945191003382206,
+      "eval_rewards/margins": 0.007231764029711485,
+      "eval_rewards/rejected": -0.020176956430077553,
+      "eval_runtime": 383.4062,
+      "eval_samples_per_second": 11.226,
+      "eval_steps_per_second": 1.403,
+      "step": 3700
+    },
+    {
+      "epoch": 1.2784286698828393,
+      "grad_norm": 1.8971093893051147,
+      "learning_rate": 2.1286381546989183e-08,
+      "logits/chosen": -2.913695812225342,
+      "logits/rejected": -2.8849892616271973,
+      "logps/chosen": -58.248626708984375,
+      "logps/rejected": -58.3831901550293,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.03286467120051384,
+      "rewards/margins": 0.02052047662436962,
+      "rewards/rejected": -0.05338514968752861,
+      "step": 3710
+    },
+    {
+      "epoch": 1.2818745692625775,
+      "grad_norm": 1.8010145425796509,
+      "learning_rate": 2.123172255583485e-08,
+      "logits/chosen": -2.9989848136901855,
+      "logits/rejected": -2.969888210296631,
+      "logps/chosen": -58.8484001159668,
+      "logps/rejected": -59.0714111328125,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.04143872112035751,
+      "rewards/margins": 0.02306785061955452,
+      "rewards/rejected": -0.06450657546520233,
+      "step": 3720
+    },
+    {
+      "epoch": 1.2853204686423156,
+      "grad_norm": 1.7168934345245361,
+      "learning_rate": 2.117696337355123e-08,
+      "logits/chosen": -3.0218636989593506,
+      "logits/rejected": -3.009770393371582,
+      "logps/chosen": -56.76362991333008,
+      "logps/rejected": -58.4400634765625,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.03719615191221237,
+      "rewards/margins": 0.019225627183914185,
+      "rewards/rejected": -0.056421779096126556,
+      "step": 3730
+    },
+    {
+      "epoch": 1.2887663680220538,
+      "grad_norm": 1.734363317489624,
+      "learning_rate": 2.1122104880534452e-08,
+      "logits/chosen": -2.9942357540130615,
+      "logits/rejected": -2.971665382385254,
+      "logps/chosen": -58.45783615112305,
+      "logps/rejected": -55.61717987060547,
+      "loss": 0.686,
+      "rewards/accuracies": 0.559374988079071,
+      "rewards/chosen": -0.038143716752529144,
+      "rewards/margins": 0.015472421422600746,
+      "rewards/rejected": -0.05361613631248474,
+      "step": 3740
+    },
+    {
+      "epoch": 1.292212267401792,
+      "grad_norm": 1.5955020189285278,
+      "learning_rate": 2.1067147958777296e-08,
+      "logits/chosen": -2.9527993202209473,
+      "logits/rejected": -2.9336400032043457,
+      "logps/chosen": -55.96992874145508,
+      "logps/rejected": -58.29621505737305,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.039525531232357025,
+      "rewards/margins": 0.017807170748710632,
+      "rewards/rejected": -0.057332705706357956,
+      "step": 3750
+    },
+    {
+      "epoch": 1.29565816678153,
+      "grad_norm": 2.0012519359588623,
+      "learning_rate": 2.1012093491855055e-08,
+      "logits/chosen": -2.9845707416534424,
+      "logits/rejected": -2.957467555999756,
+      "logps/chosen": -58.6512336730957,
+      "logps/rejected": -58.24448776245117,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.03556809574365616,
+      "rewards/margins": 0.02148427814245224,
+      "rewards/rejected": -0.0570523738861084,
+      "step": 3760
+    },
+    {
+      "epoch": 1.299104066161268,
+      "grad_norm": 1.6091116666793823,
+      "learning_rate": 2.0956942364911308e-08,
+      "logits/chosen": -3.002842426300049,
+      "logits/rejected": -2.998654842376709,
+      "logps/chosen": -55.65495681762695,
+      "logps/rejected": -58.356651306152344,
+      "loss": 0.6861,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.03776023909449577,
+      "rewards/margins": 0.01524999551475048,
+      "rewards/rejected": -0.0530102364718914,
+      "step": 3770
+    },
+    {
+      "epoch": 1.3025499655410062,
+      "grad_norm": 1.8970427513122559,
+      "learning_rate": 2.0901695464643695e-08,
+      "logits/chosen": -3.0437119007110596,
+      "logits/rejected": -3.02368426322937,
+      "logps/chosen": -55.6706657409668,
+      "logps/rejected": -59.48578643798828,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.034878116101026535,
+      "rewards/margins": 0.024099189788103104,
+      "rewards/rejected": -0.05897730588912964,
+      "step": 3780
+    },
+    {
+      "epoch": 1.3059958649207444,
+      "grad_norm": 1.8609449863433838,
+      "learning_rate": 2.0846353679289663e-08,
+      "logits/chosen": -3.0514163970947266,
+      "logits/rejected": -3.032944440841675,
+      "logps/chosen": -59.740631103515625,
+      "logps/rejected": -59.10683059692383,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.03335235267877579,
+      "rewards/margins": 0.023155713453888893,
+      "rewards/rejected": -0.05650807172060013,
+      "step": 3790
+    },
+    {
+      "epoch": 1.3094417643004825,
+      "grad_norm": 1.6657793521881104,
+      "learning_rate": 2.079091789861218e-08,
+      "logits/chosen": -3.0490851402282715,
+      "logits/rejected": -3.011460781097412,
+      "logps/chosen": -59.69109344482422,
+      "logps/rejected": -56.5496711730957,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.04261072725057602,
+      "rewards/margins": 0.022843683138489723,
+      "rewards/rejected": -0.06545441597700119,
+      "step": 3800
+    },
+    {
+      "epoch": 1.3094417643004825,
+      "eval_logits/chosen": -3.107984781265259,
+      "eval_logits/rejected": -3.1023285388946533,
+      "eval_logps/chosen": -60.157962799072266,
+      "eval_logps/rejected": -65.38496398925781,
+      "eval_loss": 0.6896213889122009,
+      "eval_rewards/accuracies": 0.5831784605979919,
+      "eval_rewards/chosen": -0.014460649341344833,
+      "eval_rewards/margins": 0.007587699685245752,
+      "eval_rewards/rejected": -0.022048350423574448,
+      "eval_runtime": 383.33,
+      "eval_samples_per_second": 11.228,
+      "eval_steps_per_second": 1.403,
+      "step": 3800
+    },
+    {
+      "epoch": 1.3128876636802205,
+      "grad_norm": 1.7834899425506592,
+      "learning_rate": 2.0735389013885436e-08,
+      "logits/chosen": -3.003415107727051,
+      "logits/rejected": -2.973167657852173,
+      "logps/chosen": -58.896873474121094,
+      "logps/rejected": -57.619667053222656,
+      "loss": 0.6841,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.03684524446725845,
+      "rewards/margins": 0.019272824749350548,
+      "rewards/rejected": -0.05611807852983475,
+      "step": 3810
+    },
+    {
+      "epoch": 1.3163335630599586,
+      "grad_norm": 2.005615472793579,
+      "learning_rate": 2.0679767917880512e-08,
+      "logits/chosen": -2.9415082931518555,
+      "logits/rejected": -2.936711072921753,
+      "logps/chosen": -57.9473991394043,
+      "logps/rejected": -58.87640380859375,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": -0.0407208576798439,
+      "rewards/margins": 0.01581592857837677,
+      "rewards/rejected": -0.05653678625822067,
+      "step": 3820
+    },
+    {
+      "epoch": 1.3197794624396968,
+      "grad_norm": 2.1090593338012695,
+      "learning_rate": 2.0624055504851023e-08,
+      "logits/chosen": -3.0444393157958984,
+      "logits/rejected": -3.0121498107910156,
+      "logps/chosen": -61.757408142089844,
+      "logps/rejected": -61.631202697753906,
+      "loss": 0.6823,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.04044077917933464,
+      "rewards/margins": 0.023105397820472717,
+      "rewards/rejected": -0.06354617327451706,
+      "step": 3830
+    },
+    {
+      "epoch": 1.323225361819435,
+      "grad_norm": 1.7832638025283813,
+      "learning_rate": 2.056825267051874e-08,
+      "logits/chosen": -3.0216939449310303,
+      "logits/rejected": -2.998185396194458,
+      "logps/chosen": -58.012786865234375,
+      "logps/rejected": -57.43598556518555,
+      "loss": 0.6829,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.039224445819854736,
+      "rewards/margins": 0.02190742827951908,
+      "rewards/rejected": -0.06113187596201897,
+      "step": 3840
+    },
+    {
+      "epoch": 1.3266712611991731,
+      "grad_norm": 1.9296550750732422,
+      "learning_rate": 2.0512360312059194e-08,
+      "logits/chosen": -3.0729854106903076,
+      "logits/rejected": -3.0337021350860596,
+      "logps/chosen": -57.72249221801758,
+      "logps/rejected": -56.97220993041992,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.03375047445297241,
+      "rewards/margins": 0.022184688597917557,
+      "rewards/rejected": -0.05593516305088997,
+      "step": 3850
+    },
+    {
+      "epoch": 1.330117160578911,
+      "grad_norm": 1.8421112298965454,
+      "learning_rate": 2.0456379328087243e-08,
+      "logits/chosen": -3.0334370136260986,
+      "logits/rejected": -3.0075058937072754,
+      "logps/chosen": -58.4967155456543,
+      "logps/rejected": -57.10326385498047,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.046391695737838745,
+      "rewards/margins": 0.01595095731317997,
+      "rewards/rejected": -0.062342651188373566,
+      "step": 3860
+    },
+    {
+      "epoch": 1.3335630599586492,
+      "grad_norm": 1.7700375318527222,
+      "learning_rate": 2.0400310618642643e-08,
+      "logits/chosen": -2.9747002124786377,
+      "logits/rejected": -2.9553043842315674,
+      "logps/chosen": -57.07194137573242,
+      "logps/rejected": -59.48151779174805,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04258308932185173,
+      "rewards/margins": 0.02198188565671444,
+      "rewards/rejected": -0.06456498056650162,
+      "step": 3870
+    },
+    {
+      "epoch": 1.3370089593383874,
+      "grad_norm": 1.8381158113479614,
+      "learning_rate": 2.0344155085175552e-08,
+      "logits/chosen": -2.939208507537842,
+      "logits/rejected": -2.91402006149292,
+      "logps/chosen": -56.20661544799805,
+      "logps/rejected": -58.39705276489258,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.03981857746839523,
+      "rewards/margins": 0.024724584072828293,
+      "rewards/rejected": -0.06454316526651382,
+      "step": 3880
+    },
+    {
+      "epoch": 1.3404548587181253,
+      "grad_norm": 1.7165347337722778,
+      "learning_rate": 2.0287913630532062e-08,
+      "logits/chosen": -3.019341468811035,
+      "logits/rejected": -2.9981906414031982,
+      "logps/chosen": -56.41069412231445,
+      "logps/rejected": -59.48908615112305,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.045831069350242615,
+      "rewards/margins": 0.01706528849899769,
+      "rewards/rejected": -0.06289635598659515,
+      "step": 3890
+    },
+    {
+      "epoch": 1.3439007580978635,
+      "grad_norm": 1.8775408267974854,
+      "learning_rate": 2.0231587158939657e-08,
+      "logits/chosen": -3.047612428665161,
+      "logits/rejected": -3.037127733230591,
+      "logps/chosen": -57.39014434814453,
+      "logps/rejected": -59.7318229675293,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.04614701122045517,
+      "rewards/margins": 0.018029648810625076,
+      "rewards/rejected": -0.06417665630578995,
+      "step": 3900
+    },
+    {
+      "epoch": 1.3439007580978635,
+      "eval_logits/chosen": -3.1063618659973145,
+      "eval_logits/rejected": -3.1006975173950195,
+      "eval_logps/chosen": -60.32647705078125,
+      "eval_logps/rejected": -65.57601928710938,
+      "eval_loss": 0.6895313858985901,
+      "eval_rewards/accuracies": 0.5834107995033264,
+      "eval_rewards/chosen": -0.01614583656191826,
+      "eval_rewards/margins": 0.007813144475221634,
+      "eval_rewards/rejected": -0.023958981037139893,
+      "eval_runtime": 383.1132,
+      "eval_samples_per_second": 11.234,
+      "eval_steps_per_second": 1.404,
+      "step": 3900
+    },
+    {
+      "epoch": 1.3473466574776016,
+      "grad_norm": 1.6905556917190552,
+      "learning_rate": 2.01751765759927e-08,
+      "logits/chosen": -3.0624446868896484,
+      "logits/rejected": -3.0323171615600586,
+      "logps/chosen": -61.1554069519043,
+      "logps/rejected": -60.69427490234375,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.03463874012231827,
+      "rewards/margins": 0.021960586309432983,
+      "rewards/rejected": -0.05659932643175125,
+      "step": 3910
+    },
+    {
+      "epoch": 1.3507925568573398,
+      "grad_norm": 1.6505118608474731,
+      "learning_rate": 2.0118682788637858e-08,
+      "logits/chosen": -2.945446491241455,
+      "logits/rejected": -2.93007755279541,
+      "logps/chosen": -56.24365234375,
+      "logps/rejected": -60.277809143066406,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.045598771423101425,
+      "rewards/margins": 0.01887628622353077,
+      "rewards/rejected": -0.06447505950927734,
+      "step": 3920
+    },
+    {
+      "epoch": 1.354238456237078,
+      "grad_norm": 1.8426750898361206,
+      "learning_rate": 2.006210670515953e-08,
+      "logits/chosen": -2.9543869495391846,
+      "logits/rejected": -2.9196228981018066,
+      "logps/chosen": -58.40874099731445,
+      "logps/rejected": -57.23945999145508,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.04171986132860184,
+      "rewards/margins": 0.025528009980916977,
+      "rewards/rejected": -0.06724786758422852,
+      "step": 3930
+    },
+    {
+      "epoch": 1.3576843556168159,
+      "grad_norm": 1.9124021530151367,
+      "learning_rate": 2.000544923516523e-08,
+      "logits/chosen": -3.024339199066162,
+      "logits/rejected": -3.0077013969421387,
+      "logps/chosen": -58.501304626464844,
+      "logps/rejected": -59.74248123168945,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.041075319051742554,
+      "rewards/margins": 0.021968619897961617,
+      "rewards/rejected": -0.06304393708705902,
+      "step": 3940
+    },
+    {
+      "epoch": 1.361130254996554,
+      "grad_norm": 2.0051000118255615,
+      "learning_rate": 1.9948711289570974e-08,
+      "logits/chosen": -2.988908052444458,
+      "logits/rejected": -2.97766375541687,
+      "logps/chosen": -57.61002731323242,
+      "logps/rejected": -60.394065856933594,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.04333911091089249,
+      "rewards/margins": 0.016101833432912827,
+      "rewards/rejected": -0.05944094806909561,
+      "step": 3950
+    },
+    {
+      "epoch": 1.3645761543762922,
+      "grad_norm": 1.8345589637756348,
+      "learning_rate": 1.9891893780586643e-08,
+      "logits/chosen": -2.987314462661743,
+      "logits/rejected": -2.9634814262390137,
+      "logps/chosen": -60.20556640625,
+      "logps/rejected": -58.74542999267578,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.04383495822548866,
+      "rewards/margins": 0.01806296408176422,
+      "rewards/rejected": -0.061897922307252884,
+      "step": 3960
+    },
+    {
+      "epoch": 1.3680220537560304,
+      "grad_norm": 1.9435667991638184,
+      "learning_rate": 1.98349976217013e-08,
+      "logits/chosen": -2.9943671226501465,
+      "logits/rejected": -2.9723329544067383,
+      "logps/chosen": -62.5230827331543,
+      "logps/rejected": -60.296409606933594,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.040756478905677795,
+      "rewards/margins": 0.024111924692988396,
+      "rewards/rejected": -0.06486840546131134,
+      "step": 3970
+    },
+    {
+      "epoch": 1.3714679531357685,
+      "grad_norm": 1.7903798818588257,
+      "learning_rate": 1.97780237276685e-08,
+      "logits/chosen": -2.9087538719177246,
+      "logits/rejected": -2.8901820182800293,
+      "logps/chosen": -55.32871627807617,
+      "logps/rejected": -59.03423309326172,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.04336182028055191,
+      "rewards/margins": 0.01821316033601761,
+      "rewards/rejected": -0.06157498434185982,
+      "step": 3980
+    },
+    {
+      "epoch": 1.3749138525155065,
+      "grad_norm": 1.9202241897583008,
+      "learning_rate": 1.97209730144916e-08,
+      "logits/chosen": -3.041247606277466,
+      "logits/rejected": -3.0187203884124756,
+      "logps/chosen": -60.6109504699707,
+      "logps/rejected": -60.707000732421875,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.0447247214615345,
+      "rewards/margins": 0.021457429975271225,
+      "rewards/rejected": -0.06618215143680573,
+      "step": 3990
+    },
+    {
+      "epoch": 1.3783597518952446,
+      "grad_norm": 1.9562245607376099,
+      "learning_rate": 1.966384639940903e-08,
+      "logits/chosen": -3.0206329822540283,
+      "logits/rejected": -3.0095901489257812,
+      "logps/chosen": -57.0920295715332,
+      "logps/rejected": -59.912017822265625,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": -0.04499662667512894,
+      "rewards/margins": 0.01461687684059143,
+      "rewards/rejected": -0.05961349606513977,
+      "step": 4000
+    },
+    {
+      "epoch": 1.3783597518952446,
+      "eval_logits/chosen": -3.104668140411377,
+      "eval_logits/rejected": -3.09900164604187,
+      "eval_logps/chosen": -60.506080627441406,
+      "eval_logps/rejected": -65.78731536865234,
+      "eval_loss": 0.6893932223320007,
+      "eval_rewards/accuracies": 0.5875929594039917,
+      "eval_rewards/chosen": -0.017941860482096672,
+      "eval_rewards/margins": 0.00813000276684761,
+      "eval_rewards/rejected": -0.026071859523653984,
+      "eval_runtime": 382.5872,
+      "eval_samples_per_second": 11.25,
+      "eval_steps_per_second": 1.406,
+      "step": 4000
+    },
+    {
+      "epoch": 1.3818056512749828,
+      "grad_norm": 1.9249037504196167,
+      "learning_rate": 1.9606644800879535e-08,
+      "logits/chosen": -2.9845852851867676,
+      "logits/rejected": -2.9537439346313477,
+      "logps/chosen": -63.1302604675293,
+      "logps/rejected": -59.92966842651367,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.044314321130514145,
+      "rewards/margins": 0.019151529297232628,
+      "rewards/rejected": -0.06346584856510162,
+      "step": 4010
+    },
+    {
+      "epoch": 1.385251550654721,
+      "grad_norm": 1.7754579782485962,
+      "learning_rate": 1.9549369138567404e-08,
+      "logits/chosen": -2.9692628383636475,
+      "logits/rejected": -2.9454288482666016,
+      "logps/chosen": -57.2863655090332,
+      "logps/rejected": -59.29479217529297,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.04501374438405037,
+      "rewards/margins": 0.020707763731479645,
+      "rewards/rejected": -0.06572151184082031,
+      "step": 4020
+    },
+    {
+      "epoch": 1.388697450034459,
+      "grad_norm": 1.95672607421875,
+      "learning_rate": 1.9492020333327714e-08,
+      "logits/chosen": -3.0381829738616943,
+      "logits/rejected": -3.0096137523651123,
+      "logps/chosen": -59.01328659057617,
+      "logps/rejected": -59.451866149902344,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.037902332842350006,
+      "rewards/margins": 0.026948988437652588,
+      "rewards/rejected": -0.0648513212800026,
+      "step": 4030
+    },
+    {
+      "epoch": 1.392143349414197,
+      "grad_norm": 1.9216549396514893,
+      "learning_rate": 1.9434599307191496e-08,
+      "logits/chosen": -2.9591314792633057,
+      "logits/rejected": -2.9411370754241943,
+      "logps/chosen": -57.56044387817383,
+      "logps/rejected": -60.2334098815918,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.0433807373046875,
+      "rewards/margins": 0.01931798830628395,
+      "rewards/rejected": -0.06269872933626175,
+      "step": 4040
+    },
+    {
+      "epoch": 1.3955892487939352,
+      "grad_norm": 1.9152655601501465,
+      "learning_rate": 1.9377106983350902e-08,
+      "logits/chosen": -2.969998836517334,
+      "logits/rejected": -2.9522013664245605,
+      "logps/chosen": -57.877784729003906,
+      "logps/rejected": -61.0090217590332,
+      "loss": 0.684,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.042776696383953094,
+      "rewards/margins": 0.019583266228437424,
+      "rewards/rejected": -0.06235996633768082,
+      "step": 4050
+    },
+    {
+      "epoch": 1.3990351481736734,
+      "grad_norm": 1.9686765670776367,
+      "learning_rate": 1.931954428614441e-08,
+      "logits/chosen": -2.9932191371917725,
+      "logits/rejected": -2.9646401405334473,
+      "logps/chosen": -59.35251998901367,
+      "logps/rejected": -58.298484802246094,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0427054762840271,
+      "rewards/margins": 0.020643148571252823,
+      "rewards/rejected": -0.06334862858057022,
+      "step": 4060
+    },
+    {
+      "epoch": 1.4024810475534115,
+      "grad_norm": 1.901611566543579,
+      "learning_rate": 1.926191214104191e-08,
+      "logits/chosen": -2.966418743133545,
+      "logits/rejected": -2.9444503784179688,
+      "logps/chosen": -58.21308135986328,
+      "logps/rejected": -57.963043212890625,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.0443955734372139,
+      "rewards/margins": 0.018185999244451523,
+      "rewards/rejected": -0.06258156895637512,
+      "step": 4070
+    },
+    {
+      "epoch": 1.4059269469331497,
+      "grad_norm": 1.9883493185043335,
+      "learning_rate": 1.920421147462986e-08,
+      "logits/chosen": -2.9970450401306152,
+      "logits/rejected": -2.976762294769287,
+      "logps/chosen": -58.84798049926758,
+      "logps/rejected": -59.698265075683594,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.04767550155520439,
+      "rewards/margins": 0.022497477009892464,
+      "rewards/rejected": -0.070172980427742,
+      "step": 4080
+    },
+    {
+      "epoch": 1.4093728463128876,
+      "grad_norm": 2.113548994064331,
+      "learning_rate": 1.914644321459637e-08,
+      "logits/chosen": -3.022745132446289,
+      "logits/rejected": -2.9992973804473877,
+      "logps/chosen": -62.230690002441406,
+      "logps/rejected": -59.467498779296875,
+      "loss": 0.685,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.04229141026735306,
+      "rewards/margins": 0.017566503956913948,
+      "rewards/rejected": -0.05985791236162186,
+      "step": 4090
+    },
+    {
+      "epoch": 1.4128187456926258,
+      "grad_norm": 1.7836883068084717,
+      "learning_rate": 1.908860828971629e-08,
+      "logits/chosen": -2.993814468383789,
+      "logits/rejected": -2.9648101329803467,
+      "logps/chosen": -60.8956413269043,
+      "logps/rejected": -59.87592315673828,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04408767819404602,
+      "rewards/margins": 0.02260351926088333,
+      "rewards/rejected": -0.06669119745492935,
+      "step": 4100
+    },
+    {
+      "epoch": 1.4128187456926258,
+      "eval_logits/chosen": -3.10246205329895,
+      "eval_logits/rejected": -3.0967519283294678,
+      "eval_logps/chosen": -60.67816162109375,
+      "eval_logps/rejected": -65.99722290039062,
+      "eval_loss": 0.6892262101173401,
+      "eval_rewards/accuracies": 0.5899163484573364,
+      "eval_rewards/chosen": -0.019662661477923393,
+      "eval_rewards/margins": 0.008508284576237202,
+      "eval_rewards/rejected": -0.02817094698548317,
+      "eval_runtime": 383.0743,
+      "eval_samples_per_second": 11.235,
+      "eval_steps_per_second": 1.404,
+      "step": 4100
+    },
+    {
+      "epoch": 1.416264645072364,
+      "grad_norm": 1.7881430387496948,
+      "learning_rate": 1.9030707629836295e-08,
+      "logits/chosen": -2.9510140419006348,
+      "logits/rejected": -2.9293253421783447,
+      "logps/chosen": -58.771156311035156,
+      "logps/rejected": -60.960304260253906,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.04672146588563919,
+      "rewards/margins": 0.016872262582182884,
+      "rewards/rejected": -0.06359373033046722,
+      "step": 4110
+    },
+    {
+      "epoch": 1.4197105444521019,
+      "grad_norm": 1.8163260221481323,
+      "learning_rate": 1.8972742165859898e-08,
+      "logits/chosen": -3.0287904739379883,
+      "logits/rejected": -3.004352331161499,
+      "logps/chosen": -60.103309631347656,
+      "logps/rejected": -58.08264923095703,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.043921858072280884,
+      "rewards/margins": 0.02099820412695408,
+      "rewards/rejected": -0.06492006033658981,
+      "step": 4120
+    },
+    {
+      "epoch": 1.42315644383184,
+      "grad_norm": 1.9235758781433105,
+      "learning_rate": 1.891471282973253e-08,
+      "logits/chosen": -3.0585458278656006,
+      "logits/rejected": -3.0358951091766357,
+      "logps/chosen": -61.297950744628906,
+      "logps/rejected": -61.27184295654297,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04265972226858139,
+      "rewards/margins": 0.022031161934137344,
+      "rewards/rejected": -0.06469088792800903,
+      "step": 4130
+    },
+    {
+      "epoch": 1.4266023432115782,
+      "grad_norm": 1.885871410369873,
+      "learning_rate": 1.8856620554426507e-08,
+      "logits/chosen": -3.022674798965454,
+      "logits/rejected": -3.011600971221924,
+      "logps/chosen": -58.532981872558594,
+      "logps/rejected": -60.877784729003906,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.05646634101867676,
+      "rewards/margins": 0.011095132678747177,
+      "rewards/rejected": -0.06756147742271423,
+      "step": 4140
+    },
+    {
+      "epoch": 1.4300482425913164,
+      "grad_norm": 1.9023128747940063,
+      "learning_rate": 1.8798466273926088e-08,
+      "logits/chosen": -3.0471909046173096,
+      "logits/rejected": -3.010711193084717,
+      "logps/chosen": -62.831748962402344,
+      "logps/rejected": -60.98163604736328,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.04073944315314293,
+      "rewards/margins": 0.025459637865424156,
+      "rewards/rejected": -0.06619907915592194,
+      "step": 4150
+    },
+    {
+      "epoch": 1.4334941419710545,
+      "grad_norm": 1.6735082864761353,
+      "learning_rate": 1.8740250923212405e-08,
+      "logits/chosen": -2.949036121368408,
+      "logits/rejected": -2.9251441955566406,
+      "logps/chosen": -58.65555953979492,
+      "logps/rejected": -58.7363166809082,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.042642705142498016,
+      "rewards/margins": 0.018682269379496574,
+      "rewards/rejected": -0.06132497638463974,
+      "step": 4160
+    },
+    {
+      "epoch": 1.4369400413507925,
+      "grad_norm": 1.7913520336151123,
+      "learning_rate": 1.868197543824845e-08,
+      "logits/chosen": -3.0192484855651855,
+      "logits/rejected": -3.0067028999328613,
+      "logps/chosen": -56.67252731323242,
+      "logps/rejected": -59.903236389160156,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.0514332577586174,
+      "rewards/margins": 0.01997419074177742,
+      "rewards/rejected": -0.07140744477510452,
+      "step": 4170
+    },
+    {
+      "epoch": 1.4403859407305306,
+      "grad_norm": 1.7646011114120483,
+      "learning_rate": 1.862364075596404e-08,
+      "logits/chosen": -2.8959903717041016,
+      "logits/rejected": -2.868590831756592,
+      "logps/chosen": -58.8744010925293,
+      "logps/rejected": -59.3974494934082,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.05011838674545288,
+      "rewards/margins": 0.021377570927143097,
+      "rewards/rejected": -0.07149595767259598,
+      "step": 4180
+    },
+    {
+      "epoch": 1.4438318401102688,
+      "grad_norm": 1.7195109128952026,
+      "learning_rate": 1.8565247814240736e-08,
+      "logits/chosen": -3.032365322113037,
+      "logits/rejected": -3.0138723850250244,
+      "logps/chosen": -56.07634735107422,
+      "logps/rejected": -60.448585510253906,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04927598312497139,
+      "rewards/margins": 0.016410915181040764,
+      "rewards/rejected": -0.065686896443367,
+      "step": 4190
+    },
+    {
+      "epoch": 1.447277739490007,
+      "grad_norm": 1.9616376161575317,
+      "learning_rate": 1.8506797551896777e-08,
+      "logits/chosen": -3.0298922061920166,
+      "logits/rejected": -3.0022029876708984,
+      "logps/chosen": -58.03533935546875,
+      "logps/rejected": -60.732139587402344,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.04534187912940979,
+      "rewards/margins": 0.027529925107955933,
+      "rewards/rejected": -0.07287180423736572,
+      "step": 4200
+    },
+    {
+      "epoch": 1.447277739490007,
+      "eval_logits/chosen": -3.100907325744629,
+      "eval_logits/rejected": -3.0952022075653076,
+      "eval_logps/chosen": -60.80016326904297,
+      "eval_logps/rejected": -66.16580200195312,
+      "eval_loss": 0.6890069246292114,
+      "eval_rewards/accuracies": 0.5922397971153259,
+      "eval_rewards/chosen": -0.02088271640241146,
+      "eval_rewards/margins": 0.008974066935479641,
+      "eval_rewards/rejected": -0.029856784269213676,
+      "eval_runtime": 383.2122,
+      "eval_samples_per_second": 11.231,
+      "eval_steps_per_second": 1.404,
+      "step": 4200
+    },
+    {
+      "epoch": 1.450723638869745,
+      "grad_norm": 1.7890739440917969,
+      "learning_rate": 1.8448290908671972e-08,
+      "logits/chosen": -2.9154767990112305,
+      "logits/rejected": -2.8918347358703613,
+      "logps/chosen": -59.66216278076172,
+      "logps/rejected": -59.23423385620117,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0500604510307312,
+      "rewards/margins": 0.02689215913414955,
+      "rewards/rejected": -0.07695260643959045,
+      "step": 4210
+    },
+    {
+      "epoch": 1.454169538249483,
+      "grad_norm": 2.014331102371216,
+      "learning_rate": 1.8389728825212608e-08,
+      "logits/chosen": -3.0045652389526367,
+      "logits/rejected": -2.987805128097534,
+      "logps/chosen": -58.788124084472656,
+      "logps/rejected": -60.974884033203125,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.04646112024784088,
+      "rewards/margins": 0.022358350455760956,
+      "rewards/rejected": -0.06881947815418243,
+      "step": 4220
+    },
+    {
+      "epoch": 1.4576154376292212,
+      "grad_norm": 1.7566518783569336,
+      "learning_rate": 1.833111224305631e-08,
+      "logits/chosen": -2.962555170059204,
+      "logits/rejected": -2.946044445037842,
+      "logps/chosen": -55.21477508544922,
+      "logps/rejected": -60.76020050048828,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0480552539229393,
+      "rewards/margins": 0.026367992162704468,
+      "rewards/rejected": -0.07442324608564377,
+      "step": 4230
+    },
+    {
+      "epoch": 1.4610613370089593,
+      "grad_norm": 1.7687503099441528,
+      "learning_rate": 1.8272442104616915e-08,
+      "logits/chosen": -2.9924545288085938,
+      "logits/rejected": -2.9628946781158447,
+      "logps/chosen": -59.634132385253906,
+      "logps/rejected": -59.739341735839844,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.05082297325134277,
+      "rewards/margins": 0.022901121526956558,
+      "rewards/rejected": -0.07372410595417023,
+      "step": 4240
+    },
+    {
+      "epoch": 1.4645072363886975,
+      "grad_norm": 1.7194262742996216,
+      "learning_rate": 1.8213719353169315e-08,
+      "logits/chosen": -2.9741430282592773,
+      "logits/rejected": -2.9631612300872803,
+      "logps/chosen": -59.46907424926758,
+      "logps/rejected": -62.38594436645508,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.049456797540187836,
+      "rewards/margins": 0.021166514605283737,
+      "rewards/rejected": -0.07062331587076187,
+      "step": 4250
+    },
+    {
+      "epoch": 1.4679531357684357,
+      "grad_norm": 1.8347423076629639,
+      "learning_rate": 1.8154944932834296e-08,
+      "logits/chosen": -2.977309465408325,
+      "logits/rejected": -2.941486358642578,
+      "logps/chosen": -60.53784942626953,
+      "logps/rejected": -57.88445281982422,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.048282440751791,
+      "rewards/margins": 0.029451463371515274,
+      "rewards/rejected": -0.07773390412330627,
+      "step": 4260
+    },
+    {
+      "epoch": 1.4713990351481736,
+      "grad_norm": 2.009308338165283,
+      "learning_rate": 1.8096119788563352e-08,
+      "logits/chosen": -3.02274489402771,
+      "logits/rejected": -3.0057437419891357,
+      "logps/chosen": -59.3822135925293,
+      "logps/rejected": -59.544395446777344,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.05091637372970581,
+      "rewards/margins": 0.015222887508571148,
+      "rewards/rejected": -0.06613925844430923,
+      "step": 4270
+    },
+    {
+      "epoch": 1.4748449345279118,
+      "grad_norm": 1.8076480627059937,
+      "learning_rate": 1.8037244866123498e-08,
+      "logits/chosen": -3.0521037578582764,
+      "logits/rejected": -3.0367977619171143,
+      "logps/chosen": -59.16364669799805,
+      "logps/rejected": -61.063934326171875,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.051410503685474396,
+      "rewards/margins": 0.020908212289214134,
+      "rewards/rejected": -0.07231871038675308,
+      "step": 4280
+    },
+    {
+      "epoch": 1.47829083390765,
+      "grad_norm": 1.8789877891540527,
+      "learning_rate": 1.7978321112082057e-08,
+      "logits/chosen": -2.9959309101104736,
+      "logits/rejected": -2.9724977016448975,
+      "logps/chosen": -58.74253463745117,
+      "logps/rejected": -60.813255310058594,
+      "loss": 0.6787,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.0449586883187294,
+      "rewards/margins": 0.030271431431174278,
+      "rewards/rejected": -0.07523011416196823,
+      "step": 4290
+    },
+    {
+      "epoch": 1.481736733287388,
+      "grad_norm": 2.009160041809082,
+      "learning_rate": 1.7919349473791454e-08,
+      "logits/chosen": -3.007607936859131,
+      "logits/rejected": -2.9768214225769043,
+      "logps/chosen": -59.56440353393555,
+      "logps/rejected": -59.418724060058594,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.043479979038238525,
+      "rewards/margins": 0.025058060884475708,
+      "rewards/rejected": -0.06853803247213364,
+      "step": 4300
+    },
+    {
+      "epoch": 1.481736733287388,
+      "eval_logits/chosen": -3.098330020904541,
+      "eval_logits/rejected": -3.092646360397339,
+      "eval_logps/chosen": -60.978885650634766,
+      "eval_logps/rejected": -66.35769653320312,
+      "eval_loss": 0.6889605522155762,
+      "eval_rewards/accuracies": 0.5878252983093262,
+      "eval_rewards/chosen": -0.02266988344490528,
+      "eval_rewards/margins": 0.009105909615755081,
+      "eval_rewards/rejected": -0.03177579492330551,
+      "eval_runtime": 383.2468,
+      "eval_samples_per_second": 11.23,
+      "eval_steps_per_second": 1.404,
+      "step": 4300
+    },
+    {
+      "epoch": 1.4851826326671262,
+      "grad_norm": 1.864001750946045,
+      "learning_rate": 1.7860330899373974e-08,
+      "logits/chosen": -2.952350378036499,
+      "logits/rejected": -2.9315459728240967,
+      "logps/chosen": -59.5748291015625,
+      "logps/rejected": -58.25684356689453,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.051153045147657394,
+      "rewards/margins": 0.022380802780389786,
+      "rewards/rejected": -0.07353384792804718,
+      "step": 4310
+    },
+    {
+      "epoch": 1.4886285320468642,
+      "grad_norm": 1.8543477058410645,
+      "learning_rate": 1.7801266337706528e-08,
+      "logits/chosen": -2.9646592140197754,
+      "logits/rejected": -2.943025588989258,
+      "logps/chosen": -58.435890197753906,
+      "logps/rejected": -58.97045135498047,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.047125715762376785,
+      "rewards/margins": 0.018387814983725548,
+      "rewards/rejected": -0.06551353633403778,
+      "step": 4320
+    },
+    {
+      "epoch": 1.4920744314266023,
+      "grad_norm": 1.7972218990325928,
+      "learning_rate": 1.774215673840538e-08,
+      "logits/chosen": -2.9578769207000732,
+      "logits/rejected": -2.9398512840270996,
+      "logps/chosen": -58.897186279296875,
+      "logps/rejected": -60.4238395690918,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.047395817935466766,
+      "rewards/margins": 0.0278643611818552,
+      "rewards/rejected": -0.07526017725467682,
+      "step": 4330
+    },
+    {
+      "epoch": 1.4955203308063405,
+      "grad_norm": 2.105286121368408,
+      "learning_rate": 1.768300305181091e-08,
+      "logits/chosen": -3.0316853523254395,
+      "logits/rejected": -3.0194015502929688,
+      "logps/chosen": -57.03474044799805,
+      "logps/rejected": -60.529991149902344,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.056976623833179474,
+      "rewards/margins": 0.02046811208128929,
+      "rewards/rejected": -0.07744473218917847,
+      "step": 4340
+    },
+    {
+      "epoch": 1.4989662301860784,
+      "grad_norm": 1.991560935974121,
+      "learning_rate": 1.7623806228972286e-08,
+      "logits/chosen": -3.021115779876709,
+      "logits/rejected": -2.989412784576416,
+      "logps/chosen": -60.84465789794922,
+      "logps/rejected": -57.801025390625,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.04827765375375748,
+      "rewards/margins": 0.02497757412493229,
+      "rewards/rejected": -0.07325522601604462,
+      "step": 4350
+    },
+    {
+      "epoch": 1.5024121295658168,
+      "grad_norm": 1.964712142944336,
+      "learning_rate": 1.7564567221632232e-08,
+      "logits/chosen": -2.9852466583251953,
+      "logits/rejected": -2.9653451442718506,
+      "logps/chosen": -59.42768478393555,
+      "logps/rejected": -60.49907302856445,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.04836093261837959,
+      "rewards/margins": 0.018619975075125694,
+      "rewards/rejected": -0.06698090583086014,
+      "step": 4360
+    },
+    {
+      "epoch": 1.5058580289455548,
+      "grad_norm": 1.79433274269104,
+      "learning_rate": 1.7505286982211685e-08,
+      "logits/chosen": -2.9940054416656494,
+      "logits/rejected": -2.972290515899658,
+      "logps/chosen": -57.57987594604492,
+      "logps/rejected": -59.9541015625,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.05742192268371582,
+      "rewards/margins": 0.01787840761244297,
+      "rewards/rejected": -0.07530032843351364,
+      "step": 4370
+    },
+    {
+      "epoch": 1.509303928325293,
+      "grad_norm": 1.7859464883804321,
+      "learning_rate": 1.744596646379449e-08,
+      "logits/chosen": -2.9818031787872314,
+      "logits/rejected": -2.964202404022217,
+      "logps/chosen": -57.210350036621094,
+      "logps/rejected": -61.47938919067383,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.04612854868173599,
+      "rewards/margins": 0.026797812432050705,
+      "rewards/rejected": -0.072926364839077,
+      "step": 4380
+    },
+    {
+      "epoch": 1.512749827705031,
+      "grad_norm": 2.046590566635132,
+      "learning_rate": 1.7386606620112092e-08,
+      "logits/chosen": -3.017162561416626,
+      "logits/rejected": -3.0030739307403564,
+      "logps/chosen": -57.23406982421875,
+      "logps/rejected": -59.33269119262695,
+      "loss": 0.681,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.051495350897312164,
+      "rewards/margins": 0.02578352950513363,
+      "rewards/rejected": -0.07727888226509094,
+      "step": 4390
+    },
+    {
+      "epoch": 1.516195727084769,
+      "grad_norm": 1.9171278476715088,
+      "learning_rate": 1.7327208405528187e-08,
+      "logits/chosen": -2.9808976650238037,
+      "logits/rejected": -2.9495692253112793,
+      "logps/chosen": -60.17070388793945,
+      "logps/rejected": -59.67145919799805,
+      "loss": 0.683,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.05209259316325188,
+      "rewards/margins": 0.022000344470143318,
+      "rewards/rejected": -0.07409293204545975,
+      "step": 4400
+    },
+    {
+      "epoch": 1.516195727084769,
+      "eval_logits/chosen": -3.096674919128418,
+      "eval_logits/rejected": -3.090972900390625,
+      "eval_logps/chosen": -61.10615921020508,
+      "eval_logps/rejected": -66.51582336425781,
+      "eval_loss": 0.6888236999511719,
+      "eval_rewards/accuracies": 0.5913103818893433,
+      "eval_rewards/chosen": -0.0239426102489233,
+      "eval_rewards/margins": 0.0094143720343709,
+      "eval_rewards/rejected": -0.033356983214616776,
+      "eval_runtime": 382.9986,
+      "eval_samples_per_second": 11.238,
+      "eval_steps_per_second": 1.405,
+      "step": 4400
+    },
+    {
+      "epoch": 1.5196416264645074,
+      "grad_norm": 1.9063295125961304,
+      "learning_rate": 1.7267772775023384e-08,
+      "logits/chosen": -3.0482637882232666,
+      "logits/rejected": -3.0266637802124023,
+      "logps/chosen": -56.29108810424805,
+      "logps/rejected": -58.86702346801758,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.05598801374435425,
+      "rewards/margins": 0.02073965221643448,
+      "rewards/rejected": -0.07672766596078873,
+      "step": 4410
+    },
+    {
+      "epoch": 1.5230875258442453,
+      "grad_norm": 1.9289114475250244,
+      "learning_rate": 1.720830068417985e-08,
+      "logits/chosen": -2.893359661102295,
+      "logits/rejected": -2.8771510124206543,
+      "logps/chosen": -58.58525466918945,
+      "logps/rejected": -60.37299346923828,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.05113077163696289,
+      "rewards/margins": 0.019904907792806625,
+      "rewards/rejected": -0.07103566825389862,
+      "step": 4420
+    },
+    {
+      "epoch": 1.5265334252239835,
+      "grad_norm": 1.8622808456420898,
+      "learning_rate": 1.714879308916594e-08,
+      "logits/chosen": -3.050687313079834,
+      "logits/rejected": -3.030336856842041,
+      "logps/chosen": -57.3572998046875,
+      "logps/rejected": -60.340492248535156,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.04641550034284592,
+      "rewards/margins": 0.02121780812740326,
+      "rewards/rejected": -0.06763330847024918,
+      "step": 4430
+    },
+    {
+      "epoch": 1.5299793246037217,
+      "grad_norm": 1.7250269651412964,
+      "learning_rate": 1.7089250946720846e-08,
+      "logits/chosen": -3.0424435138702393,
+      "logits/rejected": -3.0120060443878174,
+      "logps/chosen": -60.563575744628906,
+      "logps/rejected": -60.914894104003906,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.05453014373779297,
+      "rewards/margins": 0.021745886653661728,
+      "rewards/rejected": -0.0762760192155838,
+      "step": 4440
+    },
+    {
+      "epoch": 1.5334252239834596,
+      "grad_norm": 1.8410413265228271,
+      "learning_rate": 1.7029675214139183e-08,
+      "logits/chosen": -3.0003652572631836,
+      "logits/rejected": -2.97139310836792,
+      "logps/chosen": -59.795143127441406,
+      "logps/rejected": -59.26588821411133,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.05104909464716911,
+      "rewards/margins": 0.022785013541579247,
+      "rewards/rejected": -0.07383410632610321,
+      "step": 4450
+    },
+    {
+      "epoch": 1.5368711233631978,
+      "grad_norm": 2.036175012588501,
+      "learning_rate": 1.6970066849255625e-08,
+      "logits/chosen": -3.028881788253784,
+      "logits/rejected": -3.006293773651123,
+      "logps/chosen": -61.78435134887695,
+      "logps/rejected": -60.65825271606445,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.05423710495233536,
+      "rewards/margins": 0.02649405598640442,
+      "rewards/rejected": -0.08073116093873978,
+      "step": 4460
+    },
+    {
+      "epoch": 1.540317022742936,
+      "grad_norm": 2.03179931640625,
+      "learning_rate": 1.6910426810429494e-08,
+      "logits/chosen": -3.010624647140503,
+      "logits/rejected": -2.986023426055908,
+      "logps/chosen": -60.36248779296875,
+      "logps/rejected": -60.4931526184082,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.05200503021478653,
+      "rewards/margins": 0.02112220786511898,
+      "rewards/rejected": -0.07312722504138947,
+      "step": 4470
+    },
+    {
+      "epoch": 1.5437629221226739,
+      "grad_norm": 1.9399245977401733,
+      "learning_rate": 1.685075605652935e-08,
+      "logits/chosen": -2.972214937210083,
+      "logits/rejected": -2.959602117538452,
+      "logps/chosen": -59.70360565185547,
+      "logps/rejected": -62.060882568359375,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.054514139890670776,
+      "rewards/margins": 0.021584948524832726,
+      "rewards/rejected": -0.07609909027814865,
+      "step": 4480
+    },
+    {
+      "epoch": 1.5472088215024122,
+      "grad_norm": 1.8146494626998901,
+      "learning_rate": 1.6791055546917566e-08,
+      "logits/chosen": -3.026864528656006,
+      "logits/rejected": -2.9970510005950928,
+      "logps/chosen": -59.48024368286133,
+      "logps/rejected": -59.580543518066406,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.05597910284996033,
+      "rewards/margins": 0.02083490416407585,
+      "rewards/rejected": -0.07681401073932648,
+      "step": 4490
+    },
+    {
+      "epoch": 1.5506547208821502,
+      "grad_norm": 1.9819828271865845,
+      "learning_rate": 1.6731326241434947e-08,
+      "logits/chosen": -2.9944491386413574,
+      "logits/rejected": -2.97092866897583,
+      "logps/chosen": -60.771881103515625,
+      "logps/rejected": -60.873939514160156,
+      "loss": 0.679,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.045898132026195526,
+      "rewards/margins": 0.02983054891228676,
+      "rewards/rejected": -0.07572868466377258,
+      "step": 4500
+    },
+    {
+      "epoch": 1.5506547208821502,
+      "eval_logits/chosen": -3.0948894023895264,
+      "eval_logits/rejected": -3.0892059803009033,
+      "eval_logps/chosen": -61.26357650756836,
+      "eval_logps/rejected": -66.70375061035156,
+      "eval_loss": 0.6886882781982422,
+      "eval_rewards/accuracies": 0.5947955250740051,
+      "eval_rewards/chosen": -0.025516852736473083,
+      "eval_rewards/margins": 0.00971939880400896,
+      "eval_rewards/rejected": -0.03523625060915947,
+      "eval_runtime": 383.2368,
+      "eval_samples_per_second": 11.231,
+      "eval_steps_per_second": 1.404,
+      "step": 4500
+    },
+    {
+      "epoch": 1.5541006202618883,
+      "grad_norm": 1.7367281913757324,
+      "learning_rate": 1.667156910038524e-08,
+      "logits/chosen": -2.95780611038208,
+      "logits/rejected": -2.9364283084869385,
+      "logps/chosen": -60.31455612182617,
+      "logps/rejected": -59.803741455078125,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.05532900243997574,
+      "rewards/margins": 0.02383572980761528,
+      "rewards/rejected": -0.07916472852230072,
+      "step": 4510
+    },
+    {
+      "epoch": 1.5575465196416265,
+      "grad_norm": 1.873235821723938,
+      "learning_rate": 1.6611785084519728e-08,
+      "logits/chosen": -2.9491257667541504,
+      "logits/rejected": -2.9411144256591797,
+      "logps/chosen": -56.79343795776367,
+      "logps/rejected": -60.494842529296875,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.05632984638214111,
+      "rewards/margins": 0.01953933760523796,
+      "rewards/rejected": -0.07586918771266937,
+      "step": 4520
+    },
+    {
+      "epoch": 1.5609924190213644,
+      "grad_norm": 1.8130160570144653,
+      "learning_rate": 1.6551975155021783e-08,
+      "logits/chosen": -3.0076842308044434,
+      "logits/rejected": -2.9848225116729736,
+      "logps/chosen": -60.148963928222656,
+      "logps/rejected": -60.78029251098633,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.059136997908353806,
+      "rewards/margins": 0.020439064130187035,
+      "rewards/rejected": -0.07957606017589569,
+      "step": 4530
+    },
+    {
+      "epoch": 1.5644383184011028,
+      "grad_norm": 1.7540615797042847,
+      "learning_rate": 1.649214027349139e-08,
+      "logits/chosen": -2.969295024871826,
+      "logits/rejected": -2.95894193649292,
+      "logps/chosen": -58.30316925048828,
+      "logps/rejected": -62.52735137939453,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.050300441682338715,
+      "rewards/margins": 0.02385946735739708,
+      "rewards/rejected": -0.0741599053144455,
+      "step": 4540
+    },
+    {
+      "epoch": 1.5678842177808407,
+      "grad_norm": 1.9283931255340576,
+      "learning_rate": 1.6432281401929718e-08,
+      "logits/chosen": -2.975557565689087,
+      "logits/rejected": -2.9440577030181885,
+      "logps/chosen": -61.248138427734375,
+      "logps/rejected": -59.751243591308594,
+      "loss": 0.681,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.052152253687381744,
+      "rewards/margins": 0.025732427835464478,
+      "rewards/rejected": -0.07788468152284622,
+      "step": 4550
+    },
+    {
+      "epoch": 1.571330117160579,
+      "grad_norm": 2.0781452655792236,
+      "learning_rate": 1.637239950272363e-08,
+      "logits/chosen": -3.023554563522339,
+      "logits/rejected": -2.991983652114868,
+      "logps/chosen": -62.49182891845703,
+      "logps/rejected": -61.560020446777344,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.04729786887764931,
+      "rewards/margins": 0.029825445264577866,
+      "rewards/rejected": -0.07712330669164658,
+      "step": 4560
+    },
+    {
+      "epoch": 1.574776016540317,
+      "grad_norm": 1.998867392539978,
+      "learning_rate": 1.6312495538630223e-08,
+      "logits/chosen": -2.985407590866089,
+      "logits/rejected": -2.9472715854644775,
+      "logps/chosen": -60.774330139160156,
+      "logps/rejected": -59.276084899902344,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.04624097794294357,
+      "rewards/margins": 0.03014841303229332,
+      "rewards/rejected": -0.0763893872499466,
+      "step": 4570
+    },
+    {
+      "epoch": 1.578221915920055,
+      "grad_norm": 1.69512939453125,
+      "learning_rate": 1.6252570472761343e-08,
+      "logits/chosen": -3.0142674446105957,
+      "logits/rejected": -2.9982590675354004,
+      "logps/chosen": -59.386146545410156,
+      "logps/rejected": -61.91411590576172,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.05648191645741463,
+      "rewards/margins": 0.024850815534591675,
+      "rewards/rejected": -0.081332728266716,
+      "step": 4580
+    },
+    {
+      "epoch": 1.5816678152997934,
+      "grad_norm": 1.947900414466858,
+      "learning_rate": 1.6192625268568106e-08,
+      "logits/chosen": -3.0112600326538086,
+      "logits/rejected": -2.9842231273651123,
+      "logps/chosen": -60.81835174560547,
+      "logps/rejected": -59.41999435424805,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.06032143160700798,
+      "rewards/margins": 0.02569553256034851,
+      "rewards/rejected": -0.08601696789264679,
+      "step": 4590
+    },
+    {
+      "epoch": 1.5851137146795313,
+      "grad_norm": 1.9539490938186646,
+      "learning_rate": 1.6132660889825386e-08,
+      "logits/chosen": -3.0697503089904785,
+      "logits/rejected": -3.055741548538208,
+      "logps/chosen": -59.510353088378906,
+      "logps/rejected": -62.381141662597656,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.060057632625103,
+      "rewards/margins": 0.021062199026346207,
+      "rewards/rejected": -0.0811198353767395,
+      "step": 4600
+    },
+    {
+      "epoch": 1.5851137146795313,
+      "eval_logits/chosen": -3.0927746295928955,
+      "eval_logits/rejected": -3.087068796157837,
+      "eval_logps/chosen": -61.4617805480957,
+      "eval_logps/rejected": -66.92826080322266,
+      "eval_loss": 0.6885798573493958,
+      "eval_rewards/accuracies": 0.5934014916419983,
+      "eval_rewards/chosen": -0.02749883010983467,
+      "eval_rewards/margins": 0.009982595220208168,
+      "eval_rewards/rejected": -0.03748142719268799,
+      "eval_runtime": 382.9817,
+      "eval_samples_per_second": 11.238,
+      "eval_steps_per_second": 1.405,
+      "step": 4600
+    },
+    {
+      "epoch": 1.5885596140592695,
+      "grad_norm": 1.8216711282730103,
+      "learning_rate": 1.607267830061636e-08,
+      "logits/chosen": -2.9443492889404297,
+      "logits/rejected": -2.9191277027130127,
+      "logps/chosen": -58.79723358154297,
+      "logps/rejected": -58.813446044921875,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.05867000296711922,
+      "rewards/margins": 0.022766629233956337,
+      "rewards/rejected": -0.0814366340637207,
+      "step": 4610
+    },
+    {
+      "epoch": 1.5920055134390076,
+      "grad_norm": 1.9365017414093018,
+      "learning_rate": 1.6012678465316983e-08,
+      "logits/chosen": -2.927950620651245,
+      "logits/rejected": -2.9184749126434326,
+      "logps/chosen": -58.3230094909668,
+      "logps/rejected": -61.50208282470703,
+      "loss": 0.683,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.05594564601778984,
+      "rewards/margins": 0.021833518519997597,
+      "rewards/rejected": -0.07777915894985199,
+      "step": 4620
+    },
+    {
+      "epoch": 1.5954514128187456,
+      "grad_norm": 1.8679474592208862,
+      "learning_rate": 1.5952662348580463e-08,
+      "logits/chosen": -2.9530258178710938,
+      "logits/rejected": -2.930922746658325,
+      "logps/chosen": -57.321075439453125,
+      "logps/rejected": -59.60029220581055,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.05896700173616409,
+      "rewards/margins": 0.01904251240193844,
+      "rewards/rejected": -0.07800950855016708,
+      "step": 4630
+    },
+    {
+      "epoch": 1.598897312198484,
+      "grad_norm": 1.9925289154052734,
+      "learning_rate": 1.58926309153218e-08,
+      "logits/chosen": -3.039781332015991,
+      "logits/rejected": -3.0148491859436035,
+      "logps/chosen": -59.83821487426758,
+      "logps/rejected": -60.969947814941406,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.05606449395418167,
+      "rewards/margins": 0.025595705956220627,
+      "rewards/rejected": -0.081660196185112,
+      "step": 4640
+    },
+    {
+      "epoch": 1.602343211578222,
+      "grad_norm": 1.9926656484603882,
+      "learning_rate": 1.5832585130702236e-08,
+      "logits/chosen": -2.9337363243103027,
+      "logits/rejected": -2.9200844764709473,
+      "logps/chosen": -60.22651290893555,
+      "logps/rejected": -61.3533821105957,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.060573648661375046,
+      "rewards/margins": 0.01929016411304474,
+      "rewards/rejected": -0.07986380904912949,
+      "step": 4650
+    },
+    {
+      "epoch": 1.60578911095796,
+      "grad_norm": 1.8722528219223022,
+      "learning_rate": 1.5772525960113754e-08,
+      "logits/chosen": -3.0293638706207275,
+      "logits/rejected": -3.0044589042663574,
+      "logps/chosen": -58.535552978515625,
+      "logps/rejected": -61.14292526245117,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.06246672943234444,
+      "rewards/margins": 0.025018077343702316,
+      "rewards/rejected": -0.08748479932546616,
+      "step": 4660
+    },
+    {
+      "epoch": 1.6092350103376982,
+      "grad_norm": 2.087712287902832,
+      "learning_rate": 1.5712454369163535e-08,
+      "logits/chosen": -2.9743173122406006,
+      "logits/rejected": -2.9548697471618652,
+      "logps/chosen": -60.4631233215332,
+      "logps/rejected": -62.20234298706055,
+      "loss": 0.681,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.06269504874944687,
+      "rewards/margins": 0.026084134355187416,
+      "rewards/rejected": -0.08877918869256973,
+      "step": 4670
+    },
+    {
+      "epoch": 1.6126809097174362,
+      "grad_norm": 1.7644888162612915,
+      "learning_rate": 1.5652371323658475e-08,
+      "logits/chosen": -3.030400037765503,
+      "logits/rejected": -2.9951109886169434,
+      "logps/chosen": -60.10515213012695,
+      "logps/rejected": -60.76068115234375,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.054889362305402756,
+      "rewards/margins": 0.030241191387176514,
+      "rewards/rejected": -0.08513054996728897,
+      "step": 4680
+    },
+    {
+      "epoch": 1.6161268090971743,
+      "grad_norm": 1.9471898078918457,
+      "learning_rate": 1.5592277789589612e-08,
+      "logits/chosen": -3.0800793170928955,
+      "logits/rejected": -3.054615020751953,
+      "logps/chosen": -59.69933319091797,
+      "logps/rejected": -59.97870635986328,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.05575327202677727,
+      "rewards/margins": 0.018248992040753365,
+      "rewards/rejected": -0.07400226593017578,
+      "step": 4690
+    },
+    {
+      "epoch": 1.6195727084769125,
+      "grad_norm": 1.9379143714904785,
+      "learning_rate": 1.553217473311662e-08,
+      "logits/chosen": -2.8967673778533936,
+      "logits/rejected": -2.8806233406066895,
+      "logps/chosen": -59.744544982910156,
+      "logps/rejected": -62.49932861328125,
+      "loss": 0.685,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.06160573288798332,
+      "rewards/margins": 0.017968812957406044,
+      "rewards/rejected": -0.07957455515861511,
+      "step": 4700
+    },
+    {
+      "epoch": 1.6195727084769125,
+      "eval_logits/chosen": -3.090996503829956,
+      "eval_logits/rejected": -3.0852949619293213,
+      "eval_logps/chosen": -61.54983901977539,
+      "eval_logps/rejected": -67.04693603515625,
+      "eval_loss": 0.6884395480155945,
+      "eval_rewards/accuracies": 0.5929368138313293,
+      "eval_rewards/chosen": -0.028379401192069054,
+      "eval_rewards/margins": 0.01028869766741991,
+      "eval_rewards/rejected": -0.03866810351610184,
+      "eval_runtime": 383.5174,
+      "eval_samples_per_second": 11.222,
+      "eval_steps_per_second": 1.403,
+      "step": 4700
+    },
+    {
+      "epoch": 1.6230186078566504,
+      "grad_norm": 2.0050272941589355,
+      "learning_rate": 1.5472063120552275e-08,
+      "logits/chosen": -2.988820791244507,
+      "logits/rejected": -2.9707303047180176,
+      "logps/chosen": -60.55278396606445,
+      "logps/rejected": -60.7043571472168,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.060964517295360565,
+      "rewards/margins": 0.024478048086166382,
+      "rewards/rejected": -0.08544255793094635,
+      "step": 4710
+    },
+    {
+      "epoch": 1.6264645072363888,
+      "grad_norm": 1.8658896684646606,
+      "learning_rate": 1.5411943918346912e-08,
+      "logits/chosen": -2.9718434810638428,
+      "logits/rejected": -2.963080883026123,
+      "logps/chosen": -57.709442138671875,
+      "logps/rejected": -58.92949676513672,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.06676028668880463,
+      "rewards/margins": 0.015560664236545563,
+      "rewards/rejected": -0.08232095092535019,
+      "step": 4720
+    },
+    {
+      "epoch": 1.6299104066161267,
+      "grad_norm": 2.2711730003356934,
+      "learning_rate": 1.5351818093072883e-08,
+      "logits/chosen": -3.031459093093872,
+      "logits/rejected": -3.0251376628875732,
+      "logps/chosen": -59.604522705078125,
+      "logps/rejected": -61.35259246826172,
+      "loss": 0.6829,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.06784598529338837,
+      "rewards/margins": 0.022361842915415764,
+      "rewards/rejected": -0.09020782262086868,
+      "step": 4730
+    },
+    {
+      "epoch": 1.633356305995865,
+      "grad_norm": 1.8766865730285645,
+      "learning_rate": 1.529168661140903e-08,
+      "logits/chosen": -2.8897430896759033,
+      "logits/rejected": -2.8572967052459717,
+      "logps/chosen": -59.88768768310547,
+      "logps/rejected": -60.499420166015625,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.06568869948387146,
+      "rewards/margins": 0.023680545389652252,
+      "rewards/rejected": -0.08936922997236252,
+      "step": 4740
+    },
+    {
+      "epoch": 1.636802205375603,
+      "grad_norm": 1.7931801080703735,
+      "learning_rate": 1.5231550440125135e-08,
+      "logits/chosen": -2.9738614559173584,
+      "logits/rejected": -2.9575600624084473,
+      "logps/chosen": -59.764869689941406,
+      "logps/rejected": -61.836891174316406,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.05885421484708786,
+      "rewards/margins": 0.02276778407394886,
+      "rewards/rejected": -0.08162199705839157,
+      "step": 4750
+    },
+    {
+      "epoch": 1.640248104755341,
+      "grad_norm": 1.9326045513153076,
+      "learning_rate": 1.5171410546066374e-08,
+      "logits/chosen": -3.010842800140381,
+      "logits/rejected": -2.9955050945281982,
+      "logps/chosen": -57.499114990234375,
+      "logps/rejected": -64.4550552368164,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.05888698250055313,
+      "rewards/margins": 0.026701753959059715,
+      "rewards/rejected": -0.085588738322258,
+      "step": 4760
+    },
+    {
+      "epoch": 1.6436940041350794,
+      "grad_norm": 1.9466017484664917,
+      "learning_rate": 1.511126789613779e-08,
+      "logits/chosen": -2.9897661209106445,
+      "logits/rejected": -2.966625690460205,
+      "logps/chosen": -60.03380584716797,
+      "logps/rejected": -61.19886016845703,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.06526532769203186,
+      "rewards/margins": 0.0252664927393198,
+      "rewards/rejected": -0.09053182601928711,
+      "step": 4770
+    },
+    {
+      "epoch": 1.6471399035148173,
+      "grad_norm": 2.372635841369629,
+      "learning_rate": 1.5051123457288716e-08,
+      "logits/chosen": -3.033024311065674,
+      "logits/rejected": -3.031266689300537,
+      "logps/chosen": -59.61095428466797,
+      "logps/rejected": -63.045997619628906,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.061878979206085205,
+      "rewards/margins": 0.019319187849760056,
+      "rewards/rejected": -0.08119817078113556,
+      "step": 4780
+    },
+    {
+      "epoch": 1.6505858028945555,
+      "grad_norm": 1.8075237274169922,
+      "learning_rate": 1.499097819649725e-08,
+      "logits/chosen": -3.011930227279663,
+      "logits/rejected": -2.995335102081299,
+      "logps/chosen": -58.490821838378906,
+      "logps/rejected": -62.393821716308594,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.06120876222848892,
+      "rewards/margins": 0.024704840034246445,
+      "rewards/rejected": -0.08591359853744507,
+      "step": 4790
+    },
+    {
+      "epoch": 1.6540317022742936,
+      "grad_norm": 1.9024022817611694,
+      "learning_rate": 1.4930833080754718e-08,
+      "logits/chosen": -3.041536808013916,
+      "logits/rejected": -3.0151960849761963,
+      "logps/chosen": -61.49315643310547,
+      "logps/rejected": -62.50492477416992,
+      "loss": 0.683,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.05914570018649101,
+      "rewards/margins": 0.022537436336278915,
+      "rewards/rejected": -0.08168313652276993,
+      "step": 4800
+    },
+    {
+      "epoch": 1.6540317022742936,
+      "eval_logits/chosen": -3.0888524055480957,
+      "eval_logits/rejected": -3.0831334590911865,
+      "eval_logps/chosen": -61.64912796020508,
+      "eval_logps/rejected": -67.1814956665039,
+      "eval_loss": 0.6882779598236084,
+      "eval_rewards/accuracies": 0.5959572196006775,
+      "eval_rewards/chosen": -0.029372291639447212,
+      "eval_rewards/margins": 0.010641406290233135,
+      "eval_rewards/rejected": -0.04001370072364807,
+      "eval_runtime": 384.0901,
+      "eval_samples_per_second": 11.206,
+      "eval_steps_per_second": 1.401,
+      "step": 4800
+    },
+    {
+      "epoch": 1.6574776016540316,
+      "grad_norm": 1.9244457483291626,
+      "learning_rate": 1.4870689077050094e-08,
+      "logits/chosen": -3.0376498699188232,
+      "logits/rejected": -3.0235180854797363,
+      "logps/chosen": -60.91473388671875,
+      "logps/rejected": -62.60243606567383,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.0546736903488636,
+      "rewards/margins": 0.019553523510694504,
+      "rewards/rejected": -0.07422720640897751,
+      "step": 4810
+    },
+    {
+      "epoch": 1.66092350103377,
+      "grad_norm": 1.8402811288833618,
+      "learning_rate": 1.4810547152354485e-08,
+      "logits/chosen": -3.040675640106201,
+      "logits/rejected": -3.0113654136657715,
+      "logps/chosen": -62.241966247558594,
+      "logps/rejected": -60.458335876464844,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.06514403969049454,
+      "rewards/margins": 0.02161555550992489,
+      "rewards/rejected": -0.08675960451364517,
+      "step": 4820
+    },
+    {
+      "epoch": 1.664369400413508,
+      "grad_norm": 2.297260046005249,
+      "learning_rate": 1.4750408273605586e-08,
+      "logits/chosen": -2.9465794563293457,
+      "logits/rejected": -2.9333178997039795,
+      "logps/chosen": -59.24495315551758,
+      "logps/rejected": -61.68938446044922,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.062137871980667114,
+      "rewards/margins": 0.016546351835131645,
+      "rewards/rejected": -0.07868422567844391,
+      "step": 4830
+    },
+    {
+      "epoch": 1.667815299793246,
+      "grad_norm": 1.9780808687210083,
+      "learning_rate": 1.4690273407692089e-08,
+      "logits/chosen": -3.0298264026641846,
+      "logits/rejected": -3.0070600509643555,
+      "logps/chosen": -60.99510955810547,
+      "logps/rejected": -61.032188415527344,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.05702345445752144,
+      "rewards/margins": 0.02560465969145298,
+      "rewards/rejected": -0.08262811601161957,
+      "step": 4840
+    },
+    {
+      "epoch": 1.6712611991729842,
+      "grad_norm": 1.9595657587051392,
+      "learning_rate": 1.4630143521438199e-08,
+      "logits/chosen": -2.9330649375915527,
+      "logits/rejected": -2.907362222671509,
+      "logps/chosen": -60.4764289855957,
+      "logps/rejected": -61.985069274902344,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06209101527929306,
+      "rewards/margins": 0.023335615172982216,
+      "rewards/rejected": -0.08542662858963013,
+      "step": 4850
+    },
+    {
+      "epoch": 1.6747070985527222,
+      "grad_norm": 2.047550916671753,
+      "learning_rate": 1.457001958158803e-08,
+      "logits/chosen": -2.9194650650024414,
+      "logits/rejected": -2.905083417892456,
+      "logps/chosen": -59.5345344543457,
+      "logps/rejected": -59.8623161315918,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.06040980666875839,
+      "rewards/margins": 0.021863680332899094,
+      "rewards/rejected": -0.08227349817752838,
+      "step": 4860
+    },
+    {
+      "epoch": 1.6781529979324605,
+      "grad_norm": 1.7917754650115967,
+      "learning_rate": 1.4509902554790129e-08,
+      "logits/chosen": -2.9925308227539062,
+      "logits/rejected": -2.9763710498809814,
+      "logps/chosen": -60.0545654296875,
+      "logps/rejected": -61.87347412109375,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.05949188396334648,
+      "rewards/margins": 0.020726090297102928,
+      "rewards/rejected": -0.08021797984838486,
+      "step": 4870
+    },
+    {
+      "epoch": 1.6815988973121985,
+      "grad_norm": 1.810477614402771,
+      "learning_rate": 1.4449793407581859e-08,
+      "logits/chosen": -3.0156712532043457,
+      "logits/rejected": -2.981854200363159,
+      "logps/chosen": -59.9681282043457,
+      "logps/rejected": -59.21062088012695,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.058125089854002,
+      "rewards/margins": 0.030805563554167747,
+      "rewards/rejected": -0.0889306515455246,
+      "step": 4880
+    },
+    {
+      "epoch": 1.6850447966919366,
+      "grad_norm": 1.7966196537017822,
+      "learning_rate": 1.4389693106373934e-08,
+      "logits/chosen": -2.9723105430603027,
+      "logits/rejected": -2.93871808052063,
+      "logps/chosen": -63.222755432128906,
+      "logps/rejected": -61.55763626098633,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.05727462098002434,
+      "rewards/margins": 0.030505161732435226,
+      "rewards/rejected": -0.08777978271245956,
+      "step": 4890
+    },
+    {
+      "epoch": 1.6884906960716748,
+      "grad_norm": 2.0210258960723877,
+      "learning_rate": 1.4329602617434817e-08,
+      "logits/chosen": -2.996326208114624,
+      "logits/rejected": -2.978289842605591,
+      "logps/chosen": -60.379920959472656,
+      "logps/rejected": -64.11723327636719,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.05272985249757767,
+      "rewards/margins": 0.03202337026596069,
+      "rewards/rejected": -0.08475321531295776,
+      "step": 4900
+    },
+    {
+      "epoch": 1.6884906960716748,
+      "eval_logits/chosen": -3.087730646133423,
+      "eval_logits/rejected": -3.0819954872131348,
+      "eval_logps/chosen": -61.78580093383789,
+      "eval_logps/rejected": -67.3424072265625,
+      "eval_loss": 0.6881715655326843,
+      "eval_rewards/accuracies": 0.5950278639793396,
+      "eval_rewards/chosen": -0.030739109963178635,
+      "eval_rewards/margins": 0.010883730836212635,
+      "eval_rewards/rejected": -0.041622843593358994,
+      "eval_runtime": 383.1721,
+      "eval_samples_per_second": 11.233,
+      "eval_steps_per_second": 1.404,
+      "step": 4900
+    },
+    {
+      "epoch": 1.6919365954514127,
+      "grad_norm": 1.9389182329177856,
+      "learning_rate": 1.4269522906875224e-08,
+      "logits/chosen": -2.974872350692749,
+      "logits/rejected": -2.944887638092041,
+      "logps/chosen": -60.47908401489258,
+      "logps/rejected": -63.069252014160156,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.684374988079071,
+      "rewards/chosen": -0.05708624795079231,
+      "rewards/margins": 0.03433075547218323,
+      "rewards/rejected": -0.09141700714826584,
+      "step": 4910
+    },
+    {
+      "epoch": 1.6953824948311509,
+      "grad_norm": 1.9086792469024658,
+      "learning_rate": 1.4209454940632595e-08,
+      "logits/chosen": -3.0044236183166504,
+      "logits/rejected": -2.986051559448242,
+      "logps/chosen": -63.340232849121094,
+      "logps/rejected": -63.97481155395508,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.06407347321510315,
+      "rewards/margins": 0.018760284408926964,
+      "rewards/rejected": -0.08283375203609467,
+      "step": 4920
+    },
+    {
+      "epoch": 1.698828394210889,
+      "grad_norm": 2.0031261444091797,
+      "learning_rate": 1.414939968445552e-08,
+      "logits/chosen": -3.021125316619873,
+      "logits/rejected": -3.0100526809692383,
+      "logps/chosen": -61.77006149291992,
+      "logps/rejected": -60.79412078857422,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.559374988079071,
+      "rewards/chosen": -0.06306079030036926,
+      "rewards/margins": 0.017136866226792336,
+      "rewards/rejected": -0.08019766211509705,
+      "step": 4930
+    },
+    {
+      "epoch": 1.7022742935906272,
+      "grad_norm": 1.8223506212234497,
+      "learning_rate": 1.408935810388827e-08,
+      "logits/chosen": -2.9725186824798584,
+      "logits/rejected": -2.956904172897339,
+      "logps/chosen": -58.83582305908203,
+      "logps/rejected": -61.9322509765625,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.06437459588050842,
+      "rewards/margins": 0.02326207049190998,
+      "rewards/rejected": -0.08763666450977325,
+      "step": 4940
+    },
+    {
+      "epoch": 1.7057201929703654,
+      "grad_norm": 1.8439580202102661,
+      "learning_rate": 1.4029331164255222e-08,
+      "logits/chosen": -2.9332385063171387,
+      "logits/rejected": -2.9153711795806885,
+      "logps/chosen": -60.049598693847656,
+      "logps/rejected": -60.849517822265625,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.05987505242228508,
+      "rewards/margins": 0.030247356742620468,
+      "rewards/rejected": -0.09012240171432495,
+      "step": 4950
+    },
+    {
+      "epoch": 1.7091660923501033,
+      "grad_norm": 1.8660255670547485,
+      "learning_rate": 1.3969319830645384e-08,
+      "logits/chosen": -3.05200457572937,
+      "logits/rejected": -3.015324115753174,
+      "logps/chosen": -61.152549743652344,
+      "logps/rejected": -59.09883499145508,
+      "loss": 0.6787,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.06023973226547241,
+      "rewards/margins": 0.03084680438041687,
+      "rewards/rejected": -0.09108655154705048,
+      "step": 4960
+    },
+    {
+      "epoch": 1.7126119917298415,
+      "grad_norm": 1.8360295295715332,
+      "learning_rate": 1.3909325067896836e-08,
+      "logits/chosen": -3.030017375946045,
+      "logits/rejected": -3.0266127586364746,
+      "logps/chosen": -59.41315841674805,
+      "logps/rejected": -65.45125579833984,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.059289686381816864,
+      "rewards/margins": 0.01721843145787716,
+      "rewards/rejected": -0.07650811970233917,
+      "step": 4970
+    },
+    {
+      "epoch": 1.7160578911095796,
+      "grad_norm": 1.9150689840316772,
+      "learning_rate": 1.3849347840581255e-08,
+      "logits/chosen": -3.043647050857544,
+      "logits/rejected": -3.016939640045166,
+      "logps/chosen": -60.29804229736328,
+      "logps/rejected": -61.5930290222168,
+      "loss": 0.684,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.06597334146499634,
+      "rewards/margins": 0.020263809710741043,
+      "rewards/rejected": -0.08623714745044708,
+      "step": 4980
+    },
+    {
+      "epoch": 1.7195037904893176,
+      "grad_norm": 2.0168616771698,
+      "learning_rate": 1.378938911298837e-08,
+      "logits/chosen": -2.973952531814575,
+      "logits/rejected": -2.9574081897735596,
+      "logps/chosen": -58.52497482299805,
+      "logps/rejected": -61.601646423339844,
+      "loss": 0.6806,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0644695833325386,
+      "rewards/margins": 0.02682039514183998,
+      "rewards/rejected": -0.09128998219966888,
+      "step": 4990
+    },
+    {
+      "epoch": 1.722949689869056,
+      "grad_norm": 1.9032574892044067,
+      "learning_rate": 1.3729449849110505e-08,
+      "logits/chosen": -3.042367458343506,
+      "logits/rejected": -3.0072712898254395,
+      "logps/chosen": -60.70680618286133,
+      "logps/rejected": -61.95502471923828,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.05966181680560112,
+      "rewards/margins": 0.02546881139278412,
+      "rewards/rejected": -0.08513061702251434,
+      "step": 5000
+    },
+    {
+      "epoch": 1.722949689869056,
+      "eval_logits/chosen": -3.086275577545166,
+      "eval_logits/rejected": -3.080538749694824,
+      "eval_logps/chosen": -61.878543853759766,
+      "eval_logps/rejected": -67.44477844238281,
+      "eval_loss": 0.6881341934204102,
+      "eval_rewards/accuracies": 0.5943308472633362,
+      "eval_rewards/chosen": -0.03166646510362625,
+      "eval_rewards/margins": 0.010980132967233658,
+      "eval_rewards/rejected": -0.04264660179615021,
+      "eval_runtime": 383.5484,
+      "eval_samples_per_second": 11.222,
+      "eval_steps_per_second": 1.403,
+      "step": 5000
+    },
+    {
+      "epoch": 1.7263955892487939,
+      "grad_norm": 1.8323076963424683,
+      "learning_rate": 1.366953101262704e-08,
+      "logits/chosen": -2.990379810333252,
+      "logits/rejected": -2.963744640350342,
+      "logps/chosen": -59.51305389404297,
+      "logps/rejected": -61.07666015625,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.06487073004245758,
+      "rewards/margins": 0.027111446484923363,
+      "rewards/rejected": -0.09198217839002609,
+      "step": 5010
+    },
+    {
+      "epoch": 1.729841488628532,
+      "grad_norm": 2.003514289855957,
+      "learning_rate": 1.3609633566888928e-08,
+      "logits/chosen": -3.0712695121765137,
+      "logits/rejected": -3.04768705368042,
+      "logps/chosen": -61.8719482421875,
+      "logps/rejected": -61.40668487548828,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.05755845457315445,
+      "rewards/margins": 0.02367434836924076,
+      "rewards/rejected": -0.08123280107975006,
+      "step": 5020
+    },
+    {
+      "epoch": 1.7332873880082702,
+      "grad_norm": 1.7092092037200928,
+      "learning_rate": 1.354975847490322e-08,
+      "logits/chosen": -2.9557833671569824,
+      "logits/rejected": -2.9511876106262207,
+      "logps/chosen": -58.5129508972168,
+      "logps/rejected": -62.244842529296875,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.07158501446247101,
+      "rewards/margins": 0.007750541903078556,
+      "rewards/rejected": -0.07933555543422699,
+      "step": 5030
+    },
+    {
+      "epoch": 1.7367332873880081,
+      "grad_norm": 2.0734610557556152,
+      "learning_rate": 1.3489906699317557e-08,
+      "logits/chosen": -2.973404884338379,
+      "logits/rejected": -2.9606568813323975,
+      "logps/chosen": -60.963478088378906,
+      "logps/rejected": -62.31982421875,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06467059999704361,
+      "rewards/margins": 0.02069903537631035,
+      "rewards/rejected": -0.08536963164806366,
+      "step": 5040
+    },
+    {
+      "epoch": 1.7401791867677465,
+      "grad_norm": 2.2372896671295166,
+      "learning_rate": 1.3430079202404734e-08,
+      "logits/chosen": -3.042762517929077,
+      "logits/rejected": -3.0281307697296143,
+      "logps/chosen": -60.69879150390625,
+      "logps/rejected": -64.9217300415039,
+      "loss": 0.68,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.0601920560002327,
+      "rewards/margins": 0.027918020263314247,
+      "rewards/rejected": -0.0881100744009018,
+      "step": 5050
+    },
+    {
+      "epoch": 1.7436250861474845,
+      "grad_norm": 1.9283812046051025,
+      "learning_rate": 1.337027694604717e-08,
+      "logits/chosen": -3.037266731262207,
+      "logits/rejected": -3.0305259227752686,
+      "logps/chosen": -58.9273681640625,
+      "logps/rejected": -61.47454833984375,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06778950244188309,
+      "rewards/margins": 0.020694559440016747,
+      "rewards/rejected": -0.08848406374454498,
+      "step": 5060
+    },
+    {
+      "epoch": 1.7470709855272226,
+      "grad_norm": 2.172593593597412,
+      "learning_rate": 1.331050089172151e-08,
+      "logits/chosen": -3.0352001190185547,
+      "logits/rejected": -3.0036208629608154,
+      "logps/chosen": -62.12102508544922,
+      "logps/rejected": -62.155784606933594,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.052349530160427094,
+      "rewards/margins": 0.031124373897910118,
+      "rewards/rejected": -0.08347390592098236,
+      "step": 5070
+    },
+    {
+      "epoch": 1.7505168849069608,
+      "grad_norm": 2.024209976196289,
+      "learning_rate": 1.3250752000483106e-08,
+      "logits/chosen": -2.97279691696167,
+      "logits/rejected": -2.9558207988739014,
+      "logps/chosen": -60.439361572265625,
+      "logps/rejected": -62.91933059692383,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.06716683506965637,
+      "rewards/margins": 0.023983968421816826,
+      "rewards/rejected": -0.09115080535411835,
+      "step": 5080
+    },
+    {
+      "epoch": 1.7539627842866987,
+      "grad_norm": 1.9415674209594727,
+      "learning_rate": 1.3191031232950613e-08,
+      "logits/chosen": -2.958627223968506,
+      "logits/rejected": -2.9393832683563232,
+      "logps/chosen": -59.575523376464844,
+      "logps/rejected": -62.2266731262207,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.058996040374040604,
+      "rewards/margins": 0.024697905406355858,
+      "rewards/rejected": -0.08369394391775131,
+      "step": 5090
+    },
+    {
+      "epoch": 1.757408683666437,
+      "grad_norm": 1.792022943496704,
+      "learning_rate": 1.3131339549290514e-08,
+      "logits/chosen": -3.001885414123535,
+      "logits/rejected": -2.9799094200134277,
+      "logps/chosen": -58.2789421081543,
+      "logps/rejected": -60.12836837768555,
+      "loss": 0.6823,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.0658949762582779,
+      "rewards/margins": 0.023723982274532318,
+      "rewards/rejected": -0.08961894363164902,
+      "step": 5100
+    },
+    {
+      "epoch": 1.757408683666437,
+      "eval_logits/chosen": -3.084587574005127,
+      "eval_logits/rejected": -3.078856945037842,
+      "eval_logps/chosen": -61.99214553833008,
+      "eval_logps/rejected": -67.57994079589844,
+      "eval_loss": 0.6880365014076233,
+      "eval_rewards/accuracies": 0.5950278639793396,
+      "eval_rewards/chosen": -0.03280247002840042,
+      "eval_rewards/margins": 0.011195754632353783,
+      "eval_rewards/rejected": -0.043998222798109055,
+      "eval_runtime": 383.4867,
+      "eval_samples_per_second": 11.223,
+      "eval_steps_per_second": 1.403,
+      "step": 5100
+    },
+    {
+      "epoch": 1.760854583046175,
+      "grad_norm": 1.8438318967819214,
+      "learning_rate": 1.3071677909201689e-08,
+      "logits/chosen": -3.0111069679260254,
+      "logits/rejected": -3.006621837615967,
+      "logps/chosen": -58.1121711730957,
+      "logps/rejected": -63.20026397705078,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06912960112094879,
+      "rewards/margins": 0.021628180518746376,
+      "rewards/rejected": -0.09075777232646942,
+      "step": 5110
+    },
+    {
+      "epoch": 1.7643004824259132,
+      "grad_norm": 2.0576329231262207,
+      "learning_rate": 1.3012047271900004e-08,
+      "logits/chosen": -2.932772159576416,
+      "logits/rejected": -2.906367778778076,
+      "logps/chosen": -59.22174072265625,
+      "logps/rejected": -61.522216796875,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.06772752851247787,
+      "rewards/margins": 0.026370462030172348,
+      "rewards/rejected": -0.09409798681735992,
+      "step": 5120
+    },
+    {
+      "epoch": 1.7677463818056514,
+      "grad_norm": 1.940940260887146,
+      "learning_rate": 1.2952448596102856e-08,
+      "logits/chosen": -3.001607894897461,
+      "logits/rejected": -2.9829087257385254,
+      "logps/chosen": -61.59001922607422,
+      "logps/rejected": -62.393043518066406,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.06569873541593552,
+      "rewards/margins": 0.025472652167081833,
+      "rewards/rejected": -0.09117138385772705,
+      "step": 5130
+    },
+    {
+      "epoch": 1.7711922811853893,
+      "grad_norm": 2.0273191928863525,
+      "learning_rate": 1.28928828400138e-08,
+      "logits/chosen": -2.9652676582336426,
+      "logits/rejected": -2.950957775115967,
+      "logps/chosen": -61.56854248046875,
+      "logps/rejected": -63.5615119934082,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.06688603013753891,
+      "rewards/margins": 0.025311220437288284,
+      "rewards/rejected": -0.0921972468495369,
+      "step": 5140
+    },
+    {
+      "epoch": 1.7746381805651275,
+      "grad_norm": 2.1135692596435547,
+      "learning_rate": 1.28333509613071e-08,
+      "logits/chosen": -2.950176477432251,
+      "logits/rejected": -2.940347671508789,
+      "logps/chosen": -59.657798767089844,
+      "logps/rejected": -64.19325256347656,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06615753471851349,
+      "rewards/margins": 0.025921877473592758,
+      "rewards/rejected": -0.09207940846681595,
+      "step": 5150
+    },
+    {
+      "epoch": 1.7780840799448656,
+      "grad_norm": 1.9246599674224854,
+      "learning_rate": 1.2773853917112367e-08,
+      "logits/chosen": -2.9503133296966553,
+      "logits/rejected": -2.9211432933807373,
+      "logps/chosen": -62.7855224609375,
+      "logps/rejected": -61.96038818359375,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.06273352354764938,
+      "rewards/margins": 0.03199779987335205,
+      "rewards/rejected": -0.09473132342100143,
+      "step": 5160
+    },
+    {
+      "epoch": 1.7815299793246038,
+      "grad_norm": 2.174731731414795,
+      "learning_rate": 1.2714392663999146e-08,
+      "logits/chosen": -3.0069079399108887,
+      "logits/rejected": -2.987776756286621,
+      "logps/chosen": -60.609710693359375,
+      "logps/rejected": -63.66094970703125,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.06493788212537766,
+      "rewards/margins": 0.026249315589666367,
+      "rewards/rejected": -0.09118720144033432,
+      "step": 5170
+    },
+    {
+      "epoch": 1.784975878704342,
+      "grad_norm": 1.8664010763168335,
+      "learning_rate": 1.2654968157961556e-08,
+      "logits/chosen": -3.0261340141296387,
+      "logits/rejected": -2.995468854904175,
+      "logps/chosen": -60.84348678588867,
+      "logps/rejected": -60.67839431762695,
+      "loss": 0.6795,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.05829096958041191,
+      "rewards/margins": 0.029192816466093063,
+      "rewards/rejected": -0.08748378604650497,
+      "step": 5180
+    },
+    {
+      "epoch": 1.7884217780840799,
+      "grad_norm": 1.9086253643035889,
+      "learning_rate": 1.2595581354402916e-08,
+      "logits/chosen": -2.8968303203582764,
+      "logits/rejected": -2.8751895427703857,
+      "logps/chosen": -58.78535842895508,
+      "logps/rejected": -61.97437286376953,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.06857200711965561,
+      "rewards/margins": 0.02524719201028347,
+      "rewards/rejected": -0.09381920099258423,
+      "step": 5190
+    },
+    {
+      "epoch": 1.791867677463818,
+      "grad_norm": 1.982395887374878,
+      "learning_rate": 1.2536233208120363e-08,
+      "logits/chosen": -2.992213726043701,
+      "logits/rejected": -2.9681813716888428,
+      "logps/chosen": -60.58392333984375,
+      "logps/rejected": -64.16802978515625,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.06609203666448593,
+      "rewards/margins": 0.028584647923707962,
+      "rewards/rejected": -0.09467668831348419,
+      "step": 5200
+    },
+    {
+      "epoch": 1.791867677463818,
+      "eval_logits/chosen": -3.082876205444336,
+      "eval_logits/rejected": -3.077177047729492,
+      "eval_logps/chosen": -62.120540618896484,
+      "eval_logps/rejected": -67.7482681274414,
+      "eval_loss": 0.6878546476364136,
+      "eval_rewards/accuracies": 0.5987453460693359,
+      "eval_rewards/chosen": -0.0340864397585392,
+      "eval_rewards/margins": 0.011594949290156364,
+      "eval_rewards/rejected": -0.045681390911340714,
+      "eval_runtime": 383.3379,
+      "eval_samples_per_second": 11.228,
+      "eval_steps_per_second": 1.403,
+      "step": 5200
+    },
+    {
+      "epoch": 1.7953135768435562,
+      "grad_norm": 1.833192229270935,
+      "learning_rate": 1.2476924673289531e-08,
+      "logits/chosen": -2.9862053394317627,
+      "logits/rejected": -2.952648162841797,
+      "logps/chosen": -61.57801055908203,
+      "logps/rejected": -62.84089279174805,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.06258958578109741,
+      "rewards/margins": 0.029811212792992592,
+      "rewards/rejected": -0.09240079671144485,
+      "step": 5210
+    },
+    {
+      "epoch": 1.7987594762232941,
+      "grad_norm": 2.0118930339813232,
+      "learning_rate": 1.2417656703449191e-08,
+      "logits/chosen": -3.0556540489196777,
+      "logits/rejected": -3.0419697761535645,
+      "logps/chosen": -62.9283447265625,
+      "logps/rejected": -64.03778076171875,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.0655805692076683,
+      "rewards/margins": 0.023056553676724434,
+      "rewards/rejected": -0.08863712102174759,
+      "step": 5220
+    },
+    {
+      "epoch": 1.8022053756030325,
+      "grad_norm": 2.1077942848205566,
+      "learning_rate": 1.2358430251485935e-08,
+      "logits/chosen": -3.0104470252990723,
+      "logits/rejected": -2.9949755668640137,
+      "logps/chosen": -60.530006408691406,
+      "logps/rejected": -64.44468688964844,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.06431502103805542,
+      "rewards/margins": 0.02680242620408535,
+      "rewards/rejected": -0.09111744910478592,
+      "step": 5230
+    },
+    {
+      "epoch": 1.8056512749827704,
+      "grad_norm": 1.9842970371246338,
+      "learning_rate": 1.2299246269618828e-08,
+      "logits/chosen": -2.9317679405212402,
+      "logits/rejected": -2.901221752166748,
+      "logps/chosen": -60.41461181640625,
+      "logps/rejected": -60.68870162963867,
+      "loss": 0.681,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.06475957483053207,
+      "rewards/margins": 0.026215368881821632,
+      "rewards/rejected": -0.09097495675086975,
+      "step": 5240
+    },
+    {
+      "epoch": 1.8090971743625086,
+      "grad_norm": 1.9803575277328491,
+      "learning_rate": 1.2240105709384139e-08,
+      "logits/chosen": -2.9966797828674316,
+      "logits/rejected": -2.974655866622925,
+      "logps/chosen": -59.34581756591797,
+      "logps/rejected": -62.8516845703125,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.06090864539146423,
+      "rewards/margins": 0.029287615790963173,
+      "rewards/rejected": -0.09019625931978226,
+      "step": 5250
+    },
+    {
+      "epoch": 1.8125430737422468,
+      "grad_norm": 1.9502676725387573,
+      "learning_rate": 1.218100952162002e-08,
+      "logits/chosen": -3.012594223022461,
+      "logits/rejected": -3.0025241374969482,
+      "logps/chosen": -59.93048095703125,
+      "logps/rejected": -63.25605010986328,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.06893669068813324,
+      "rewards/margins": 0.02504999004304409,
+      "rewards/rejected": -0.09398667514324188,
+      "step": 5260
+    },
+    {
+      "epoch": 1.8159889731219847,
+      "grad_norm": 1.9223151206970215,
+      "learning_rate": 1.2121958656451198e-08,
+      "logits/chosen": -2.9450719356536865,
+      "logits/rejected": -2.920938014984131,
+      "logps/chosen": -59.67364501953125,
+      "logps/rejected": -60.0206298828125,
+      "loss": 0.682,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.06837842613458633,
+      "rewards/margins": 0.024228191003203392,
+      "rewards/rejected": -0.09260661900043488,
+      "step": 5270
+    },
+    {
+      "epoch": 1.819434872501723,
+      "grad_norm": 2.009779453277588,
+      "learning_rate": 1.2062954063273754e-08,
+      "logits/chosen": -3.0213570594787598,
+      "logits/rejected": -3.011331796646118,
+      "logps/chosen": -61.696380615234375,
+      "logps/rejected": -64.6453628540039,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.07004810124635696,
+      "rewards/margins": 0.024392934516072273,
+      "rewards/rejected": -0.09444103389978409,
+      "step": 5280
+    },
+    {
+      "epoch": 1.822880771881461,
+      "grad_norm": 1.8378812074661255,
+      "learning_rate": 1.200399669073979e-08,
+      "logits/chosen": -3.02955961227417,
+      "logits/rejected": -3.012268543243408,
+      "logps/chosen": -61.351226806640625,
+      "logps/rejected": -63.946495056152344,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.06696822494268417,
+      "rewards/margins": 0.0246070958673954,
+      "rewards/rejected": -0.09157532453536987,
+      "step": 5290
+    },
+    {
+      "epoch": 1.8263266712611992,
+      "grad_norm": 1.8869976997375488,
+      "learning_rate": 1.194508748674225e-08,
+      "logits/chosen": -3.0217196941375732,
+      "logits/rejected": -2.9956753253936768,
+      "logps/chosen": -61.11738967895508,
+      "logps/rejected": -61.92692947387695,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06723842769861221,
+      "rewards/margins": 0.028916900977492332,
+      "rewards/rejected": -0.0961553230881691,
+      "step": 5300
+    },
+    {
+      "epoch": 1.8263266712611992,
+      "eval_logits/chosen": -3.0814082622528076,
+      "eval_logits/rejected": -3.0756895542144775,
+      "eval_logps/chosen": -62.24217224121094,
+      "eval_logps/rejected": -67.8957748413086,
+      "eval_loss": 0.6877470016479492,
+      "eval_rewards/accuracies": 0.5952602028846741,
+      "eval_rewards/chosen": -0.03530285879969597,
+      "eval_rewards/margins": 0.011853594332933426,
+      "eval_rewards/rejected": -0.04715645685791969,
+      "eval_runtime": 383.4302,
+      "eval_samples_per_second": 11.225,
+      "eval_steps_per_second": 1.403,
+      "step": 5300
+    },
+    {
+      "epoch": 1.8297725706409373,
+      "grad_norm": 2.059708833694458,
+      "learning_rate": 1.1886227398399615e-08,
+      "logits/chosen": -2.942136287689209,
+      "logits/rejected": -2.925572395324707,
+      "logps/chosen": -59.2825813293457,
+      "logps/rejected": -61.301055908203125,
+      "loss": 0.6823,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.06848634034395218,
+      "rewards/margins": 0.02354987896978855,
+      "rewards/rejected": -0.09203621745109558,
+      "step": 5310
+    },
+    {
+      "epoch": 1.8332184700206753,
+      "grad_norm": 2.0242207050323486,
+      "learning_rate": 1.1827417372040728e-08,
+      "logits/chosen": -2.984877586364746,
+      "logits/rejected": -2.9639220237731934,
+      "logps/chosen": -61.44487380981445,
+      "logps/rejected": -62.75986862182617,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.0678262785077095,
+      "rewards/margins": 0.026033345609903336,
+      "rewards/rejected": -0.09385962039232254,
+      "step": 5320
+    },
+    {
+      "epoch": 1.8366643694004137,
+      "grad_norm": 1.8396081924438477,
+      "learning_rate": 1.176865835318953e-08,
+      "logits/chosen": -2.9993016719818115,
+      "logits/rejected": -2.981963634490967,
+      "logps/chosen": -61.11114501953125,
+      "logps/rejected": -64.19661712646484,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.06951332092285156,
+      "rewards/margins": 0.02746453322470188,
+      "rewards/rejected": -0.0969778522849083,
+      "step": 5330
+    },
+    {
+      "epoch": 1.8401102687801516,
+      "grad_norm": 1.927969217300415,
+      "learning_rate": 1.17099512865499e-08,
+      "logits/chosen": -2.989689588546753,
+      "logits/rejected": -2.957886219024658,
+      "logps/chosen": -61.49607467651367,
+      "logps/rejected": -61.511817932128906,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.0728243812918663,
+      "rewards/margins": 0.03056991472840309,
+      "rewards/rejected": -0.1033942922949791,
+      "step": 5340
+    },
+    {
+      "epoch": 1.8435561681598898,
+      "grad_norm": 1.790000081062317,
+      "learning_rate": 1.1651297115990458e-08,
+      "logits/chosen": -3.0189595222473145,
+      "logits/rejected": -2.9954025745391846,
+      "logps/chosen": -61.566429138183594,
+      "logps/rejected": -62.53955841064453,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.06607662886381149,
+      "rewards/margins": 0.0257400032132864,
+      "rewards/rejected": -0.09181663393974304,
+      "step": 5350
+    },
+    {
+      "epoch": 1.847002067539628,
+      "grad_norm": 1.908780813217163,
+      "learning_rate": 1.159269678452936e-08,
+      "logits/chosen": -2.97200345993042,
+      "logits/rejected": -2.9413533210754395,
+      "logps/chosen": -60.155479431152344,
+      "logps/rejected": -60.78470993041992,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.07139667868614197,
+      "rewards/margins": 0.027856219559907913,
+      "rewards/rejected": -0.09925289452075958,
+      "step": 5360
+    },
+    {
+      "epoch": 1.8504479669193659,
+      "grad_norm": 2.324455738067627,
+      "learning_rate": 1.153415123431918e-08,
+      "logits/chosen": -3.008014678955078,
+      "logits/rejected": -2.984591245651245,
+      "logps/chosen": -61.46697998046875,
+      "logps/rejected": -61.96287155151367,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.07142254710197449,
+      "rewards/margins": 0.023063164204359055,
+      "rewards/rejected": -0.09448571503162384,
+      "step": 5370
+    },
+    {
+      "epoch": 1.853893866299104,
+      "grad_norm": 1.9188162088394165,
+      "learning_rate": 1.147566140663172e-08,
+      "logits/chosen": -2.951540231704712,
+      "logits/rejected": -2.926616907119751,
+      "logps/chosen": -59.47803497314453,
+      "logps/rejected": -62.21937942504883,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.06608495116233826,
+      "rewards/margins": 0.02885495498776436,
+      "rewards/rejected": -0.09493990987539291,
+      "step": 5380
+    },
+    {
+      "epoch": 1.8573397656788422,
+      "grad_norm": 2.025850772857666,
+      "learning_rate": 1.141722824184292e-08,
+      "logits/chosen": -3.0059762001037598,
+      "logits/rejected": -2.9890596866607666,
+      "logps/chosen": -61.54661178588867,
+      "logps/rejected": -63.857269287109375,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.06425952911376953,
+      "rewards/margins": 0.025941062718629837,
+      "rewards/rejected": -0.09020058810710907,
+      "step": 5390
+    },
+    {
+      "epoch": 1.8607856650585803,
+      "grad_norm": 2.035921335220337,
+      "learning_rate": 1.1358852679417695e-08,
+      "logits/chosen": -3.0762641429901123,
+      "logits/rejected": -3.0497498512268066,
+      "logps/chosen": -58.71436309814453,
+      "logps/rejected": -63.388763427734375,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.06564133614301682,
+      "rewards/margins": 0.03161710128188133,
+      "rewards/rejected": -0.09725844115018845,
+      "step": 5400
+    },
+    {
+      "epoch": 1.8607856650585803,
+      "eval_logits/chosen": -3.0798468589782715,
+      "eval_logits/rejected": -3.0741541385650635,
+      "eval_logps/chosen": -62.38753128051758,
+      "eval_logps/rejected": -68.07237243652344,
+      "eval_loss": 0.6876041293144226,
+      "eval_rewards/accuracies": 0.5968866348266602,
+      "eval_rewards/chosen": -0.03675634041428566,
+      "eval_rewards/margins": 0.012166152708232403,
+      "eval_rewards/rejected": -0.048922501504421234,
+      "eval_runtime": 383.3445,
+      "eval_samples_per_second": 11.228,
+      "eval_steps_per_second": 1.403,
+      "step": 5400
+    },
+    {
+      "epoch": 1.8642315644383185,
+      "grad_norm": 1.8324499130249023,
+      "learning_rate": 1.1300535657894873e-08,
+      "logits/chosen": -2.974513530731201,
+      "logits/rejected": -2.9407286643981934,
+      "logps/chosen": -61.5993766784668,
+      "logps/rejected": -60.348609924316406,
+      "loss": 0.678,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.06361991167068481,
+      "rewards/margins": 0.03229387849569321,
+      "rewards/rejected": -0.09591379016637802,
+      "step": 5410
+    },
+    {
+      "epoch": 1.8676774638180564,
+      "grad_norm": 1.862282633781433,
+      "learning_rate": 1.1242278114872066e-08,
+      "logits/chosen": -2.989893674850464,
+      "logits/rejected": -2.9610233306884766,
+      "logps/chosen": -59.18886184692383,
+      "logps/rejected": -61.5391845703125,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.06785368174314499,
+      "rewards/margins": 0.02647547423839569,
+      "rewards/rejected": -0.09432916343212128,
+      "step": 5420
+    },
+    {
+      "epoch": 1.8711233631977946,
+      "grad_norm": 1.9856367111206055,
+      "learning_rate": 1.1184080986990623e-08,
+      "logits/chosen": -2.999823808670044,
+      "logits/rejected": -2.9753172397613525,
+      "logps/chosen": -59.26038360595703,
+      "logps/rejected": -62.894805908203125,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.06550584733486176,
+      "rewards/margins": 0.029098371043801308,
+      "rewards/rejected": -0.09460422396659851,
+      "step": 5430
+    },
+    {
+      "epoch": 1.8745692625775328,
+      "grad_norm": 1.9605991840362549,
+      "learning_rate": 1.1125945209920578e-08,
+      "logits/chosen": -2.937490463256836,
+      "logits/rejected": -2.9249072074890137,
+      "logps/chosen": -59.132774353027344,
+      "logps/rejected": -63.99884796142578,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06785356998443604,
+      "rewards/margins": 0.028354208916425705,
+      "rewards/rejected": -0.09620777517557144,
+      "step": 5440
+    },
+    {
+      "epoch": 1.8780151619572707,
+      "grad_norm": 1.8402197360992432,
+      "learning_rate": 1.1067871718345561e-08,
+      "logits/chosen": -3.0612518787384033,
+      "logits/rejected": -3.0359318256378174,
+      "logps/chosen": -60.74707794189453,
+      "logps/rejected": -62.024925231933594,
+      "loss": 0.6808,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.06670702248811722,
+      "rewards/margins": 0.02639928087592125,
+      "rewards/rejected": -0.09310630708932877,
+      "step": 5450
+    },
+    {
+      "epoch": 1.881461061337009,
+      "grad_norm": 1.926167368888855,
+      "learning_rate": 1.1009861445947828e-08,
+      "logits/chosen": -2.965944528579712,
+      "logits/rejected": -2.936561107635498,
+      "logps/chosen": -62.18370819091797,
+      "logps/rejected": -63.43079376220703,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.0643167495727539,
+      "rewards/margins": 0.034487538039684296,
+      "rewards/rejected": -0.0988042801618576,
+      "step": 5460
+    },
+    {
+      "epoch": 1.884906960716747,
+      "grad_norm": 2.1777851581573486,
+      "learning_rate": 1.0951915325393195e-08,
+      "logits/chosen": -2.9049878120422363,
+      "logits/rejected": -2.8795011043548584,
+      "logps/chosen": -62.2586784362793,
+      "logps/rejected": -62.83353805541992,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.06923080235719681,
+      "rewards/margins": 0.025808030739426613,
+      "rewards/rejected": -0.09503882378339767,
+      "step": 5470
+    },
+    {
+      "epoch": 1.8883528600964852,
+      "grad_norm": 2.065607786178589,
+      "learning_rate": 1.0894034288316101e-08,
+      "logits/chosen": -2.9698750972747803,
+      "logits/rejected": -2.9514172077178955,
+      "logps/chosen": -61.50615310668945,
+      "logps/rejected": -63.49835968017578,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.07297713309526443,
+      "rewards/margins": 0.025665078312158585,
+      "rewards/rejected": -0.09864220768213272,
+      "step": 5480
+    },
+    {
+      "epoch": 1.8917987594762233,
+      "grad_norm": 2.092161178588867,
+      "learning_rate": 1.0836219265304575e-08,
+      "logits/chosen": -3.0130615234375,
+      "logits/rejected": -2.9906563758850098,
+      "logps/chosen": -62.321929931640625,
+      "logps/rejected": -64.8061294555664,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.06603894382715225,
+      "rewards/margins": 0.030573895201086998,
+      "rewards/rejected": -0.096612848341465,
+      "step": 5490
+    },
+    {
+      "epoch": 1.8952446588559613,
+      "grad_norm": 2.1955559253692627,
+      "learning_rate": 1.0778471185885316e-08,
+      "logits/chosen": -3.0195870399475098,
+      "logits/rejected": -3.012105941772461,
+      "logps/chosen": -60.3916130065918,
+      "logps/rejected": -64.20580291748047,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.07703298330307007,
+      "rewards/margins": 0.01762479543685913,
+      "rewards/rejected": -0.0946577787399292,
+      "step": 5500
+    },
+    {
+      "epoch": 1.8952446588559613,
+      "eval_logits/chosen": -3.079150438308716,
+      "eval_logits/rejected": -3.0734682083129883,
+      "eval_logps/chosen": -62.481964111328125,
+      "eval_logps/rejected": -68.17646026611328,
+      "eval_loss": 0.6875699162483215,
+      "eval_rewards/accuracies": 0.5945631861686707,
+      "eval_rewards/chosen": -0.0377005897462368,
+      "eval_rewards/margins": 0.01226290874183178,
+      "eval_rewards/rejected": -0.04996349662542343,
+      "eval_runtime": 383.3172,
+      "eval_samples_per_second": 11.228,
+      "eval_steps_per_second": 1.404,
+      "step": 5500
+    },
+    {
+      "epoch": 1.8986905582356997,
+      "grad_norm": 2.01926326751709,
+      "learning_rate": 1.0720790978508716e-08,
+      "logits/chosen": -3.0456366539001465,
+      "logits/rejected": -3.0202975273132324,
+      "logps/chosen": -60.36711502075195,
+      "logps/rejected": -61.6754035949707,
+      "loss": 0.6783,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.06261946260929108,
+      "rewards/margins": 0.03158234432339668,
+      "rewards/rejected": -0.09420181810855865,
+      "step": 5510
+    },
+    {
+      "epoch": 1.9021364576154376,
+      "grad_norm": 1.9523613452911377,
+      "learning_rate": 1.0663179570533956e-08,
+      "logits/chosen": -3.0350778102874756,
+      "logits/rejected": -3.023207426071167,
+      "logps/chosen": -62.792091369628906,
+      "logps/rejected": -65.1396484375,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.07164590060710907,
+      "rewards/margins": 0.02959667146205902,
+      "rewards/rejected": -0.10124257951974869,
+      "step": 5520
+    },
+    {
+      "epoch": 1.9055823569951758,
+      "grad_norm": 2.0733156204223633,
+      "learning_rate": 1.060563788821409e-08,
+      "logits/chosen": -2.9948294162750244,
+      "logits/rejected": -2.9633102416992188,
+      "logps/chosen": -59.6869010925293,
+      "logps/rejected": -63.27789306640625,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.06567399203777313,
+      "rewards/margins": 0.032089781016111374,
+      "rewards/rejected": -0.0977637767791748,
+      "step": 5530
+    },
+    {
+      "epoch": 1.909028256374914,
+      "grad_norm": 2.0087571144104004,
+      "learning_rate": 1.0548166856681137e-08,
+      "logits/chosen": -3.034428596496582,
+      "logits/rejected": -2.9971375465393066,
+      "logps/chosen": -60.9912223815918,
+      "logps/rejected": -61.26765060424805,
+      "loss": 0.6763,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.06651772558689117,
+      "rewards/margins": 0.03586167097091675,
+      "rewards/rejected": -0.10237940400838852,
+      "step": 5540
+    },
+    {
+      "epoch": 1.9124741557546519,
+      "grad_norm": 1.9044840335845947,
+      "learning_rate": 1.0490767399931243e-08,
+      "logits/chosen": -2.934100389480591,
+      "logits/rejected": -2.9166135787963867,
+      "logps/chosen": -61.42578125,
+      "logps/rejected": -62.978599548339844,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.07074996829032898,
+      "rewards/margins": 0.025288432836532593,
+      "rewards/rejected": -0.09603839367628098,
+      "step": 5550
+    },
+    {
+      "epoch": 1.9159200551343902,
+      "grad_norm": 1.9555554389953613,
+      "learning_rate": 1.0433440440809782e-08,
+      "logits/chosen": -3.0657012462615967,
+      "logits/rejected": -3.0381171703338623,
+      "logps/chosen": -64.40907287597656,
+      "logps/rejected": -64.29598236083984,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07700801640748978,
+      "rewards/margins": 0.029971275478601456,
+      "rewards/rejected": -0.10697929561138153,
+      "step": 5560
+    },
+    {
+      "epoch": 1.9193659545141282,
+      "grad_norm": 1.9132229089736938,
+      "learning_rate": 1.0376186900996559e-08,
+      "logits/chosen": -2.9229512214660645,
+      "logits/rejected": -2.9015214443206787,
+      "logps/chosen": -62.2775993347168,
+      "logps/rejected": -62.2065315246582,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.07432884722948074,
+      "rewards/margins": 0.023921765387058258,
+      "rewards/rejected": -0.098250612616539,
+      "step": 5570
+    },
+    {
+      "epoch": 1.9228118538938663,
+      "grad_norm": 1.8418365716934204,
+      "learning_rate": 1.0319007700990953e-08,
+      "logits/chosen": -3.035928249359131,
+      "logits/rejected": -3.020050525665283,
+      "logps/chosen": -62.053375244140625,
+      "logps/rejected": -62.46696090698242,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.06947265565395355,
+      "rewards/margins": 0.01756802573800087,
+      "rewards/rejected": -0.08704067766666412,
+      "step": 5580
+    },
+    {
+      "epoch": 1.9262577532736045,
+      "grad_norm": 1.947690486907959,
+      "learning_rate": 1.026190376009716e-08,
+      "logits/chosen": -2.9716455936431885,
+      "logits/rejected": -2.9453558921813965,
+      "logps/chosen": -62.50593948364258,
+      "logps/rejected": -59.667640686035156,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.06941942870616913,
+      "rewards/margins": 0.028025507926940918,
+      "rewards/rejected": -0.09744493663311005,
+      "step": 5590
+    },
+    {
+      "epoch": 1.9297036526533424,
+      "grad_norm": 1.976467490196228,
+      "learning_rate": 1.0204875996409364e-08,
+      "logits/chosen": -3.0388638973236084,
+      "logits/rejected": -3.0130972862243652,
+      "logps/chosen": -60.6129264831543,
+      "logps/rejected": -62.66754150390625,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.06515444815158844,
+      "rewards/margins": 0.03459722548723221,
+      "rewards/rejected": -0.09975167363882065,
+      "step": 5600
+    },
+    {
+      "epoch": 1.9297036526533424,
+      "eval_logits/chosen": -3.0770630836486816,
+      "eval_logits/rejected": -3.0713460445404053,
+      "eval_logps/chosen": -62.627803802490234,
+      "eval_logps/rejected": -68.34706115722656,
+      "eval_loss": 0.6874624490737915,
+      "eval_rewards/accuracies": 0.5940985083580017,
+      "eval_rewards/chosen": -0.03915909305214882,
+      "eval_rewards/margins": 0.012510345317423344,
+      "eval_rewards/rejected": -0.051669441163539886,
+      "eval_runtime": 383.7396,
+      "eval_samples_per_second": 11.216,
+      "eval_steps_per_second": 1.402,
+      "step": 5600
+    },
+    {
+      "epoch": 1.9331495520330806,
+      "grad_norm": 2.0224292278289795,
+      "learning_rate": 1.0147925326797022e-08,
+      "logits/chosen": -2.944052219390869,
+      "logits/rejected": -2.928832530975342,
+      "logps/chosen": -58.519447326660156,
+      "logps/rejected": -64.93287658691406,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.0728921964764595,
+      "rewards/margins": 0.02828667126595974,
+      "rewards/rejected": -0.10117886960506439,
+      "step": 5610
+    },
+    {
+      "epoch": 1.9365954514128187,
+      "grad_norm": 1.8759541511535645,
+      "learning_rate": 1.009105266689011e-08,
+      "logits/chosen": -2.9557347297668457,
+      "logits/rejected": -2.941450357437134,
+      "logps/chosen": -63.048973083496094,
+      "logps/rejected": -62.200172424316406,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.07839448750019073,
+      "rewards/margins": 0.018764179199934006,
+      "rewards/rejected": -0.09715865552425385,
+      "step": 5620
+    },
+    {
+      "epoch": 1.940041350792557,
+      "grad_norm": 2.0119643211364746,
+      "learning_rate": 1.003425893106437e-08,
+      "logits/chosen": -2.9537787437438965,
+      "logits/rejected": -2.923032760620117,
+      "logps/chosen": -62.9396858215332,
+      "logps/rejected": -62.45795822143555,
+      "loss": 0.6792,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.06878829747438431,
+      "rewards/margins": 0.029859939590096474,
+      "rewards/rejected": -0.09864823520183563,
+      "step": 5630
+    },
+    {
+      "epoch": 1.943487250172295,
+      "grad_norm": 2.007880449295044,
+      "learning_rate": 9.977545032426657e-09,
+      "logits/chosen": -3.01012921333313,
+      "logits/rejected": -2.979600429534912,
+      "logps/chosen": -62.872344970703125,
+      "logps/rejected": -61.77165985107422,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0693301409482956,
+      "rewards/margins": 0.031130719929933548,
+      "rewards/rejected": -0.10046084970235825,
+      "step": 5640
+    },
+    {
+      "epoch": 1.946933149552033,
+      "grad_norm": 2.0887415409088135,
+      "learning_rate": 9.920911882800218e-09,
+      "logits/chosen": -2.9794821739196777,
+      "logits/rejected": -2.958545684814453,
+      "logps/chosen": -66.03761291503906,
+      "logps/rejected": -64.25897979736328,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.06596143543720245,
+      "rewards/margins": 0.027365287765860558,
+      "rewards/rejected": -0.09332672506570816,
+      "step": 5650
+    },
+    {
+      "epoch": 1.9503790489317712,
+      "grad_norm": 2.0464489459991455,
+      "learning_rate": 9.864360392710066e-09,
+      "logits/chosen": -3.035749912261963,
+      "logits/rejected": -3.0177814960479736,
+      "logps/chosen": -59.415611267089844,
+      "logps/rejected": -63.14741134643555,
+      "loss": 0.6757,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.06723272055387497,
+      "rewards/margins": 0.03708181530237198,
+      "rewards/rejected": -0.10431454330682755,
+      "step": 5660
+    },
+    {
+      "epoch": 1.9538249483115093,
+      "grad_norm": 1.9978302717208862,
+      "learning_rate": 9.807891471368312e-09,
+      "logits/chosen": -2.951850414276123,
+      "logits/rejected": -2.9236338138580322,
+      "logps/chosen": -61.29400634765625,
+      "logps/rejected": -62.02106857299805,
+      "loss": 0.6779,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07324939221143723,
+      "rewards/margins": 0.0324929878115654,
+      "rewards/rejected": -0.10574238002300262,
+      "step": 5670
+    },
+    {
+      "epoch": 1.9572708476912473,
+      "grad_norm": 1.87251877784729,
+      "learning_rate": 9.751506026659573e-09,
+      "logits/chosen": -3.027592182159424,
+      "logits/rejected": -3.006997585296631,
+      "logps/chosen": -62.29991912841797,
+      "logps/rejected": -62.658912658691406,
+      "loss": 0.681,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.07719240337610245,
+      "rewards/margins": 0.026362096890807152,
+      "rewards/rejected": -0.10355450212955475,
+      "step": 5680
+    },
+    {
+      "epoch": 1.9607167470709856,
+      "grad_norm": 1.823298454284668,
+      "learning_rate": 9.695204965126343e-09,
+      "logits/chosen": -2.951239585876465,
+      "logits/rejected": -2.9320054054260254,
+      "logps/chosen": -61.47679901123047,
+      "logps/rejected": -61.892799377441406,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.07422500848770142,
+      "rewards/margins": 0.02622462436556816,
+      "rewards/rejected": -0.10044963657855988,
+      "step": 5690
+    },
+    {
+      "epoch": 1.9641626464507236,
+      "grad_norm": 2.022496461868286,
+      "learning_rate": 9.638989191954457e-09,
+      "logits/chosen": -3.0318164825439453,
+      "logits/rejected": -3.004577398300171,
+      "logps/chosen": -60.8552360534668,
+      "logps/rejected": -64.1363754272461,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.06736666709184647,
+      "rewards/margins": 0.030862916260957718,
+      "rewards/rejected": -0.09822957217693329,
+      "step": 5700
+    },
+    {
+      "epoch": 1.9641626464507236,
+      "eval_logits/chosen": -3.0758543014526367,
+      "eval_logits/rejected": -3.070146322250366,
+      "eval_logps/chosen": -62.70294952392578,
+      "eval_logps/rejected": -68.44388580322266,
+      "eval_loss": 0.6873654723167419,
+      "eval_rewards/accuracies": 0.5940985083580017,
+      "eval_rewards/chosen": -0.039910539984703064,
+      "eval_rewards/margins": 0.012727024033665657,
+      "eval_rewards/rejected": -0.05263756960630417,
+      "eval_runtime": 383.4354,
+      "eval_samples_per_second": 11.225,
+      "eval_steps_per_second": 1.403,
+      "step": 5700
+    },
+    {
+      "epoch": 1.9676085458304617,
+      "grad_norm": 2.0420546531677246,
+      "learning_rate": 9.58285961095851e-09,
+      "logits/chosen": -3.004363536834717,
+      "logits/rejected": -2.974085569381714,
+      "logps/chosen": -61.52935791015625,
+      "logps/rejected": -62.240211486816406,
+      "loss": 0.6806,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.06844013929367065,
+      "rewards/margins": 0.027322733774781227,
+      "rewards/rejected": -0.09576286375522614,
+      "step": 5710
+    },
+    {
+      "epoch": 1.9710544452102,
+      "grad_norm": 1.9030342102050781,
+      "learning_rate": 9.52681712456733e-09,
+      "logits/chosen": -2.9542064666748047,
+      "logits/rejected": -2.9350638389587402,
+      "logps/chosen": -61.27290725708008,
+      "logps/rejected": -62.10396194458008,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.07512662559747696,
+      "rewards/margins": 0.019520198926329613,
+      "rewards/rejected": -0.09464682638645172,
+      "step": 5720
+    },
+    {
+      "epoch": 1.9745003445899378,
+      "grad_norm": 2.104343891143799,
+      "learning_rate": 9.470862633809482e-09,
+      "logits/chosen": -2.9374020099639893,
+      "logits/rejected": -2.915998697280884,
+      "logps/chosen": -61.51774215698242,
+      "logps/rejected": -63.5736198425293,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.07394744455814362,
+      "rewards/margins": 0.031605564057826996,
+      "rewards/rejected": -0.10555301606655121,
+      "step": 5730
+    },
+    {
+      "epoch": 1.9779462439696762,
+      "grad_norm": 2.028592348098755,
+      "learning_rate": 9.414997038298761e-09,
+      "logits/chosen": -3.029654026031494,
+      "logits/rejected": -2.997885227203369,
+      "logps/chosen": -63.5238037109375,
+      "logps/rejected": -61.163330078125,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07055319100618362,
+      "rewards/margins": 0.034039177000522614,
+      "rewards/rejected": -0.10459236800670624,
+      "step": 5740
+    },
+    {
+      "epoch": 1.9813921433494142,
+      "grad_norm": 1.7321242094039917,
+      "learning_rate": 9.359221236219755e-09,
+      "logits/chosen": -2.992835521697998,
+      "logits/rejected": -2.9676997661590576,
+      "logps/chosen": -60.078529357910156,
+      "logps/rejected": -60.62860107421875,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.07588743418455124,
+      "rewards/margins": 0.020864050835371017,
+      "rewards/rejected": -0.09675148874521255,
+      "step": 5750
+    },
+    {
+      "epoch": 1.9848380427291523,
+      "grad_norm": 1.9875448942184448,
+      "learning_rate": 9.303536124313367e-09,
+      "logits/chosen": -2.971609592437744,
+      "logits/rejected": -2.938375473022461,
+      "logps/chosen": -64.27455139160156,
+      "logps/rejected": -61.8266487121582,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.06635001301765442,
+      "rewards/margins": 0.035691313445568085,
+      "rewards/rejected": -0.1020413190126419,
+      "step": 5760
+    },
+    {
+      "epoch": 1.9882839421088905,
+      "grad_norm": 1.8601410388946533,
+      "learning_rate": 9.24794259786245e-09,
+      "logits/chosen": -3.011448860168457,
+      "logits/rejected": -2.9878077507019043,
+      "logps/chosen": -59.161956787109375,
+      "logps/rejected": -62.191368103027344,
+      "loss": 0.679,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07057367265224457,
+      "rewards/margins": 0.03015434369444847,
+      "rewards/rejected": -0.10072802007198334,
+      "step": 5770
+    },
+    {
+      "epoch": 1.9917298414886284,
+      "grad_norm": 1.99493408203125,
+      "learning_rate": 9.192441550677355e-09,
+      "logits/chosen": -2.9747166633605957,
+      "logits/rejected": -2.9498984813690186,
+      "logps/chosen": -60.608306884765625,
+      "logps/rejected": -62.46123123168945,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07452099025249481,
+      "rewards/margins": 0.027043115347623825,
+      "rewards/rejected": -0.10156410932540894,
+      "step": 5780
+    },
+    {
+      "epoch": 1.9951757408683668,
+      "grad_norm": 2.1565020084381104,
+      "learning_rate": 9.137033875081596e-09,
+      "logits/chosen": -3.007127285003662,
+      "logits/rejected": -2.9871790409088135,
+      "logps/chosen": -63.29633331298828,
+      "logps/rejected": -66.03068542480469,
+      "loss": 0.683,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.0811171680688858,
+      "rewards/margins": 0.022318918257951736,
+      "rewards/rejected": -0.10343609005212784,
+      "step": 5790
+    },
+    {
+      "epoch": 1.9986216402481047,
+      "grad_norm": 1.9688308238983154,
+      "learning_rate": 9.081720461897517e-09,
+      "logits/chosen": -2.9429879188537598,
+      "logits/rejected": -2.9171478748321533,
+      "logps/chosen": -61.838096618652344,
+      "logps/rejected": -62.12662887573242,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.06862715631723404,
+      "rewards/margins": 0.028616968542337418,
+      "rewards/rejected": -0.09724412113428116,
+      "step": 5800
+    },
+    {
+      "epoch": 1.9986216402481047,
+      "eval_logits/chosen": -3.0751535892486572,
+      "eval_logits/rejected": -3.069427967071533,
+      "eval_logps/chosen": -62.81398010253906,
+      "eval_logps/rejected": -68.56317901611328,
+      "eval_loss": 0.6873329877853394,
+      "eval_rewards/accuracies": 0.5924721360206604,
+      "eval_rewards/chosen": -0.041020870208740234,
+      "eval_rewards/margins": 0.012809707783162594,
+      "eval_rewards/rejected": -0.053830586373806,
+      "eval_runtime": 383.5698,
+      "eval_samples_per_second": 11.221,
+      "eval_steps_per_second": 1.403,
+      "step": 5800
+    },
+    {
+      "epoch": 2.0020675396278427,
+      "grad_norm": 2.279977321624756,
+      "learning_rate": 9.026502200431913e-09,
+      "logits/chosen": -2.9637136459350586,
+      "logits/rejected": -2.945895195007324,
+      "logps/chosen": -61.433616638183594,
+      "logps/rejected": -63.94523239135742,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.07496887445449829,
+      "rewards/margins": 0.02477206103503704,
+      "rewards/rejected": -0.09974093735218048,
+      "step": 5810
+    },
+    {
+      "epoch": 2.005513439007581,
+      "grad_norm": 2.0256128311157227,
+      "learning_rate": 8.971379978461802e-09,
+      "logits/chosen": -3.028080463409424,
+      "logits/rejected": -3.0011210441589355,
+      "logps/chosen": -62.13984298706055,
+      "logps/rejected": -61.86695098876953,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.06886421144008636,
+      "rewards/margins": 0.03483487293124199,
+      "rewards/rejected": -0.10369908809661865,
+      "step": 5820
+    },
+    {
+      "epoch": 2.008959338387319,
+      "grad_norm": 1.9036868810653687,
+      "learning_rate": 8.916354682220094e-09,
+      "logits/chosen": -2.986652135848999,
+      "logits/rejected": -2.947807788848877,
+      "logps/chosen": -61.61494827270508,
+      "logps/rejected": -61.35460662841797,
+      "loss": 0.675,
+      "rewards/accuracies": 0.684374988079071,
+      "rewards/chosen": -0.06822475790977478,
+      "rewards/margins": 0.03826071321964264,
+      "rewards/rejected": -0.10648547112941742,
+      "step": 5830
+    },
+    {
+      "epoch": 2.0124052377670574,
+      "grad_norm": 2.044987678527832,
+      "learning_rate": 8.861427196381379e-09,
+      "logits/chosen": -2.9417223930358887,
+      "logits/rejected": -2.9151341915130615,
+      "logps/chosen": -62.5283203125,
+      "logps/rejected": -66.14372253417969,
+      "loss": 0.6742,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.06608079373836517,
+      "rewards/margins": 0.04054456949234009,
+      "rewards/rejected": -0.10662537813186646,
+      "step": 5840
+    },
+    {
+      "epoch": 2.0158511371467953,
+      "grad_norm": 1.888162612915039,
+      "learning_rate": 8.806598404047676e-09,
+      "logits/chosen": -3.0402235984802246,
+      "logits/rejected": -3.0071463584899902,
+      "logps/chosen": -65.22200775146484,
+      "logps/rejected": -65.580322265625,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.06796949356794357,
+      "rewards/margins": 0.031590692698955536,
+      "rewards/rejected": -0.0995601937174797,
+      "step": 5850
+    },
+    {
+      "epoch": 2.0192970365265333,
+      "grad_norm": 1.9894851446151733,
+      "learning_rate": 8.751869186734272e-09,
+      "logits/chosen": -3.029601573944092,
+      "logits/rejected": -3.0097529888153076,
+      "logps/chosen": -61.56315231323242,
+      "logps/rejected": -62.81795120239258,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.06944890320301056,
+      "rewards/margins": 0.02943781577050686,
+      "rewards/rejected": -0.09888671338558197,
+      "step": 5860
+    },
+    {
+      "epoch": 2.0227429359062716,
+      "grad_norm": 1.9734573364257812,
+      "learning_rate": 8.697240424355508e-09,
+      "logits/chosen": -3.0121312141418457,
+      "logits/rejected": -2.996581554412842,
+      "logps/chosen": -61.764434814453125,
+      "logps/rejected": -65.39985656738281,
+      "loss": 0.683,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0770271047949791,
+      "rewards/margins": 0.02229788899421692,
+      "rewards/rejected": -0.09932499378919601,
+      "step": 5870
+    },
+    {
+      "epoch": 2.0261888352860096,
+      "grad_norm": 2.1268270015716553,
+      "learning_rate": 8.642712995210662e-09,
+      "logits/chosen": -3.0251002311706543,
+      "logits/rejected": -3.0166232585906982,
+      "logps/chosen": -58.933074951171875,
+      "logps/rejected": -64.60674285888672,
+      "loss": 0.6797,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.07400555908679962,
+      "rewards/margins": 0.028901178389787674,
+      "rewards/rejected": -0.1029067412018776,
+      "step": 5880
+    },
+    {
+      "epoch": 2.029634734665748,
+      "grad_norm": 2.0823442935943604,
+      "learning_rate": 8.588287775969809e-09,
+      "logits/chosen": -2.933594226837158,
+      "logits/rejected": -2.909440755844116,
+      "logps/chosen": -60.955535888671875,
+      "logps/rejected": -62.182350158691406,
+      "loss": 0.68,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.07374948263168335,
+      "rewards/margins": 0.02842150628566742,
+      "rewards/rejected": -0.10217098891735077,
+      "step": 5890
+    },
+    {
+      "epoch": 2.033080634045486,
+      "grad_norm": 1.9282052516937256,
+      "learning_rate": 8.533965641659738e-09,
+      "logits/chosen": -2.971388339996338,
+      "logits/rejected": -2.9638326168060303,
+      "logps/chosen": -61.447418212890625,
+      "logps/rejected": -64.0962905883789,
+      "loss": 0.683,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08197839558124542,
+      "rewards/margins": 0.022282596677541733,
+      "rewards/rejected": -0.10426100343465805,
+      "step": 5900
+    },
+    {
+      "epoch": 2.033080634045486,
+      "eval_logits/chosen": -3.0734519958496094,
+      "eval_logits/rejected": -3.067736864089966,
+      "eval_logps/chosen": -62.89165496826172,
+      "eval_logps/rejected": -68.66985321044922,
+      "eval_loss": 0.687201976776123,
+      "eval_rewards/accuracies": 0.5934014916419983,
+      "eval_rewards/chosen": -0.04179762676358223,
+      "eval_rewards/margins": 0.013099705800414085,
+      "eval_rewards/rejected": -0.054897334426641464,
+      "eval_runtime": 383.2119,
+      "eval_samples_per_second": 11.231,
+      "eval_steps_per_second": 1.404,
+      "step": 5900
+    },
+    {
+      "epoch": 2.036526533425224,
+      "grad_norm": 2.184607982635498,
+      "learning_rate": 8.479747465649882e-09,
+      "logits/chosen": -2.980534791946411,
+      "logits/rejected": -2.9656107425689697,
+      "logps/chosen": -60.94620895385742,
+      "logps/rejected": -62.71619415283203,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.08011116087436676,
+      "rewards/margins": 0.0249998327344656,
+      "rewards/rejected": -0.1051110029220581,
+      "step": 5910
+    },
+    {
+      "epoch": 2.039972432804962,
+      "grad_norm": 2.0466675758361816,
+      "learning_rate": 8.42563411963827e-09,
+      "logits/chosen": -2.9591071605682373,
+      "logits/rejected": -2.944737672805786,
+      "logps/chosen": -59.10858917236328,
+      "logps/rejected": -63.06071090698242,
+      "loss": 0.6823,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.07478350400924683,
+      "rewards/margins": 0.02361924760043621,
+      "rewards/rejected": -0.09840276092290878,
+      "step": 5920
+    },
+    {
+      "epoch": 2.0434183321847,
+      "grad_norm": 1.9219083786010742,
+      "learning_rate": 8.371626473637526e-09,
+      "logits/chosen": -2.9583218097686768,
+      "logits/rejected": -2.9300689697265625,
+      "logps/chosen": -63.11259841918945,
+      "logps/rejected": -63.8694953918457,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.08291836082935333,
+      "rewards/margins": 0.023417048156261444,
+      "rewards/rejected": -0.10633540153503418,
+      "step": 5930
+    },
+    {
+      "epoch": 2.0468642315644385,
+      "grad_norm": 2.1625397205352783,
+      "learning_rate": 8.317725395960845e-09,
+      "logits/chosen": -2.950472354888916,
+      "logits/rejected": -2.933504819869995,
+      "logps/chosen": -59.67982864379883,
+      "logps/rejected": -64.77800750732422,
+      "loss": 0.6776,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.06993059068918228,
+      "rewards/margins": 0.03344365209341049,
+      "rewards/rejected": -0.10337424278259277,
+      "step": 5940
+    },
+    {
+      "epoch": 2.0503101309441765,
+      "grad_norm": 1.9321199655532837,
+      "learning_rate": 8.2639317532081e-09,
+      "logits/chosen": -2.956042528152466,
+      "logits/rejected": -2.9313361644744873,
+      "logps/chosen": -62.14270782470703,
+      "logps/rejected": -62.8015022277832,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.07490450143814087,
+      "rewards/margins": 0.031131360679864883,
+      "rewards/rejected": -0.10603587329387665,
+      "step": 5950
+    },
+    {
+      "epoch": 2.0537560303239144,
+      "grad_norm": 2.1241021156311035,
+      "learning_rate": 8.210246410251831e-09,
+      "logits/chosen": -3.0088963508605957,
+      "logits/rejected": -3.0028107166290283,
+      "logps/chosen": -62.5322380065918,
+      "logps/rejected": -67.2044906616211,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.07370878010988235,
+      "rewards/margins": 0.024065060541033745,
+      "rewards/rejected": -0.09777384251356125,
+      "step": 5960
+    },
+    {
+      "epoch": 2.057201929703653,
+      "grad_norm": 2.1052968502044678,
+      "learning_rate": 8.156670230223402e-09,
+      "logits/chosen": -2.976123571395874,
+      "logits/rejected": -2.960484266281128,
+      "logps/chosen": -60.7321891784668,
+      "logps/rejected": -64.26460266113281,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.0726715698838234,
+      "rewards/margins": 0.024902159348130226,
+      "rewards/rejected": -0.09757371991872787,
+      "step": 5970
+    },
+    {
+      "epoch": 2.0606478290833907,
+      "grad_norm": 2.034003257751465,
+      "learning_rate": 8.103204074499092e-09,
+      "logits/chosen": -3.0056068897247314,
+      "logits/rejected": -2.9824938774108887,
+      "logps/chosen": -62.354156494140625,
+      "logps/rejected": -64.16876983642578,
+      "loss": 0.683,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.0733756422996521,
+      "rewards/margins": 0.022388001903891563,
+      "rewards/rejected": -0.09576363861560822,
+      "step": 5980
+    },
+    {
+      "epoch": 2.0640937284631287,
+      "grad_norm": 2.0037052631378174,
+      "learning_rate": 8.049848802686247e-09,
+      "logits/chosen": -2.995795249938965,
+      "logits/rejected": -2.9743247032165527,
+      "logps/chosen": -62.59307098388672,
+      "logps/rejected": -63.9239616394043,
+      "loss": 0.68,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.07116366922855377,
+      "rewards/margins": 0.028568390756845474,
+      "rewards/rejected": -0.09973206371068954,
+      "step": 5990
+    },
+    {
+      "epoch": 2.067539627842867,
+      "grad_norm": 2.040750741958618,
+      "learning_rate": 7.996605272609481e-09,
+      "logits/chosen": -3.008589506149292,
+      "logits/rejected": -2.9717698097229004,
+      "logps/chosen": -61.6326904296875,
+      "logps/rejected": -64.22892761230469,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.07474110275506973,
+      "rewards/margins": 0.035318586975336075,
+      "rewards/rejected": -0.11005967855453491,
+      "step": 6000
+    },
+    {
+      "epoch": 2.067539627842867,
+      "eval_logits/chosen": -3.0732250213623047,
+      "eval_logits/rejected": -3.067477226257324,
+      "eval_logps/chosen": -62.95998764038086,
+      "eval_logps/rejected": -68.73140716552734,
+      "eval_loss": 0.6872404217720032,
+      "eval_rewards/accuracies": 0.591775119304657,
+      "eval_rewards/chosen": -0.042480871081352234,
+      "eval_rewards/margins": 0.013031977228820324,
+      "eval_rewards/rejected": -0.05551284924149513,
+      "eval_runtime": 383.5362,
+      "eval_samples_per_second": 11.222,
+      "eval_steps_per_second": 1.403,
+      "step": 6000
+    },
+    {
+      "epoch": 2.070985527222605,
+      "grad_norm": 1.9154592752456665,
+      "learning_rate": 7.94347434029684e-09,
+      "logits/chosen": -2.997335433959961,
+      "logits/rejected": -2.9779839515686035,
+      "logps/chosen": -60.9912223815918,
+      "logps/rejected": -64.15106964111328,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07163038104772568,
+      "rewards/margins": 0.027485957369208336,
+      "rewards/rejected": -0.09911634027957916,
+      "step": 6010
+    },
+    {
+      "epoch": 2.0744314266023434,
+      "grad_norm": 2.0360772609710693,
+      "learning_rate": 7.890456859966114e-09,
+      "logits/chosen": -3.032085418701172,
+      "logits/rejected": -3.009577751159668,
+      "logps/chosen": -59.37373733520508,
+      "logps/rejected": -61.25800704956055,
+      "loss": 0.679,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07022441178560257,
+      "rewards/margins": 0.030777905136346817,
+      "rewards/rejected": -0.10100232064723969,
+      "step": 6020
+    },
+    {
+      "epoch": 2.0778773259820813,
+      "grad_norm": 2.1235880851745605,
+      "learning_rate": 7.837553684011005e-09,
+      "logits/chosen": -2.999788522720337,
+      "logits/rejected": -2.9670815467834473,
+      "logps/chosen": -62.13616943359375,
+      "logps/rejected": -63.023963928222656,
+      "loss": 0.6779,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.07085924595594406,
+      "rewards/margins": 0.032673902809619904,
+      "rewards/rejected": -0.10353314876556396,
+      "step": 6030
+    },
+    {
+      "epoch": 2.0813232253618192,
+      "grad_norm": 2.0067641735076904,
+      "learning_rate": 7.784765662987527e-09,
+      "logits/chosen": -2.9324898719787598,
+      "logits/rejected": -2.902146100997925,
+      "logps/chosen": -62.274925231933594,
+      "logps/rejected": -62.84496307373047,
+      "loss": 0.6771,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.07156813144683838,
+      "rewards/margins": 0.03470544144511223,
+      "rewards/rejected": -0.10627356916666031,
+      "step": 6040
+    },
+    {
+      "epoch": 2.0847691247415576,
+      "grad_norm": 2.1125595569610596,
+      "learning_rate": 7.732093645600237e-09,
+      "logits/chosen": -2.9779458045959473,
+      "logits/rejected": -2.951784133911133,
+      "logps/chosen": -64.39039611816406,
+      "logps/rejected": -62.5811882019043,
+      "loss": 0.68,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.07079866528511047,
+      "rewards/margins": 0.02864238992333412,
+      "rewards/rejected": -0.0994410514831543,
+      "step": 6050
+    },
+    {
+      "epoch": 2.0882150241212956,
+      "grad_norm": 1.9622465372085571,
+      "learning_rate": 7.679538478688648e-09,
+      "logits/chosen": -2.986208438873291,
+      "logits/rejected": -2.95941162109375,
+      "logps/chosen": -64.37217712402344,
+      "logps/rejected": -63.79829025268555,
+      "loss": 0.6792,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.07895328104496002,
+      "rewards/margins": 0.030250677838921547,
+      "rewards/rejected": -0.10920395702123642,
+      "step": 6060
+    },
+    {
+      "epoch": 2.091660923501034,
+      "grad_norm": 1.86594557762146,
+      "learning_rate": 7.627101007213619e-09,
+      "logits/chosen": -2.9775543212890625,
+      "logits/rejected": -2.9579451084136963,
+      "logps/chosen": -60.78815460205078,
+      "logps/rejected": -64.31053924560547,
+      "loss": 0.6806,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.07592712342739105,
+      "rewards/margins": 0.02719021774828434,
+      "rewards/rejected": -0.10311733186244965,
+      "step": 6070
+    },
+    {
+      "epoch": 2.095106822880772,
+      "grad_norm": 2.0590782165527344,
+      "learning_rate": 7.574782074243703e-09,
+      "logits/chosen": -2.9331212043762207,
+      "logits/rejected": -2.910733461380005,
+      "logps/chosen": -61.803184509277344,
+      "logps/rejected": -63.19152069091797,
+      "loss": 0.68,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.07108216732740402,
+      "rewards/margins": 0.02835419774055481,
+      "rewards/rejected": -0.09943635761737823,
+      "step": 6080
+    },
+    {
+      "epoch": 2.09855272226051,
+      "grad_norm": 1.9900931119918823,
+      "learning_rate": 7.522582520941693e-09,
+      "logits/chosen": -2.914452075958252,
+      "logits/rejected": -2.9096360206604004,
+      "logps/chosen": -59.50273513793945,
+      "logps/rejected": -67.2176513671875,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0795271173119545,
+      "rewards/margins": 0.023171132430434227,
+      "rewards/rejected": -0.10269824415445328,
+      "step": 6090
+    },
+    {
+      "epoch": 2.101998621640248,
+      "grad_norm": 1.964848518371582,
+      "learning_rate": 7.470503186550998e-09,
+      "logits/chosen": -2.9823858737945557,
+      "logits/rejected": -2.953029155731201,
+      "logps/chosen": -63.8648681640625,
+      "logps/rejected": -64.69880676269531,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.07009677588939667,
+      "rewards/margins": 0.037469152361154556,
+      "rewards/rejected": -0.10756593942642212,
+      "step": 6100
+    },
+    {
+      "epoch": 2.101998621640248,
+      "eval_logits/chosen": -3.071739673614502,
+      "eval_logits/rejected": -3.0659902095794678,
+      "eval_logps/chosen": -62.99589920043945,
+      "eval_logps/rejected": -68.79497528076172,
+      "eval_loss": 0.6871066689491272,
+      "eval_rewards/accuracies": 0.5922397971153259,
+      "eval_rewards/chosen": -0.042840003967285156,
+      "eval_rewards/margins": 0.013308479450643063,
+      "eval_rewards/rejected": -0.056148480623960495,
+      "eval_runtime": 383.345,
+      "eval_samples_per_second": 11.227,
+      "eval_steps_per_second": 1.403,
+      "step": 6100
+    },
+    {
+      "epoch": 2.105444521019986,
+      "grad_norm": 1.9654420614242554,
+      "learning_rate": 7.418544908382215e-09,
+      "logits/chosen": -2.9355955123901367,
+      "logits/rejected": -2.912510395050049,
+      "logps/chosen": -61.56586837768555,
+      "logps/rejected": -63.9070930480957,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07963390648365021,
+      "rewards/margins": 0.03037925437092781,
+      "rewards/rejected": -0.11001316457986832,
+      "step": 6110
+    },
+    {
+      "epoch": 2.1088904203997245,
+      "grad_norm": 1.911759853363037,
+      "learning_rate": 7.366708521799648e-09,
+      "logits/chosen": -2.9101290702819824,
+      "logits/rejected": -2.8934082984924316,
+      "logps/chosen": -60.84368896484375,
+      "logps/rejected": -62.7339973449707,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08548275381326675,
+      "rewards/margins": 0.021761415526270866,
+      "rewards/rejected": -0.10724417120218277,
+      "step": 6120
+    },
+    {
+      "epoch": 2.1123363197794625,
+      "grad_norm": 1.9455486536026,
+      "learning_rate": 7.3149948602078774e-09,
+      "logits/chosen": -2.9502012729644775,
+      "logits/rejected": -2.931739330291748,
+      "logps/chosen": -60.4312629699707,
+      "logps/rejected": -64.15070343017578,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.07931514084339142,
+      "rewards/margins": 0.023938264697790146,
+      "rewards/rejected": -0.10325340926647186,
+      "step": 6130
+    },
+    {
+      "epoch": 2.1157822191592004,
+      "grad_norm": 2.076716899871826,
+      "learning_rate": 7.263404755038358e-09,
+      "logits/chosen": -2.987335205078125,
+      "logits/rejected": -2.960381031036377,
+      "logps/chosen": -63.1358642578125,
+      "logps/rejected": -64.80619812011719,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.0747191309928894,
+      "rewards/margins": 0.036992788314819336,
+      "rewards/rejected": -0.11171190440654755,
+      "step": 6140
+    },
+    {
+      "epoch": 2.1192281185389388,
+      "grad_norm": 2.0410284996032715,
+      "learning_rate": 7.211939035736047e-09,
+      "logits/chosen": -2.9341626167297363,
+      "logits/rejected": -2.9067935943603516,
+      "logps/chosen": -61.7495002746582,
+      "logps/rejected": -63.37883758544922,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.07518855482339859,
+      "rewards/margins": 0.03164491802453995,
+      "rewards/rejected": -0.10683347284793854,
+      "step": 6150
+    },
+    {
+      "epoch": 2.1226740179186767,
+      "grad_norm": 2.1463143825531006,
+      "learning_rate": 7.160598529746104e-09,
+      "logits/chosen": -2.967750310897827,
+      "logits/rejected": -2.9465582370758057,
+      "logps/chosen": -61.61671829223633,
+      "logps/rejected": -63.64253616333008,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07017774879932404,
+      "rewards/margins": 0.030852749943733215,
+      "rewards/rejected": -0.10103049129247665,
+      "step": 6160
+    },
+    {
+      "epoch": 2.126119917298415,
+      "grad_norm": 2.081512212753296,
+      "learning_rate": 7.1093840625005165e-09,
+      "logits/chosen": -2.986746311187744,
+      "logits/rejected": -2.969165325164795,
+      "logps/chosen": -62.87702560424805,
+      "logps/rejected": -64.62913513183594,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0795002430677414,
+      "rewards/margins": 0.023195894435048103,
+      "rewards/rejected": -0.10269614309072495,
+      "step": 6170
+    },
+    {
+      "epoch": 2.129565816678153,
+      "grad_norm": 2.276735544204712,
+      "learning_rate": 7.058296457404905e-09,
+      "logits/chosen": -2.9371447563171387,
+      "logits/rejected": -2.905818223953247,
+      "logps/chosen": -62.342926025390625,
+      "logps/rejected": -62.10832977294922,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.07544146478176117,
+      "rewards/margins": 0.033278729766607285,
+      "rewards/rejected": -0.10872019827365875,
+      "step": 6180
+    },
+    {
+      "epoch": 2.133011716057891,
+      "grad_norm": 2.0503451824188232,
+      "learning_rate": 7.007336535825236e-09,
+      "logits/chosen": -2.961604595184326,
+      "logits/rejected": -2.9490857124328613,
+      "logps/chosen": -59.247650146484375,
+      "logps/rejected": -63.7967643737793,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.07756395637989044,
+      "rewards/margins": 0.025081371888518333,
+      "rewards/rejected": -0.10264533758163452,
+      "step": 6190
+    },
+    {
+      "epoch": 2.1364576154376294,
+      "grad_norm": 2.2487685680389404,
+      "learning_rate": 6.956505117074638e-09,
+      "logits/chosen": -3.0378546714782715,
+      "logits/rejected": -3.0309805870056152,
+      "logps/chosen": -62.0113639831543,
+      "logps/rejected": -66.18428039550781,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.07807514816522598,
+      "rewards/margins": 0.02754230797290802,
+      "rewards/rejected": -0.1056174635887146,
+      "step": 6200
+    },
+    {
+      "epoch": 2.1364576154376294,
+      "eval_logits/chosen": -3.071140766143799,
+      "eval_logits/rejected": -3.065408945083618,
+      "eval_logps/chosen": -63.061134338378906,
+      "eval_logps/rejected": -68.86215209960938,
+      "eval_loss": 0.6871089339256287,
+      "eval_rewards/accuracies": 0.5903810262680054,
+      "eval_rewards/chosen": -0.043492451310157776,
+      "eval_rewards/margins": 0.01332781556993723,
+      "eval_rewards/rejected": -0.05682026594877243,
+      "eval_runtime": 383.4428,
+      "eval_samples_per_second": 11.225,
+      "eval_steps_per_second": 1.403,
+      "step": 6200
+    },
+    {
+      "epoch": 2.1399035148173673,
+      "grad_norm": 1.998321771621704,
+      "learning_rate": 6.90580301840022e-09,
+      "logits/chosen": -2.882282257080078,
+      "logits/rejected": -2.8518736362457275,
+      "logps/chosen": -61.777442932128906,
+      "logps/rejected": -62.86517333984375,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07116181403398514,
+      "rewards/margins": 0.03244227543473244,
+      "rewards/rejected": -0.10360409319400787,
+      "step": 6210
+    },
+    {
+      "epoch": 2.1433494141971057,
+      "grad_norm": 1.8551884889602661,
+      "learning_rate": 6.855231054969938e-09,
+      "logits/chosen": -2.9320473670959473,
+      "logits/rejected": -2.917422294616699,
+      "logps/chosen": -61.67425537109375,
+      "logps/rejected": -64.0980453491211,
+      "loss": 0.6797,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.07553770393133163,
+      "rewards/margins": 0.02908666431903839,
+      "rewards/rejected": -0.10462436825037003,
+      "step": 6220
+    },
+    {
+      "epoch": 2.1467953135768436,
+      "grad_norm": 1.9977219104766846,
+      "learning_rate": 6.804790039859479e-09,
+      "logits/chosen": -2.990032196044922,
+      "logits/rejected": -2.9682323932647705,
+      "logps/chosen": -60.44028854370117,
+      "logps/rejected": -62.64056396484375,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.07876847684383392,
+      "rewards/margins": 0.028518591076135635,
+      "rewards/rejected": -0.10728706419467926,
+      "step": 6230
+    },
+    {
+      "epoch": 2.1502412129565815,
+      "grad_norm": 1.965030550956726,
+      "learning_rate": 6.754480784039204e-09,
+      "logits/chosen": -3.049013137817383,
+      "logits/rejected": -3.0324606895446777,
+      "logps/chosen": -62.7895622253418,
+      "logps/rejected": -66.26988220214844,
+      "loss": 0.6787,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0793498083949089,
+      "rewards/margins": 0.031418927013874054,
+      "rewards/rejected": -0.11076873540878296,
+      "step": 6240
+    },
+    {
+      "epoch": 2.15368711233632,
+      "grad_norm": 1.9353604316711426,
+      "learning_rate": 6.704304096361096e-09,
+      "logits/chosen": -2.979348659515381,
+      "logits/rejected": -2.9551796913146973,
+      "logps/chosen": -61.64031982421875,
+      "logps/rejected": -64.72476959228516,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.07714991271495819,
+      "rewards/margins": 0.03489960357546806,
+      "rewards/rejected": -0.11204953491687775,
+      "step": 6250
+    },
+    {
+      "epoch": 2.157133011716058,
+      "grad_norm": 1.9478212594985962,
+      "learning_rate": 6.6542607835457605e-09,
+      "logits/chosen": -2.9587082862854004,
+      "logits/rejected": -2.937373638153076,
+      "logps/chosen": -60.916038513183594,
+      "logps/rejected": -64.54906463623047,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07657292485237122,
+      "rewards/margins": 0.033051785081624985,
+      "rewards/rejected": -0.1096247062087059,
+      "step": 6260
+    },
+    {
+      "epoch": 2.160578911095796,
+      "grad_norm": 1.9702144861221313,
+      "learning_rate": 6.604351650169458e-09,
+      "logits/chosen": -2.9709510803222656,
+      "logits/rejected": -2.955023765563965,
+      "logps/chosen": -61.792076110839844,
+      "logps/rejected": -65.45343017578125,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.07719365507364273,
+      "rewards/margins": 0.029838278889656067,
+      "rewards/rejected": -0.1070319190621376,
+      "step": 6270
+    },
+    {
+      "epoch": 2.164024810475534,
+      "grad_norm": 1.9763787984848022,
+      "learning_rate": 6.554577498651164e-09,
+      "logits/chosen": -3.052248477935791,
+      "logits/rejected": -3.02431321144104,
+      "logps/chosen": -61.69286346435547,
+      "logps/rejected": -61.5143928527832,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.07548774778842926,
+      "rewards/margins": 0.030979227274656296,
+      "rewards/rejected": -0.10646696388721466,
+      "step": 6280
+    },
+    {
+      "epoch": 2.167470709855272,
+      "grad_norm": 1.9427931308746338,
+      "learning_rate": 6.504939129239672e-09,
+      "logits/chosen": -2.979668140411377,
+      "logits/rejected": -2.943617343902588,
+      "logps/chosen": -63.29692459106445,
+      "logps/rejected": -62.54668045043945,
+      "loss": 0.6742,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.07073553651571274,
+      "rewards/margins": 0.040321726351976395,
+      "rewards/rejected": -0.11105725914239883,
+      "step": 6290
+    },
+    {
+      "epoch": 2.1709166092350105,
+      "grad_norm": 2.118011713027954,
+      "learning_rate": 6.455437340000723e-09,
+      "logits/chosen": -2.9488353729248047,
+      "logits/rejected": -2.9182028770446777,
+      "logps/chosen": -64.58491516113281,
+      "logps/rejected": -63.82842254638672,
+      "loss": 0.6797,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.0769583061337471,
+      "rewards/margins": 0.02924424409866333,
+      "rewards/rejected": -0.10620255768299103,
+      "step": 6300
+    },
+    {
+      "epoch": 2.1709166092350105,
+      "eval_logits/chosen": -3.0702764987945557,
+      "eval_logits/rejected": -3.0645322799682617,
+      "eval_logps/chosen": -63.13775634765625,
+      "eval_logps/rejected": -68.94925689697266,
+      "eval_loss": 0.6870641708374023,
+      "eval_rewards/accuracies": 0.5929368138313293,
+      "eval_rewards/chosen": -0.04425855726003647,
+      "eval_rewards/margins": 0.013432751409709454,
+      "eval_rewards/rejected": -0.057691313326358795,
+      "eval_runtime": 383.4046,
+      "eval_samples_per_second": 11.226,
+      "eval_steps_per_second": 1.403,
+      "step": 6300
+    },
+    {
+      "epoch": 2.1743625086147484,
+      "grad_norm": 2.124711513519287,
+      "learning_rate": 6.406072926804185e-09,
+      "logits/chosen": -3.0565030574798584,
+      "logits/rejected": -3.043586492538452,
+      "logps/chosen": -62.907958984375,
+      "logps/rejected": -64.94691467285156,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.08194388449192047,
+      "rewards/margins": 0.026618700474500656,
+      "rewards/rejected": -0.10856258869171143,
+      "step": 6310
+    },
+    {
+      "epoch": 2.1778084079944864,
+      "grad_norm": 2.1683335304260254,
+      "learning_rate": 6.356846683311226e-09,
+      "logits/chosen": -3.0117733478546143,
+      "logits/rejected": -3.0000782012939453,
+      "logps/chosen": -59.89496612548828,
+      "logps/rejected": -63.113037109375,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.07654803991317749,
+      "rewards/margins": 0.02702518366277218,
+      "rewards/rejected": -0.10357322543859482,
+      "step": 6320
+    },
+    {
+      "epoch": 2.1812543073742248,
+      "grad_norm": 2.0678176879882812,
+      "learning_rate": 6.3077594009616086e-09,
+      "logits/chosen": -2.928382635116577,
+      "logits/rejected": -2.908520221710205,
+      "logps/chosen": -60.35748291015625,
+      "logps/rejected": -62.62036895751953,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07794377207756042,
+      "rewards/margins": 0.023805899545550346,
+      "rewards/rejected": -0.10174968093633652,
+      "step": 6330
+    },
+    {
+      "epoch": 2.1847002067539627,
+      "grad_norm": 1.8298227787017822,
+      "learning_rate": 6.258811868960914e-09,
+      "logits/chosen": -3.0441901683807373,
+      "logits/rejected": -3.0136749744415283,
+      "logps/chosen": -59.760597229003906,
+      "logps/rejected": -62.6243896484375,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.07398231327533722,
+      "rewards/margins": 0.035731829702854156,
+      "rewards/rejected": -0.10971413552761078,
+      "step": 6340
+    },
+    {
+      "epoch": 2.188146106133701,
+      "grad_norm": 1.9831953048706055,
+      "learning_rate": 6.2100048742678775e-09,
+      "logits/chosen": -2.942831516265869,
+      "logits/rejected": -2.924875259399414,
+      "logps/chosen": -61.86394119262695,
+      "logps/rejected": -64.1031723022461,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.07543181627988815,
+      "rewards/margins": 0.02806379832327366,
+      "rewards/rejected": -0.10349562019109726,
+      "step": 6350
+    },
+    {
+      "epoch": 2.191592005513439,
+      "grad_norm": 2.127967357635498,
+      "learning_rate": 6.16133920158174e-09,
+      "logits/chosen": -3.0327537059783936,
+      "logits/rejected": -3.0112240314483643,
+      "logps/chosen": -62.401947021484375,
+      "logps/rejected": -65.04664611816406,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07288150489330292,
+      "rewards/margins": 0.03192085400223732,
+      "rewards/rejected": -0.10480235517024994,
+      "step": 6360
+    },
+    {
+      "epoch": 2.195037904893177,
+      "grad_norm": 2.07889723777771,
+      "learning_rate": 6.112815633329598e-09,
+      "logits/chosen": -2.94187593460083,
+      "logits/rejected": -2.9152863025665283,
+      "logps/chosen": -59.51696014404297,
+      "logps/rejected": -62.47137451171875,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.07994683086872101,
+      "rewards/margins": 0.03132204711437225,
+      "rewards/rejected": -0.11126887798309326,
+      "step": 6370
+    },
+    {
+      "epoch": 2.1984838042729153,
+      "grad_norm": 2.0071518421173096,
+      "learning_rate": 6.064434949653895e-09,
+      "logits/chosen": -2.938615083694458,
+      "logits/rejected": -2.9184181690216064,
+      "logps/chosen": -59.94279861450195,
+      "logps/rejected": -63.59552001953125,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.07112361490726471,
+      "rewards/margins": 0.027679506689310074,
+      "rewards/rejected": -0.09880311787128448,
+      "step": 6380
+    },
+    {
+      "epoch": 2.2019297036526533,
+      "grad_norm": 2.140047073364258,
+      "learning_rate": 6.0161979283997815e-09,
+      "logits/chosen": -3.0037620067596436,
+      "logits/rejected": -2.9761061668395996,
+      "logps/chosen": -63.174339294433594,
+      "logps/rejected": -63.59212112426758,
+      "loss": 0.683,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.0822693407535553,
+      "rewards/margins": 0.022446615621447563,
+      "rewards/rejected": -0.10471595823764801,
+      "step": 6390
+    },
+    {
+      "epoch": 2.205375603032391,
+      "grad_norm": 2.0242576599121094,
+      "learning_rate": 5.968105345102711e-09,
+      "logits/chosen": -2.9597809314727783,
+      "logits/rejected": -2.9422004222869873,
+      "logps/chosen": -63.55103302001953,
+      "logps/rejected": -65.6887435913086,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08325214684009552,
+      "rewards/margins": 0.02807902917265892,
+      "rewards/rejected": -0.11133117973804474,
+      "step": 6400
+    },
+    {
+      "epoch": 2.205375603032391,
+      "eval_logits/chosen": -3.069845199584961,
+      "eval_logits/rejected": -3.064060688018799,
+      "eval_logps/chosen": -63.13124465942383,
+      "eval_logps/rejected": -68.95304870605469,
+      "eval_loss": 0.6870155334472656,
+      "eval_rewards/accuracies": 0.5913103818893433,
+      "eval_rewards/chosen": -0.04419339820742607,
+      "eval_rewards/margins": 0.013535933569073677,
+      "eval_rewards/rejected": -0.0577293299138546,
+      "eval_runtime": 383.2616,
+      "eval_samples_per_second": 11.23,
+      "eval_steps_per_second": 1.404,
+      "step": 6400
+    },
+    {
+      "epoch": 2.2088215024121296,
+      "grad_norm": 2.0987372398376465,
+      "learning_rate": 5.920157972975877e-09,
+      "logits/chosen": -2.9663925170898438,
+      "logits/rejected": -2.9575889110565186,
+      "logps/chosen": -60.018898010253906,
+      "logps/rejected": -65.28751373291016,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.08195268362760544,
+      "rewards/margins": 0.02169257402420044,
+      "rewards/rejected": -0.10364526510238647,
+      "step": 6410
+    },
+    {
+      "epoch": 2.2122674017918675,
+      "grad_norm": 2.155315399169922,
+      "learning_rate": 5.8723565828978646e-09,
+      "logits/chosen": -2.9655935764312744,
+      "logits/rejected": -2.9475045204162598,
+      "logps/chosen": -61.3104248046875,
+      "logps/rejected": -64.29899597167969,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.08491487801074982,
+      "rewards/margins": 0.02359841763973236,
+      "rewards/rejected": -0.10851327329874039,
+      "step": 6420
+    },
+    {
+      "epoch": 2.215713301171606,
+      "grad_norm": 2.1627821922302246,
+      "learning_rate": 5.824701943400198e-09,
+      "logits/chosen": -2.9593396186828613,
+      "logits/rejected": -2.937176465988159,
+      "logps/chosen": -62.94902420043945,
+      "logps/rejected": -62.81300735473633,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0829174816608429,
+      "rewards/margins": 0.03100886382162571,
+      "rewards/rejected": -0.11392633616924286,
+      "step": 6430
+    },
+    {
+      "epoch": 2.219159200551344,
+      "grad_norm": 1.8996787071228027,
+      "learning_rate": 5.777194820654995e-09,
+      "logits/chosen": -2.9815545082092285,
+      "logits/rejected": -2.9659335613250732,
+      "logps/chosen": -60.490562438964844,
+      "logps/rejected": -63.5893669128418,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.07705775648355484,
+      "rewards/margins": 0.02753683552145958,
+      "rewards/rejected": -0.10459460318088531,
+      "step": 6440
+    },
+    {
+      "epoch": 2.222605099931082,
+      "grad_norm": 1.977771520614624,
+      "learning_rate": 5.729835978462689e-09,
+      "logits/chosen": -3.006312847137451,
+      "logits/rejected": -2.983654499053955,
+      "logps/chosen": -63.2271614074707,
+      "logps/rejected": -65.09318542480469,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07835768163204193,
+      "rewards/margins": 0.0304982028901577,
+      "rewards/rejected": -0.10885588079690933,
+      "step": 6450
+    },
+    {
+      "epoch": 2.22605099931082,
+      "grad_norm": 2.085115909576416,
+      "learning_rate": 5.68262617823968e-09,
+      "logits/chosen": -3.0096936225891113,
+      "logits/rejected": -2.9968369007110596,
+      "logps/chosen": -59.0042839050293,
+      "logps/rejected": -65.32162475585938,
+      "loss": 0.676,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.07355882227420807,
+      "rewards/margins": 0.03668289631605148,
+      "rewards/rejected": -0.11024172604084015,
+      "step": 6460
+    },
+    {
+      "epoch": 2.229496898690558,
+      "grad_norm": 2.0791873931884766,
+      "learning_rate": 5.635566179006173e-09,
+      "logits/chosen": -2.9672462940216064,
+      "logits/rejected": -2.946897506713867,
+      "logps/chosen": -58.38322067260742,
+      "logps/rejected": -61.30283737182617,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.07619024068117142,
+      "rewards/margins": 0.026396874338388443,
+      "rewards/rejected": -0.10258710384368896,
+      "step": 6470
+    },
+    {
+      "epoch": 2.2329427980702965,
+      "grad_norm": 1.94746732711792,
+      "learning_rate": 5.588656737373896e-09,
+      "logits/chosen": -2.929197072982788,
+      "logits/rejected": -2.904510021209717,
+      "logps/chosen": -63.93778610229492,
+      "logps/rejected": -64.65699768066406,
+      "loss": 0.677,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.07612042874097824,
+      "rewards/margins": 0.034740954637527466,
+      "rewards/rejected": -0.1108613833785057,
+      "step": 6480
+    },
+    {
+      "epoch": 2.2363886974500344,
+      "grad_norm": 2.1137585639953613,
+      "learning_rate": 5.541898607533994e-09,
+      "logits/chosen": -2.8964250087738037,
+      "logits/rejected": -2.8731698989868164,
+      "logps/chosen": -59.87591552734375,
+      "logps/rejected": -63.110572814941406,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.08127310127019882,
+      "rewards/margins": 0.03876413032412529,
+      "rewards/rejected": -0.12003723531961441,
+      "step": 6490
+    },
+    {
+      "epoch": 2.2398345968297724,
+      "grad_norm": 2.0684080123901367,
+      "learning_rate": 5.495292541244873e-09,
+      "logits/chosen": -3.005554676055908,
+      "logits/rejected": -2.9830431938171387,
+      "logps/chosen": -64.42156219482422,
+      "logps/rejected": -63.5334587097168,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.08337266743183136,
+      "rewards/margins": 0.027867352589964867,
+      "rewards/rejected": -0.11124001443386078,
+      "step": 6500
+    },
+    {
+      "epoch": 2.2398345968297724,
+      "eval_logits/chosen": -3.0689847469329834,
+      "eval_logits/rejected": -3.063261032104492,
+      "eval_logps/chosen": -63.15791320800781,
+      "eval_logps/rejected": -68.98910522460938,
+      "eval_loss": 0.6869712471961975,
+      "eval_rewards/accuracies": 0.5934014916419983,
+      "eval_rewards/chosen": -0.04446011781692505,
+      "eval_rewards/margins": 0.013629709370434284,
+      "eval_rewards/rejected": -0.058089833706617355,
+      "eval_runtime": 383.3848,
+      "eval_samples_per_second": 11.226,
+      "eval_steps_per_second": 1.403,
+      "step": 6500
+    },
+    {
+      "epoch": 2.2432804962095108,
+      "grad_norm": 1.9375041723251343,
+      "learning_rate": 5.448839287820122e-09,
+      "logits/chosen": -2.9162724018096924,
+      "logits/rejected": -2.896331310272217,
+      "logps/chosen": -59.616180419921875,
+      "logps/rejected": -61.519378662109375,
+      "loss": 0.6787,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08608312159776688,
+      "rewards/margins": 0.030905192717909813,
+      "rewards/rejected": -0.11698831617832184,
+      "step": 6510
+    },
+    {
+      "epoch": 2.2467263955892487,
+      "grad_norm": 2.123417615890503,
+      "learning_rate": 5.402539594116483e-09,
+      "logits/chosen": -3.04290509223938,
+      "logits/rejected": -3.0173213481903076,
+      "logps/chosen": -61.662208557128906,
+      "logps/rejected": -64.8007583618164,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.07568902522325516,
+      "rewards/margins": 0.03171851485967636,
+      "rewards/rejected": -0.10740754753351212,
+      "step": 6520
+    },
+    {
+      "epoch": 2.250172294968987,
+      "grad_norm": 2.1539101600646973,
+      "learning_rate": 5.356394204521798e-09,
+      "logits/chosen": -3.017493486404419,
+      "logits/rejected": -3.0126800537109375,
+      "logps/chosen": -60.36297607421875,
+      "logps/rejected": -64.49449920654297,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.08065467327833176,
+      "rewards/margins": 0.02114265225827694,
+      "rewards/rejected": -0.10179731994867325,
+      "step": 6530
+    },
+    {
+      "epoch": 2.253618194348725,
+      "grad_norm": 2.0042881965637207,
+      "learning_rate": 5.310403860943089e-09,
+      "logits/chosen": -2.9160268306732178,
+      "logits/rejected": -2.8954906463623047,
+      "logps/chosen": -61.35206985473633,
+      "logps/rejected": -62.508628845214844,
+      "loss": 0.6808,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07269518077373505,
+      "rewards/margins": 0.026623418554663658,
+      "rewards/rejected": -0.09931859374046326,
+      "step": 6540
+    },
+    {
+      "epoch": 2.257064093728463,
+      "grad_norm": 1.8219677209854126,
+      "learning_rate": 5.264569302794608e-09,
+      "logits/chosen": -3.055886745452881,
+      "logits/rejected": -3.022547483444214,
+      "logps/chosen": -61.92055130004883,
+      "logps/rejected": -62.03279495239258,
+      "loss": 0.6797,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.07779103517532349,
+      "rewards/margins": 0.028883641585707664,
+      "rewards/rejected": -0.1066746860742569,
+      "step": 6550
+    },
+    {
+      "epoch": 2.2605099931082013,
+      "grad_norm": 1.9836839437484741,
+      "learning_rate": 5.218891266985945e-09,
+      "logits/chosen": -2.9859132766723633,
+      "logits/rejected": -2.9672086238861084,
+      "logps/chosen": -62.449256896972656,
+      "logps/rejected": -63.257713317871094,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": -0.0827869400382042,
+      "rewards/margins": 0.02680940553545952,
+      "rewards/rejected": -0.10959634929895401,
+      "step": 6560
+    },
+    {
+      "epoch": 2.2639558924879393,
+      "grad_norm": 2.1861941814422607,
+      "learning_rate": 5.173370487910197e-09,
+      "logits/chosen": -2.987881898880005,
+      "logits/rejected": -2.9695162773132324,
+      "logps/chosen": -61.28582000732422,
+      "logps/rejected": -67.1212387084961,
+      "loss": 0.6779,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.07468996196985245,
+      "rewards/margins": 0.03266452997922897,
+      "rewards/rejected": -0.10735448449850082,
+      "step": 6570
+    },
+    {
+      "epoch": 2.2674017918676777,
+      "grad_norm": 2.1709415912628174,
+      "learning_rate": 5.128007697432139e-09,
+      "logits/chosen": -2.932931423187256,
+      "logits/rejected": -2.9049956798553467,
+      "logps/chosen": -63.754661560058594,
+      "logps/rejected": -65.15232849121094,
+      "loss": 0.6767,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.07896430790424347,
+      "rewards/margins": 0.035401780158281326,
+      "rewards/rejected": -0.1143660917878151,
+      "step": 6580
+    },
+    {
+      "epoch": 2.2708476912474156,
+      "grad_norm": 2.040640354156494,
+      "learning_rate": 5.0828036248764785e-09,
+      "logits/chosen": -2.972365140914917,
+      "logits/rejected": -2.9552135467529297,
+      "logps/chosen": -61.4955940246582,
+      "logps/rejected": -62.395652770996094,
+      "loss": 0.681,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.07918331772089005,
+      "rewards/margins": 0.026166219264268875,
+      "rewards/rejected": -0.10534952580928802,
+      "step": 6590
+    },
+    {
+      "epoch": 2.2742935906271535,
+      "grad_norm": 2.039576530456543,
+      "learning_rate": 5.037758997016115e-09,
+      "logits/chosen": -2.972411632537842,
+      "logits/rejected": -2.9473156929016113,
+      "logps/chosen": -62.5654182434082,
+      "logps/rejected": -62.522705078125,
+      "loss": 0.6806,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.07281356304883957,
+      "rewards/margins": 0.027093613520264626,
+      "rewards/rejected": -0.09990718960762024,
+      "step": 6600
+    },
+    {
+      "epoch": 2.2742935906271535,
+      "eval_logits/chosen": -3.068153142929077,
+      "eval_logits/rejected": -3.062441349029541,
+      "eval_logps/chosen": -63.196388244628906,
+      "eval_logps/rejected": -69.02894592285156,
+      "eval_loss": 0.686967670917511,
+      "eval_rewards/accuracies": 0.5924721360206604,
+      "eval_rewards/chosen": -0.044844891875982285,
+      "eval_rewards/margins": 0.01364336907863617,
+      "eval_rewards/rejected": -0.05848825350403786,
+      "eval_runtime": 383.3362,
+      "eval_samples_per_second": 11.228,
+      "eval_steps_per_second": 1.403,
+      "step": 6600
+    },
+    {
+      "epoch": 2.277739490006892,
+      "grad_norm": 2.113435745239258,
+      "learning_rate": 4.99287453806046e-09,
+      "logits/chosen": -2.9961793422698975,
+      "logits/rejected": -2.9841551780700684,
+      "logps/chosen": -61.475624084472656,
+      "logps/rejected": -64.68501281738281,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.07913829386234283,
+      "rewards/margins": 0.022053148597478867,
+      "rewards/rejected": -0.101191446185112,
+      "step": 6610
+    },
+    {
+      "epoch": 2.28118538938663,
+      "grad_norm": 2.0852317810058594,
+      "learning_rate": 4.948150969643797e-09,
+      "logits/chosen": -2.9722280502319336,
+      "logits/rejected": -2.9506795406341553,
+      "logps/chosen": -58.52437210083008,
+      "logps/rejected": -63.79155349731445,
+      "loss": 0.6776,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.07406620681285858,
+      "rewards/margins": 0.033300697803497314,
+      "rewards/rejected": -0.1073668971657753,
+      "step": 6620
+    },
+    {
+      "epoch": 2.2846312887663682,
+      "grad_norm": 1.8827736377716064,
+      "learning_rate": 4.903589010813672e-09,
+      "logits/chosen": -3.0136234760284424,
+      "logits/rejected": -3.0090816020965576,
+      "logps/chosen": -58.59986114501953,
+      "logps/rejected": -65.20610046386719,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.08743450790643692,
+      "rewards/margins": 0.021747268736362457,
+      "rewards/rejected": -0.10918178409337997,
+      "step": 6630
+    },
+    {
+      "epoch": 2.288077188146106,
+      "grad_norm": 1.9654409885406494,
+      "learning_rate": 4.859189378019342e-09,
+      "logits/chosen": -2.931558132171631,
+      "logits/rejected": -2.913980007171631,
+      "logps/chosen": -61.3321647644043,
+      "logps/rejected": -63.54027557373047,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.07209762930870056,
+      "rewards/margins": 0.0330507718026638,
+      "rewards/rejected": -0.10514841228723526,
+      "step": 6640
+    },
+    {
+      "epoch": 2.291523087525844,
+      "grad_norm": 2.056849718093872,
+      "learning_rate": 4.814952785100247e-09,
+      "logits/chosen": -2.9754951000213623,
+      "logits/rejected": -2.9572343826293945,
+      "logps/chosen": -61.062255859375,
+      "logps/rejected": -63.30437469482422,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08561302721500397,
+      "rewards/margins": 0.024649715051054955,
+      "rewards/rejected": -0.11026275157928467,
+      "step": 6650
+    },
+    {
+      "epoch": 2.2949689869055825,
+      "grad_norm": 2.063321828842163,
+      "learning_rate": 4.770879943274538e-09,
+      "logits/chosen": -2.9996743202209473,
+      "logits/rejected": -2.9703822135925293,
+      "logps/chosen": -64.98420715332031,
+      "logps/rejected": -61.96540069580078,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.08479741960763931,
+      "rewards/margins": 0.025747036561369896,
+      "rewards/rejected": -0.11054445803165436,
+      "step": 6660
+    },
+    {
+      "epoch": 2.2984148862853204,
+      "grad_norm": 2.1494953632354736,
+      "learning_rate": 4.726971561127645e-09,
+      "logits/chosen": -2.9827725887298584,
+      "logits/rejected": -2.9649369716644287,
+      "logps/chosen": -63.42902374267578,
+      "logps/rejected": -62.469764709472656,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.08089326322078705,
+      "rewards/margins": 0.023197535425424576,
+      "rewards/rejected": -0.10409079492092133,
+      "step": 6670
+    },
+    {
+      "epoch": 2.301860785665059,
+      "grad_norm": 1.8773868083953857,
+      "learning_rate": 4.683228344600876e-09,
+      "logits/chosen": -2.9862887859344482,
+      "logits/rejected": -2.9579925537109375,
+      "logps/chosen": -65.40532684326172,
+      "logps/rejected": -63.72175979614258,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.08292077481746674,
+      "rewards/margins": 0.028120696544647217,
+      "rewards/rejected": -0.11104147136211395,
+      "step": 6680
+    },
+    {
+      "epoch": 2.3053066850447967,
+      "grad_norm": 2.011620044708252,
+      "learning_rate": 4.6396509969800795e-09,
+      "logits/chosen": -2.9238243103027344,
+      "logits/rejected": -2.8928542137145996,
+      "logps/chosen": -63.63115310668945,
+      "logps/rejected": -64.5469741821289,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.08068819344043732,
+      "rewards/margins": 0.03330891951918602,
+      "rewards/rejected": -0.11399711668491364,
+      "step": 6690
+    },
+    {
+      "epoch": 2.3087525844245347,
+      "grad_norm": 1.9596617221832275,
+      "learning_rate": 4.596240218884326e-09,
+      "logits/chosen": -2.9025719165802,
+      "logits/rejected": -2.8705971240997314,
+      "logps/chosen": -63.27996826171875,
+      "logps/rejected": -62.29181671142578,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.07224036753177643,
+      "rewards/margins": 0.03780435770750046,
+      "rewards/rejected": -0.11004471778869629,
+      "step": 6700
+    },
+    {
+      "epoch": 2.3087525844245347,
+      "eval_logits/chosen": -3.067509412765503,
+      "eval_logits/rejected": -3.061769723892212,
+      "eval_logps/chosen": -63.23828125,
+      "eval_logps/rejected": -69.08137512207031,
+      "eval_loss": 0.6869193911552429,
+      "eval_rewards/accuracies": 0.591775119304657,
+      "eval_rewards/chosen": -0.0452638640999794,
+      "eval_rewards/margins": 0.013748648576438427,
+      "eval_rewards/rejected": -0.0590125136077404,
+      "eval_runtime": 383.2176,
+      "eval_samples_per_second": 11.231,
+      "eval_steps_per_second": 1.404,
+      "step": 6700
+    },
+    {
+      "epoch": 2.312198483804273,
+      "grad_norm": 2.228132486343384,
+      "learning_rate": 4.552996708254646e-09,
+      "logits/chosen": -2.963857650756836,
+      "logits/rejected": -2.9511008262634277,
+      "logps/chosen": -62.77131271362305,
+      "logps/rejected": -65.06932067871094,
+      "loss": 0.6829,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08972633630037308,
+      "rewards/margins": 0.022793501615524292,
+      "rewards/rejected": -0.11251983791589737,
+      "step": 6710
+    },
+    {
+      "epoch": 2.315644383184011,
+      "grad_norm": 1.9115924835205078,
+      "learning_rate": 4.509921160342817e-09,
+      "logits/chosen": -2.979947566986084,
+      "logits/rejected": -2.96016001701355,
+      "logps/chosen": -61.1608772277832,
+      "logps/rejected": -64.14488983154297,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07296943664550781,
+      "rewards/margins": 0.024569574743509293,
+      "rewards/rejected": -0.0975390076637268,
+      "step": 6720
+    },
+    {
+      "epoch": 2.3190902825637494,
+      "grad_norm": 2.1054441928863525,
+      "learning_rate": 4.467014267700174e-09,
+      "logits/chosen": -2.9670557975769043,
+      "logits/rejected": -2.939519166946411,
+      "logps/chosen": -59.53334426879883,
+      "logps/rejected": -62.467994689941406,
+      "loss": 0.6767,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.07316529005765915,
+      "rewards/margins": 0.0351564958691597,
+      "rewards/rejected": -0.10832178592681885,
+      "step": 6730
+    },
+    {
+      "epoch": 2.3225361819434873,
+      "grad_norm": 2.2091872692108154,
+      "learning_rate": 4.424276720166488e-09,
+      "logits/chosen": -2.9352853298187256,
+      "logits/rejected": -2.901637315750122,
+      "logps/chosen": -61.73224639892578,
+      "logps/rejected": -62.09523391723633,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07789238542318344,
+      "rewards/margins": 0.03297952190041542,
+      "rewards/rejected": -0.11087191104888916,
+      "step": 6740
+    },
+    {
+      "epoch": 2.3259820813232253,
+      "grad_norm": 2.035029888153076,
+      "learning_rate": 4.381709204858848e-09,
+      "logits/chosen": -2.987893581390381,
+      "logits/rejected": -2.9595227241516113,
+      "logps/chosen": -63.224937438964844,
+      "logps/rejected": -64.77478790283203,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.08427334576845169,
+      "rewards/margins": 0.036116186529397964,
+      "rewards/rejected": -0.12038953602313995,
+      "step": 6750
+    },
+    {
+      "epoch": 2.3294279807029636,
+      "grad_norm": 1.9183896780014038,
+      "learning_rate": 4.339312406160667e-09,
+      "logits/chosen": -2.8806443214416504,
+      "logits/rejected": -2.8750879764556885,
+      "logps/chosen": -58.65558624267578,
+      "logps/rejected": -63.28642654418945,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08562487363815308,
+      "rewards/margins": 0.02060927078127861,
+      "rewards/rejected": -0.10623414814472198,
+      "step": 6760
+    },
+    {
+      "epoch": 2.3328738800827016,
+      "grad_norm": 1.9252411127090454,
+      "learning_rate": 4.297087005710607e-09,
+      "logits/chosen": -2.9280569553375244,
+      "logits/rejected": -2.901935577392578,
+      "logps/chosen": -63.981285095214844,
+      "logps/rejected": -62.1091194152832,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.08636753261089325,
+      "rewards/margins": 0.022963711991906166,
+      "rewards/rejected": -0.10933125019073486,
+      "step": 6770
+    },
+    {
+      "epoch": 2.3363197794624395,
+      "grad_norm": 2.021364450454712,
+      "learning_rate": 4.2550336823916895e-09,
+      "logits/chosen": -2.972935914993286,
+      "logits/rejected": -2.956766128540039,
+      "logps/chosen": -60.01947784423828,
+      "logps/rejected": -62.73969650268555,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.08624240756034851,
+      "rewards/margins": 0.02445306070148945,
+      "rewards/rejected": -0.11069546639919281,
+      "step": 6780
+    },
+    {
+      "epoch": 2.339765678842178,
+      "grad_norm": 2.0013866424560547,
+      "learning_rate": 4.213153112320334e-09,
+      "logits/chosen": -2.910156726837158,
+      "logits/rejected": -2.896698236465454,
+      "logps/chosen": -59.563758850097656,
+      "logps/rejected": -63.2475471496582,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.08298875391483307,
+      "rewards/margins": 0.02646462619304657,
+      "rewards/rejected": -0.10945338010787964,
+      "step": 6790
+    },
+    {
+      "epoch": 2.343211578221916,
+      "grad_norm": 2.240124225616455,
+      "learning_rate": 4.171445968835494e-09,
+      "logits/chosen": -2.918390989303589,
+      "logits/rejected": -2.910045862197876,
+      "logps/chosen": -64.03179931640625,
+      "logps/rejected": -66.36988830566406,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.08458518236875534,
+      "rewards/margins": 0.0233664158731699,
+      "rewards/rejected": -0.10795160382986069,
+      "step": 6800
+    },
+    {
+      "epoch": 2.343211578221916,
+      "eval_logits/chosen": -3.0669119358062744,
+      "eval_logits/rejected": -3.06115460395813,
+      "eval_logps/chosen": -63.263702392578125,
+      "eval_logps/rejected": -69.10945892333984,
+      "eval_loss": 0.686912477016449,
+      "eval_rewards/accuracies": 0.5961896181106567,
+      "eval_rewards/chosen": -0.045518096536397934,
+      "eval_rewards/margins": 0.013775307685136795,
+      "eval_rewards/rejected": -0.05929340422153473,
+      "eval_runtime": 382.9228,
+      "eval_samples_per_second": 11.24,
+      "eval_steps_per_second": 1.405,
+      "step": 6800
+    },
+    {
+      "epoch": 2.346657477601654,
+      "grad_norm": 2.048499345779419,
+      "learning_rate": 4.129912922487867e-09,
+      "logits/chosen": -3.048257350921631,
+      "logits/rejected": -3.016570568084717,
+      "logps/chosen": -61.76752853393555,
+      "logps/rejected": -62.9517707824707,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.07294553518295288,
+      "rewards/margins": 0.031889233738183975,
+      "rewards/rejected": -0.10483477264642715,
+      "step": 6810
+    },
+    {
+      "epoch": 2.350103376981392,
+      "grad_norm": 2.0057716369628906,
+      "learning_rate": 4.088554641029052e-09,
+      "logits/chosen": -2.9089341163635254,
+      "logits/rejected": -2.880011558532715,
+      "logps/chosen": -63.3249626159668,
+      "logps/rejected": -63.93933868408203,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.08543984591960907,
+      "rewards/margins": 0.0308440662920475,
+      "rewards/rejected": -0.11628390848636627,
+      "step": 6820
+    },
+    {
+      "epoch": 2.35354927636113,
+      "grad_norm": 1.9985370635986328,
+      "learning_rate": 4.0473717894008865e-09,
+      "logits/chosen": -2.9375593662261963,
+      "logits/rejected": -2.913762092590332,
+      "logps/chosen": -61.80808639526367,
+      "logps/rejected": -63.417236328125,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.08284831047058105,
+      "rewards/margins": 0.023359550163149834,
+      "rewards/rejected": -0.10620786994695663,
+      "step": 6830
+    },
+    {
+      "epoch": 2.3569951757408685,
+      "grad_norm": 1.9310389757156372,
+      "learning_rate": 4.006365029724677e-09,
+      "logits/chosen": -2.9983019828796387,
+      "logits/rejected": -2.9727163314819336,
+      "logps/chosen": -62.390655517578125,
+      "logps/rejected": -63.756996154785156,
+      "loss": 0.6779,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.07705055177211761,
+      "rewards/margins": 0.03315809369087219,
+      "rewards/rejected": -0.1102086529135704,
+      "step": 6840
+    },
+    {
+      "epoch": 2.3604410751206064,
+      "grad_norm": 2.0402777194976807,
+      "learning_rate": 3.965535021290637e-09,
+      "logits/chosen": -2.972381114959717,
+      "logits/rejected": -2.941338062286377,
+      "logps/chosen": -64.06748962402344,
+      "logps/rejected": -64.29527282714844,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.07651872932910919,
+      "rewards/margins": 0.026753172278404236,
+      "rewards/rejected": -0.10327188670635223,
+      "step": 6850
+    },
+    {
+      "epoch": 2.3638869745003444,
+      "grad_norm": 1.9945515394210815,
+      "learning_rate": 3.924882420547208e-09,
+      "logits/chosen": -2.913161516189575,
+      "logits/rejected": -2.87919282913208,
+      "logps/chosen": -61.37636184692383,
+      "logps/rejected": -64.03874206542969,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07428363710641861,
+      "rewards/margins": 0.03437066823244095,
+      "rewards/rejected": -0.10865430533885956,
+      "step": 6860
+    },
+    {
+      "epoch": 2.3673328738800827,
+      "grad_norm": 2.017606496810913,
+      "learning_rate": 3.884407881090556e-09,
+      "logits/chosen": -3.045126438140869,
+      "logits/rejected": -3.019214153289795,
+      "logps/chosen": -63.448951721191406,
+      "logps/rejected": -66.59571075439453,
+      "loss": 0.677,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.07240886986255646,
+      "rewards/margins": 0.03460530936717987,
+      "rewards/rejected": -0.10701417922973633,
+      "step": 6870
+    },
+    {
+      "epoch": 2.3707787732598207,
+      "grad_norm": 2.2688515186309814,
+      "learning_rate": 3.844112053654067e-09,
+      "logits/chosen": -3.030827045440674,
+      "logits/rejected": -3.002824306488037,
+      "logps/chosen": -63.06003952026367,
+      "logps/rejected": -63.54497146606445,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0774284154176712,
+      "rewards/margins": 0.031221244484186172,
+      "rewards/rejected": -0.10864964872598648,
+      "step": 6880
+    },
+    {
+      "epoch": 2.374224672639559,
+      "grad_norm": 2.087963104248047,
+      "learning_rate": 3.8039955860978285e-09,
+      "logits/chosen": -3.084402561187744,
+      "logits/rejected": -3.058399200439453,
+      "logps/chosen": -62.28232955932617,
+      "logps/rejected": -64.21376037597656,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.07491100579500198,
+      "rewards/margins": 0.035271693021059036,
+      "rewards/rejected": -0.11018270254135132,
+      "step": 6890
+    },
+    {
+      "epoch": 2.377670572019297,
+      "grad_norm": 2.0661580562591553,
+      "learning_rate": 3.764059123398292e-09,
+      "logits/chosen": -3.025024890899658,
+      "logits/rejected": -2.993600368499756,
+      "logps/chosen": -64.94926452636719,
+      "logps/rejected": -64.00762176513672,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08072112500667572,
+      "rewards/margins": 0.031116947531700134,
+      "rewards/rejected": -0.11183806508779526,
+      "step": 6900
+    },
+    {
+      "epoch": 2.377670572019297,
+      "eval_logits/chosen": -3.0664236545562744,
+      "eval_logits/rejected": -3.060697555541992,
+      "eval_logps/chosen": -63.30455017089844,
+      "eval_logps/rejected": -69.15803527832031,
+      "eval_loss": 0.6868799924850464,
+      "eval_rewards/accuracies": 0.589219331741333,
+      "eval_rewards/chosen": -0.04592651128768921,
+      "eval_rewards/margins": 0.013852635398507118,
+      "eval_rewards/rejected": -0.05977915599942207,
+      "eval_runtime": 383.1631,
+      "eval_samples_per_second": 11.233,
+      "eval_steps_per_second": 1.404,
+      "step": 6900
+    },
+    {
+      "epoch": 2.381116471399035,
+      "grad_norm": 2.208677291870117,
+      "learning_rate": 3.724303307637835e-09,
+      "logits/chosen": -2.9915390014648438,
+      "logits/rejected": -2.9654622077941895,
+      "logps/chosen": -63.62794876098633,
+      "logps/rejected": -64.45814514160156,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.07239101827144623,
+      "rewards/margins": 0.03430667519569397,
+      "rewards/rejected": -0.1066977009177208,
+      "step": 6910
+    },
+    {
+      "epoch": 2.3845623707787733,
+      "grad_norm": 1.9476802349090576,
+      "learning_rate": 3.684728777994479e-09,
+      "logits/chosen": -3.0261778831481934,
+      "logits/rejected": -3.00431227684021,
+      "logps/chosen": -63.495819091796875,
+      "logps/rejected": -63.14453125,
+      "loss": 0.6771,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.0762554481625557,
+      "rewards/margins": 0.034322015941143036,
+      "rewards/rejected": -0.11057744920253754,
+      "step": 6920
+    },
+    {
+      "epoch": 2.3880082701585112,
+      "grad_norm": 1.9374191761016846,
+      "learning_rate": 3.6453361707315948e-09,
+      "logits/chosen": -2.926621437072754,
+      "logits/rejected": -2.9021012783050537,
+      "logps/chosen": -61.45134353637695,
+      "logps/rejected": -62.521018981933594,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.08622835576534271,
+      "rewards/margins": 0.03004477359354496,
+      "rewards/rejected": -0.11627312749624252,
+      "step": 6930
+    },
+    {
+      "epoch": 2.3914541695382496,
+      "grad_norm": 2.0553388595581055,
+      "learning_rate": 3.6061261191876854e-09,
+      "logits/chosen": -2.9713051319122314,
+      "logits/rejected": -2.942218780517578,
+      "logps/chosen": -63.416351318359375,
+      "logps/rejected": -64.38191223144531,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08769957721233368,
+      "rewards/margins": 0.031032536178827286,
+      "rewards/rejected": -0.11873210966587067,
+      "step": 6940
+    },
+    {
+      "epoch": 2.3949000689179876,
+      "grad_norm": 1.9845832586288452,
+      "learning_rate": 3.5670992537661872e-09,
+      "logits/chosen": -2.9261555671691895,
+      "logits/rejected": -2.9042840003967285,
+      "logps/chosen": -63.12639617919922,
+      "logps/rejected": -63.07514572143555,
+      "loss": 0.6808,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": -0.08590146899223328,
+      "rewards/margins": 0.027376655489206314,
+      "rewards/rejected": -0.11327812820672989,
+      "step": 6950
+    },
+    {
+      "epoch": 2.3983459682977255,
+      "grad_norm": 2.095578908920288,
+      "learning_rate": 3.528256201925354e-09,
+      "logits/chosen": -2.9624876976013184,
+      "logits/rejected": -2.9294800758361816,
+      "logps/chosen": -65.71220397949219,
+      "logps/rejected": -63.20676803588867,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.07257644087076187,
+      "rewards/margins": 0.03603377565741539,
+      "rewards/rejected": -0.10861021280288696,
+      "step": 6960
+    },
+    {
+      "epoch": 2.401791867677464,
+      "grad_norm": 2.1703383922576904,
+      "learning_rate": 3.489597588168148e-09,
+      "logits/chosen": -2.924067497253418,
+      "logits/rejected": -2.903395652770996,
+      "logps/chosen": -59.467018127441406,
+      "logps/rejected": -64.2640151977539,
+      "loss": 0.6765,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.07120572030544281,
+      "rewards/margins": 0.03583453968167305,
+      "rewards/rejected": -0.10704026371240616,
+      "step": 6970
+    },
+    {
+      "epoch": 2.405237767057202,
+      "grad_norm": 1.9971457719802856,
+      "learning_rate": 3.4511240340322176e-09,
+      "logits/chosen": -2.908133029937744,
+      "logits/rejected": -2.8911354541778564,
+      "logps/chosen": -61.85535430908203,
+      "logps/rejected": -63.51459503173828,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.08105114102363586,
+      "rewards/margins": 0.027881663292646408,
+      "rewards/rejected": -0.10893280804157257,
+      "step": 6980
+    },
+    {
+      "epoch": 2.40868366643694,
+      "grad_norm": 2.2944717407226562,
+      "learning_rate": 3.4128361580798957e-09,
+      "logits/chosen": -2.946964740753174,
+      "logits/rejected": -2.9297404289245605,
+      "logps/chosen": -64.24407958984375,
+      "logps/rejected": -64.5073013305664,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0768330916762352,
+      "rewards/margins": 0.028220772743225098,
+      "rewards/rejected": -0.1050538644194603,
+      "step": 6990
+    },
+    {
+      "epoch": 2.412129565816678,
+      "grad_norm": 1.980916142463684,
+      "learning_rate": 3.3747345758882532e-09,
+      "logits/chosen": -2.918398380279541,
+      "logits/rejected": -2.894869565963745,
+      "logps/chosen": -62.280731201171875,
+      "logps/rejected": -64.48884582519531,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08212022483348846,
+      "rewards/margins": 0.029055576771497726,
+      "rewards/rejected": -0.1111757904291153,
+      "step": 7000
+    },
+    {
+      "epoch": 2.412129565816678,
+      "eval_logits/chosen": -3.065822124481201,
+      "eval_logits/rejected": -3.060060977935791,
+      "eval_logps/chosen": -63.339088439941406,
+      "eval_logps/rejected": -69.20113372802734,
+      "eval_loss": 0.6868377923965454,
+      "eval_rewards/accuracies": 0.5934014916419983,
+      "eval_rewards/chosen": -0.04627193510532379,
+      "eval_rewards/margins": 0.01393812894821167,
+      "eval_rewards/rejected": -0.06021006032824516,
+      "eval_runtime": 382.9747,
+      "eval_samples_per_second": 11.238,
+      "eval_steps_per_second": 1.405,
+      "step": 7000
+    },
+    {
+      "epoch": 2.415575465196416,
+      "grad_norm": 2.189199924468994,
+      "learning_rate": 3.336819900039209e-09,
+      "logits/chosen": -2.9231619834899902,
+      "logits/rejected": -2.9080989360809326,
+      "logps/chosen": -63.381988525390625,
+      "logps/rejected": -62.556739807128906,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.08425577729940414,
+      "rewards/margins": 0.020847072824835777,
+      "rewards/rejected": -0.10510285198688507,
+      "step": 7010
+    },
+    {
+      "epoch": 2.4190213645761545,
+      "grad_norm": 2.1160049438476562,
+      "learning_rate": 3.299092740109673e-09,
+      "logits/chosen": -2.9253087043762207,
+      "logits/rejected": -2.8908231258392334,
+      "logps/chosen": -63.116798400878906,
+      "logps/rejected": -61.04767990112305,
+      "loss": 0.6763,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.07819709181785583,
+      "rewards/margins": 0.03630400076508522,
+      "rewards/rejected": -0.11450110375881195,
+      "step": 7020
+    },
+    {
+      "epoch": 2.4224672639558924,
+      "grad_norm": 2.061826705932617,
+      "learning_rate": 3.2615537026617495e-09,
+      "logits/chosen": -2.998417377471924,
+      "logits/rejected": -2.9638619422912598,
+      "logps/chosen": -62.727256774902344,
+      "logps/rejected": -62.12153244018555,
+      "loss": 0.6774,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.07609546929597855,
+      "rewards/margins": 0.03368159011006355,
+      "rewards/rejected": -0.1097770556807518,
+      "step": 7030
+    },
+    {
+      "epoch": 2.425913163335631,
+      "grad_norm": 1.9902064800262451,
+      "learning_rate": 3.22420339123299e-09,
+      "logits/chosen": -2.9278411865234375,
+      "logits/rejected": -2.8994972705841064,
+      "logps/chosen": -64.94743347167969,
+      "logps/rejected": -65.59359741210938,
+      "loss": 0.68,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.08330345153808594,
+      "rewards/margins": 0.028552964329719543,
+      "rewards/rejected": -0.11185641586780548,
+      "step": 7040
+    },
+    {
+      "epoch": 2.4293590627153687,
+      "grad_norm": 2.029334545135498,
+      "learning_rate": 3.187042406326682e-09,
+      "logits/chosen": -2.9725863933563232,
+      "logits/rejected": -2.9366915225982666,
+      "logps/chosen": -64.47479248046875,
+      "logps/rejected": -63.95454025268555,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.07281167060136795,
+      "rewards/margins": 0.03416246920824051,
+      "rewards/rejected": -0.10697413980960846,
+      "step": 7050
+    },
+    {
+      "epoch": 2.4328049620951067,
+      "grad_norm": 2.0095479488372803,
+      "learning_rate": 3.1500713454021952e-09,
+      "logits/chosen": -2.968057155609131,
+      "logits/rejected": -2.9494175910949707,
+      "logps/chosen": -58.92118453979492,
+      "logps/rejected": -62.62757110595703,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08240500092506409,
+      "rewards/margins": 0.024019746109843254,
+      "rewards/rejected": -0.10642474889755249,
+      "step": 7060
+    },
+    {
+      "epoch": 2.436250861474845,
+      "grad_norm": 1.9209061861038208,
+      "learning_rate": 3.1132908028653813e-09,
+      "logits/chosen": -3.0238218307495117,
+      "logits/rejected": -3.0023298263549805,
+      "logps/chosen": -62.1815071105957,
+      "logps/rejected": -62.867095947265625,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.08772210031747818,
+      "rewards/margins": 0.0245465487241745,
+      "rewards/rejected": -0.11226864904165268,
+      "step": 7070
+    },
+    {
+      "epoch": 2.439696760854583,
+      "grad_norm": 2.2448008060455322,
+      "learning_rate": 3.0767013700590115e-09,
+      "logits/chosen": -2.9852852821350098,
+      "logits/rejected": -2.9711408615112305,
+      "logps/chosen": -61.419715881347656,
+      "logps/rejected": -66.57906341552734,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07789698988199234,
+      "rewards/margins": 0.02713080681860447,
+      "rewards/rejected": -0.10502779483795166,
+      "step": 7080
+    },
+    {
+      "epoch": 2.4431426602343214,
+      "grad_norm": 2.1201388835906982,
+      "learning_rate": 3.0403036352532733e-09,
+      "logits/chosen": -2.915024757385254,
+      "logits/rejected": -2.8817262649536133,
+      "logps/chosen": -64.65522003173828,
+      "logps/rejected": -63.5480842590332,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.08138259500265121,
+      "rewards/margins": 0.029240155592560768,
+      "rewards/rejected": -0.11062274128198624,
+      "step": 7090
+    },
+    {
+      "epoch": 2.4465885596140593,
+      "grad_norm": 2.0068109035491943,
+      "learning_rate": 3.0040981836363097e-09,
+      "logits/chosen": -2.9325790405273438,
+      "logits/rejected": -2.9108712673187256,
+      "logps/chosen": -63.9586296081543,
+      "logps/rejected": -65.59080505371094,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07711119204759598,
+      "rewards/margins": 0.03638283163309097,
+      "rewards/rejected": -0.11349401623010635,
+      "step": 7100
+    },
+    {
+      "epoch": 2.4465885596140593,
+      "eval_logits/chosen": -3.0655503273010254,
+      "eval_logits/rejected": -3.059835433959961,
+      "eval_logps/chosen": -63.369930267333984,
+      "eval_logps/rejected": -69.24136352539062,
+      "eval_loss": 0.6867955923080444,
+      "eval_rewards/accuracies": 0.5936338305473328,
+      "eval_rewards/chosen": -0.04658038541674614,
+      "eval_rewards/margins": 0.01403201837092638,
+      "eval_rewards/rejected": -0.060612406581640244,
+      "eval_runtime": 382.8664,
+      "eval_samples_per_second": 11.242,
+      "eval_steps_per_second": 1.405,
+      "step": 7100
+    },
+    {
+      "epoch": 2.4500344589937972,
+      "grad_norm": 1.9566879272460938,
+      "learning_rate": 2.9680855973048136e-09,
+      "logits/chosen": -3.0345640182495117,
+      "logits/rejected": -3.0166900157928467,
+      "logps/chosen": -63.97553634643555,
+      "logps/rejected": -64.29792785644531,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08425076305866241,
+      "rewards/margins": 0.021211406216025352,
+      "rewards/rejected": -0.10546215623617172,
+      "step": 7110
+    },
+    {
+      "epoch": 2.4534803583735356,
+      "grad_norm": 2.1206424236297607,
+      "learning_rate": 2.932266455254669e-09,
+      "logits/chosen": -2.90120005607605,
+      "logits/rejected": -2.8817355632781982,
+      "logps/chosen": -62.767974853515625,
+      "logps/rejected": -65.0909423828125,
+      "loss": 0.6753,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.07892992347478867,
+      "rewards/margins": 0.038298554718494415,
+      "rewards/rejected": -0.11722848564386368,
+      "step": 7120
+    },
+    {
+      "epoch": 2.4569262577532736,
+      "grad_norm": 2.0844945907592773,
+      "learning_rate": 2.896641333371625e-09,
+      "logits/chosen": -2.949631452560425,
+      "logits/rejected": -2.91741943359375,
+      "logps/chosen": -63.775367736816406,
+      "logps/rejected": -62.26091766357422,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07786668092012405,
+      "rewards/margins": 0.02903074584901333,
+      "rewards/rejected": -0.10689742863178253,
+      "step": 7130
+    },
+    {
+      "epoch": 2.460372157133012,
+      "grad_norm": 1.969530701637268,
+      "learning_rate": 2.8612108044220796e-09,
+      "logits/chosen": -2.9219441413879395,
+      "logits/rejected": -2.901432514190674,
+      "logps/chosen": -61.4847412109375,
+      "logps/rejected": -63.32045364379883,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08454372733831406,
+      "rewards/margins": 0.0282395388931036,
+      "rewards/rejected": -0.1127832755446434,
+      "step": 7140
+    },
+    {
+      "epoch": 2.46381805651275,
+      "grad_norm": 2.3570196628570557,
+      "learning_rate": 2.8259754380438243e-09,
+      "logits/chosen": -2.9652512073516846,
+      "logits/rejected": -2.945669412612915,
+      "logps/chosen": -64.569091796875,
+      "logps/rejected": -64.79557800292969,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.07439015060663223,
+      "rewards/margins": 0.03154607489705086,
+      "rewards/rejected": -0.1059362143278122,
+      "step": 7150
+    },
+    {
+      "epoch": 2.467263955892488,
+      "grad_norm": 1.9754143953323364,
+      "learning_rate": 2.7909358007369114e-09,
+      "logits/chosen": -2.903033971786499,
+      "logits/rejected": -2.876586437225342,
+      "logps/chosen": -62.22881317138672,
+      "logps/rejected": -64.0363540649414,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.07096528261899948,
+      "rewards/margins": 0.03174969553947449,
+      "rewards/rejected": -0.10271497070789337,
+      "step": 7160
+    },
+    {
+      "epoch": 2.470709855272226,
+      "grad_norm": 2.059311628341675,
+      "learning_rate": 2.75609245585454e-09,
+      "logits/chosen": -3.023206949234009,
+      "logits/rejected": -3.0140633583068848,
+      "logps/chosen": -63.181236267089844,
+      "logps/rejected": -66.89746856689453,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.08238411694765091,
+      "rewards/margins": 0.03434644639492035,
+      "rewards/rejected": -0.11673055589199066,
+      "step": 7170
+    },
+    {
+      "epoch": 2.474155754651964,
+      "grad_norm": 2.1164567470550537,
+      "learning_rate": 2.72144596359399e-09,
+      "logits/chosen": -2.964547872543335,
+      "logits/rejected": -2.9476771354675293,
+      "logps/chosen": -63.594024658203125,
+      "logps/rejected": -64.28129577636719,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.08373961597681046,
+      "rewards/margins": 0.023477869108319283,
+      "rewards/rejected": -0.10721747577190399,
+      "step": 7180
+    },
+    {
+      "epoch": 2.4776016540317025,
+      "grad_norm": 2.0862808227539062,
+      "learning_rate": 2.6869968809876434e-09,
+      "logits/chosen": -3.0178937911987305,
+      "logits/rejected": -2.9893698692321777,
+      "logps/chosen": -62.014610290527344,
+      "logps/rejected": -62.04815673828125,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08312245458364487,
+      "rewards/margins": 0.025412684306502342,
+      "rewards/rejected": -0.10853514820337296,
+      "step": 7190
+    },
+    {
+      "epoch": 2.4810475534114405,
+      "grad_norm": 2.2686166763305664,
+      "learning_rate": 2.6527457618939827e-09,
+      "logits/chosen": -2.9739935398101807,
+      "logits/rejected": -2.9472146034240723,
+      "logps/chosen": -62.2974853515625,
+      "logps/rejected": -61.510032653808594,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.07879314571619034,
+      "rewards/margins": 0.03228793293237686,
+      "rewards/rejected": -0.1110810786485672,
+      "step": 7200
+    },
+    {
+      "epoch": 2.4810475534114405,
+      "eval_logits/chosen": -3.065236806869507,
+      "eval_logits/rejected": -3.05948805809021,
+      "eval_logps/chosen": -63.41672134399414,
+      "eval_logps/rejected": -69.29266357421875,
+      "eval_loss": 0.6867810487747192,
+      "eval_rewards/accuracies": 0.591775119304657,
+      "eval_rewards/chosen": -0.04704828932881355,
+      "eval_rewards/margins": 0.014077108353376389,
+      "eval_rewards/rejected": -0.06112539768218994,
+      "eval_runtime": 382.9336,
+      "eval_samples_per_second": 11.24,
+      "eval_steps_per_second": 1.405,
+      "step": 7200
+    },
+    {
+      "epoch": 2.4844934527911784,
+      "grad_norm": 2.0169923305511475,
+      "learning_rate": 2.6186931569887447e-09,
+      "logits/chosen": -3.017768621444702,
+      "logits/rejected": -2.999447822570801,
+      "logps/chosen": -59.347633361816406,
+      "logps/rejected": -63.118629455566406,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.07927621901035309,
+      "rewards/margins": 0.026039402931928635,
+      "rewards/rejected": -0.10531560331583023,
+      "step": 7210
+    },
+    {
+      "epoch": 2.4879393521709168,
+      "grad_norm": 2.2294697761535645,
+      "learning_rate": 2.5848396137560022e-09,
+      "logits/chosen": -2.945204973220825,
+      "logits/rejected": -2.92164945602417,
+      "logps/chosen": -61.3857307434082,
+      "logps/rejected": -63.99627685546875,
+      "loss": 0.6776,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.0800754502415657,
+      "rewards/margins": 0.03359379991889,
+      "rewards/rejected": -0.11366923898458481,
+      "step": 7220
+    },
+    {
+      "epoch": 2.4913852515506547,
+      "grad_norm": 2.0970354080200195,
+      "learning_rate": 2.5511856764794138e-09,
+      "logits/chosen": -2.942345142364502,
+      "logits/rejected": -2.924081325531006,
+      "logps/chosen": -62.87592697143555,
+      "logps/rejected": -63.538230895996094,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08122959733009338,
+      "rewards/margins": 0.026027921587228775,
+      "rewards/rejected": -0.10725750774145126,
+      "step": 7230
+    },
+    {
+      "epoch": 2.4948311509303926,
+      "grad_norm": 2.071495771408081,
+      "learning_rate": 2.517731886233459e-09,
+      "logits/chosen": -3.036172389984131,
+      "logits/rejected": -3.015839099884033,
+      "logps/chosen": -63.566673278808594,
+      "logps/rejected": -63.90369415283203,
+      "loss": 0.6753,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.0739656537771225,
+      "rewards/margins": 0.03801025450229645,
+      "rewards/rejected": -0.11197590827941895,
+      "step": 7240
+    },
+    {
+      "epoch": 2.498277050310131,
+      "grad_norm": 1.9486442804336548,
+      "learning_rate": 2.484478780874709e-09,
+      "logits/chosen": -2.950180768966675,
+      "logits/rejected": -2.920321226119995,
+      "logps/chosen": -61.33951950073242,
+      "logps/rejected": -64.16856384277344,
+      "loss": 0.6753,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": -0.07595699280500412,
+      "rewards/margins": 0.03804602101445198,
+      "rewards/rejected": -0.1140030175447464,
+      "step": 7250
+    },
+    {
+      "epoch": 2.501722949689869,
+      "grad_norm": 2.0358164310455322,
+      "learning_rate": 2.4514268950332345e-09,
+      "logits/chosen": -2.9881210327148438,
+      "logits/rejected": -2.959876298904419,
+      "logps/chosen": -63.28278732299805,
+      "logps/rejected": -64.56076049804688,
+      "loss": 0.6774,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07716509699821472,
+      "rewards/margins": 0.0340382345020771,
+      "rewards/rejected": -0.11120332777500153,
+      "step": 7260
+    },
+    {
+      "epoch": 2.505168849069607,
+      "grad_norm": 2.1807868480682373,
+      "learning_rate": 2.418576760103951e-09,
+      "logits/chosen": -2.9753646850585938,
+      "logits/rejected": -2.953519821166992,
+      "logps/chosen": -63.667259216308594,
+      "logps/rejected": -65.01050567626953,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.07849963754415512,
+      "rewards/margins": 0.033728159964084625,
+      "rewards/rejected": -0.11222779750823975,
+      "step": 7270
+    },
+    {
+      "epoch": 2.5086147484493453,
+      "grad_norm": 1.887880802154541,
+      "learning_rate": 2.3859289042381326e-09,
+      "logits/chosen": -2.9476847648620605,
+      "logits/rejected": -2.9179346561431885,
+      "logps/chosen": -62.416053771972656,
+      "logps/rejected": -65.17350769042969,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.07551194727420807,
+      "rewards/margins": 0.03647969290614128,
+      "rewards/rejected": -0.11199164390563965,
+      "step": 7280
+    },
+    {
+      "epoch": 2.5120606478290832,
+      "grad_norm": 2.0514917373657227,
+      "learning_rate": 2.3534838523348654e-09,
+      "logits/chosen": -2.915740489959717,
+      "logits/rejected": -2.910400390625,
+      "logps/chosen": -59.8321533203125,
+      "logps/rejected": -63.657310485839844,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.08930571377277374,
+      "rewards/margins": 0.02317456342279911,
+      "rewards/rejected": -0.1124802678823471,
+      "step": 7290
+    },
+    {
+      "epoch": 2.5155065472088216,
+      "grad_norm": 1.8079864978790283,
+      "learning_rate": 2.3212421260326523e-09,
+      "logits/chosen": -2.9351046085357666,
+      "logits/rejected": -2.9125595092773438,
+      "logps/chosen": -62.08086013793945,
+      "logps/rejected": -62.60234451293945,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.08487927913665771,
+      "rewards/margins": 0.02429775893688202,
+      "rewards/rejected": -0.10917703807353973,
+      "step": 7300
+    },
+    {
+      "epoch": 2.5155065472088216,
+      "eval_logits/chosen": -3.064688205718994,
+      "eval_logits/rejected": -3.0589301586151123,
+      "eval_logps/chosen": -63.43446350097656,
+      "eval_logps/rejected": -69.30500793457031,
+      "eval_loss": 0.6868080496788025,
+      "eval_rewards/accuracies": 0.5943308472633362,
+      "eval_rewards/chosen": -0.04722566530108452,
+      "eval_rewards/margins": 0.014023199677467346,
+      "eval_rewards/rejected": -0.06124887242913246,
+      "eval_runtime": 382.6456,
+      "eval_samples_per_second": 11.248,
+      "eval_steps_per_second": 1.406,
+      "step": 7300
+    },
+    {
+      "epoch": 2.5189524465885595,
+      "grad_norm": 1.9987940788269043,
+      "learning_rate": 2.2892042437009994e-09,
+      "logits/chosen": -2.9183449745178223,
+      "logits/rejected": -2.8967013359069824,
+      "logps/chosen": -61.731666564941406,
+      "logps/rejected": -62.60581588745117,
+      "loss": 0.6779,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.07953477650880814,
+      "rewards/margins": 0.03277719020843506,
+      "rewards/rejected": -0.11231197416782379,
+      "step": 7310
+    },
+    {
+      "epoch": 2.5223983459682975,
+      "grad_norm": 1.8800829648971558,
+      "learning_rate": 2.257370720432095e-09,
+      "logits/chosen": -2.88580584526062,
+      "logits/rejected": -2.8739523887634277,
+      "logps/chosen": -60.92256546020508,
+      "logps/rejected": -64.12057495117188,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08468299359083176,
+      "rewards/margins": 0.027800049632787704,
+      "rewards/rejected": -0.11248304694890976,
+      "step": 7320
+    },
+    {
+      "epoch": 2.525844245348036,
+      "grad_norm": 2.0560145378112793,
+      "learning_rate": 2.225742068032535e-09,
+      "logits/chosen": -2.9628348350524902,
+      "logits/rejected": -2.9304370880126953,
+      "logps/chosen": -62.15205001831055,
+      "logps/rejected": -62.72063064575195,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.08573584258556366,
+      "rewards/margins": 0.032484184950590134,
+      "rewards/rejected": -0.1182200089097023,
+      "step": 7330
+    },
+    {
+      "epoch": 2.529290144727774,
+      "grad_norm": 1.9690426588058472,
+      "learning_rate": 2.1943187950150642e-09,
+      "logits/chosen": -2.9246444702148438,
+      "logits/rejected": -2.906567096710205,
+      "logps/chosen": -57.478736877441406,
+      "logps/rejected": -62.276092529296875,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.08098022639751434,
+      "rewards/margins": 0.023182792589068413,
+      "rewards/rejected": -0.1041630282998085,
+      "step": 7340
+    },
+    {
+      "epoch": 2.532736044107512,
+      "grad_norm": 2.1474804878234863,
+      "learning_rate": 2.1631014065904293e-09,
+      "logits/chosen": -3.0194637775421143,
+      "logits/rejected": -3.0061986446380615,
+      "logps/chosen": -61.079734802246094,
+      "logps/rejected": -66.23876953125,
+      "loss": 0.6779,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08022590726613998,
+      "rewards/margins": 0.033261995762586594,
+      "rewards/rejected": -0.11348791420459747,
+      "step": 7350
+    },
+    {
+      "epoch": 2.53618194348725,
+      "grad_norm": 2.0754661560058594,
+      "learning_rate": 2.1320904046592536e-09,
+      "logits/chosen": -2.9539966583251953,
+      "logits/rejected": -2.931269884109497,
+      "logps/chosen": -63.61191940307617,
+      "logps/rejected": -64.77875518798828,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.08040911704301834,
+      "rewards/margins": 0.03234824165701866,
+      "rewards/rejected": -0.1127573698759079,
+      "step": 7360
+    },
+    {
+      "epoch": 2.539627842866988,
+      "grad_norm": 1.9922994375228882,
+      "learning_rate": 2.1012862878039497e-09,
+      "logits/chosen": -2.9693853855133057,
+      "logits/rejected": -2.957793712615967,
+      "logps/chosen": -62.152191162109375,
+      "logps/rejected": -64.0713882446289,
+      "loss": 0.6783,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.0801047831773758,
+      "rewards/margins": 0.03207314759492874,
+      "rewards/rejected": -0.11217793077230453,
+      "step": 7370
+    },
+    {
+      "epoch": 2.5430737422467264,
+      "grad_norm": 2.051330089569092,
+      "learning_rate": 2.0706895512807265e-09,
+      "logits/chosen": -2.977459669113159,
+      "logits/rejected": -2.9487881660461426,
+      "logps/chosen": -60.29890060424805,
+      "logps/rejected": -62.684722900390625,
+      "loss": 0.6752,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.0749792829155922,
+      "rewards/margins": 0.03817629814147949,
+      "rewards/rejected": -0.11315558105707169,
+      "step": 7380
+    },
+    {
+      "epoch": 2.5465196416264644,
+      "grad_norm": 1.9628040790557861,
+      "learning_rate": 2.040300687011606e-09,
+      "logits/chosen": -2.9646685123443604,
+      "logits/rejected": -2.953291177749634,
+      "logps/chosen": -60.7525749206543,
+      "logps/rejected": -63.151885986328125,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08055076003074646,
+      "rewards/margins": 0.02526286244392395,
+      "rewards/rejected": -0.10581362247467041,
+      "step": 7390
+    },
+    {
+      "epoch": 2.5499655410062028,
+      "grad_norm": 2.218071222305298,
+      "learning_rate": 2.0101201835765305e-09,
+      "logits/chosen": -2.9574568271636963,
+      "logits/rejected": -2.9440371990203857,
+      "logps/chosen": -63.35209274291992,
+      "logps/rejected": -66.01641845703125,
+      "loss": 0.6806,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.09040242433547974,
+      "rewards/margins": 0.02723805047571659,
+      "rewards/rejected": -0.11764047294855118,
+      "step": 7400
+    },
+    {
+      "epoch": 2.5499655410062028,
+      "eval_logits/chosen": -3.0645809173583984,
+      "eval_logits/rejected": -3.058812141418457,
+      "eval_logps/chosen": -63.443214416503906,
+      "eval_logps/rejected": -69.32136535644531,
+      "eval_loss": 0.6867730617523193,
+      "eval_rewards/accuracies": 0.5908457040786743,
+      "eval_rewards/chosen": -0.04731323570013046,
+      "eval_rewards/margins": 0.01409915927797556,
+      "eval_rewards/rejected": -0.061412401497364044,
+      "eval_runtime": 383.0791,
+      "eval_samples_per_second": 11.235,
+      "eval_steps_per_second": 1.404,
+      "step": 7400
+    },
+    {
+      "epoch": 2.5534114403859407,
+      "grad_norm": 2.053398370742798,
+      "learning_rate": 1.980148526205496e-09,
+      "logits/chosen": -3.0648350715637207,
+      "logits/rejected": -3.0261282920837402,
+      "logps/chosen": -64.26838684082031,
+      "logps/rejected": -62.773521423339844,
+      "loss": 0.6741,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.08164142072200775,
+      "rewards/margins": 0.04082329198718071,
+      "rewards/rejected": -0.12246473133563995,
+      "step": 7410
+    },
+    {
+      "epoch": 2.5568573397656786,
+      "grad_norm": 2.107619047164917,
+      "learning_rate": 1.9503861967707563e-09,
+      "logits/chosen": -2.995915174484253,
+      "logits/rejected": -2.969769239425659,
+      "logps/chosen": -63.47101593017578,
+      "logps/rejected": -65.07491302490234,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.0803145170211792,
+      "rewards/margins": 0.03210834413766861,
+      "rewards/rejected": -0.11242286115884781,
+      "step": 7420
+    },
+    {
+      "epoch": 2.560303239145417,
+      "grad_norm": 2.022606611251831,
+      "learning_rate": 1.9208336737790787e-09,
+      "logits/chosen": -2.924938201904297,
+      "logits/rejected": -2.903688907623291,
+      "logps/chosen": -61.8421516418457,
+      "logps/rejected": -64.5462875366211,
+      "loss": 0.6779,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.08056792616844177,
+      "rewards/margins": 0.03304043412208557,
+      "rewards/rejected": -0.11360837519168854,
+      "step": 7430
+    },
+    {
+      "epoch": 2.563749138525155,
+      "grad_norm": 2.0033159255981445,
+      "learning_rate": 1.891491432364041e-09,
+      "logits/chosen": -2.93757963180542,
+      "logits/rejected": -2.91202449798584,
+      "logps/chosen": -63.2408332824707,
+      "logps/rejected": -64.58965301513672,
+      "loss": 0.676,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07652803510427475,
+      "rewards/margins": 0.0367257185280323,
+      "rewards/rejected": -0.11325374990701675,
+      "step": 7440
+    },
+    {
+      "epoch": 2.5671950379048933,
+      "grad_norm": 2.055680751800537,
+      "learning_rate": 1.862359944278401e-09,
+      "logits/chosen": -2.8591468334198,
+      "logits/rejected": -2.834376573562622,
+      "logps/chosen": -63.55411911010742,
+      "logps/rejected": -65.2736587524414,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.08503410965204239,
+      "rewards/margins": 0.02370402216911316,
+      "rewards/rejected": -0.10873813927173615,
+      "step": 7450
+    },
+    {
+      "epoch": 2.5706409372846313,
+      "grad_norm": 2.0052928924560547,
+      "learning_rate": 1.8334396778865118e-09,
+      "logits/chosen": -2.9343700408935547,
+      "logits/rejected": -2.9222748279571533,
+      "logps/chosen": -59.9294548034668,
+      "logps/rejected": -65.30386352539062,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0823691338300705,
+      "rewards/margins": 0.024742256850004196,
+      "rewards/rejected": -0.10711139440536499,
+      "step": 7460
+    },
+    {
+      "epoch": 2.574086836664369,
+      "grad_norm": 2.022693634033203,
+      "learning_rate": 1.8047310981567853e-09,
+      "logits/chosen": -3.035606861114502,
+      "logits/rejected": -3.0171689987182617,
+      "logps/chosen": -60.47563934326172,
+      "logps/rejected": -62.827125549316406,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.07703608274459839,
+      "rewards/margins": 0.03745538368821144,
+      "rewards/rejected": -0.11449146270751953,
+      "step": 7470
+    },
+    {
+      "epoch": 2.5775327360441076,
+      "grad_norm": 2.1794679164886475,
+      "learning_rate": 1.7762346666542243e-09,
+      "logits/chosen": -2.9826889038085938,
+      "logits/rejected": -2.9500606060028076,
+      "logps/chosen": -63.264366149902344,
+      "logps/rejected": -63.730735778808594,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.0742504820227623,
+      "rewards/margins": 0.03649234026670456,
+      "rewards/rejected": -0.11074282228946686,
+      "step": 7480
+    },
+    {
+      "epoch": 2.5809786354238455,
+      "grad_norm": 2.1208388805389404,
+      "learning_rate": 1.747950841532987e-09,
+      "logits/chosen": -3.0055651664733887,
+      "logits/rejected": -2.9733738899230957,
+      "logps/chosen": -62.3234748840332,
+      "logps/rejected": -63.95361328125,
+      "loss": 0.68,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08307138830423355,
+      "rewards/margins": 0.02842291072010994,
+      "rewards/rejected": -0.1114942878484726,
+      "step": 7490
+    },
+    {
+      "epoch": 2.584424534803584,
+      "grad_norm": 2.228555679321289,
+      "learning_rate": 1.7198800775290484e-09,
+      "logits/chosen": -3.014490842819214,
+      "logits/rejected": -3.0031001567840576,
+      "logps/chosen": -62.1905517578125,
+      "logps/rejected": -64.10279846191406,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.0845569521188736,
+      "rewards/margins": 0.02359553799033165,
+      "rewards/rejected": -0.10815248638391495,
+      "step": 7500
+    },
+    {
+      "epoch": 2.584424534803584,
+      "eval_logits/chosen": -3.064666748046875,
+      "eval_logits/rejected": -3.058906078338623,
+      "eval_logps/chosen": -63.45846176147461,
+      "eval_logps/rejected": -69.34259033203125,
+      "eval_loss": 0.6867456436157227,
+      "eval_rewards/accuracies": 0.591775119304657,
+      "eval_rewards/chosen": -0.04746565595269203,
+      "eval_rewards/margins": 0.014158952049911022,
+      "eval_rewards/rejected": -0.06162460520863533,
+      "eval_runtime": 382.7755,
+      "eval_samples_per_second": 11.244,
+      "eval_steps_per_second": 1.406,
+      "step": 7500
+    },
+    {
+      "epoch": 2.587870434183322,
+      "grad_norm": 2.0514116287231445,
+      "learning_rate": 1.6920228259528612e-09,
+      "logits/chosen": -2.9792871475219727,
+      "logits/rejected": -2.9727113246917725,
+      "logps/chosen": -61.624717712402344,
+      "logps/rejected": -64.2152099609375,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08951710164546967,
+      "rewards/margins": 0.020899388939142227,
+      "rewards/rejected": -0.11041649430990219,
+      "step": 7510
+    },
+    {
+      "epoch": 2.59131633356306,
+      "grad_norm": 2.1249096393585205,
+      "learning_rate": 1.6643795346821154e-09,
+      "logits/chosen": -2.94403338432312,
+      "logits/rejected": -2.922790765762329,
+      "logps/chosen": -61.676170349121094,
+      "logps/rejected": -64.55792236328125,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.07843580842018127,
+      "rewards/margins": 0.031975157558918,
+      "rewards/rejected": -0.11041097342967987,
+      "step": 7520
+    },
+    {
+      "epoch": 2.594762232942798,
+      "grad_norm": 1.8587762117385864,
+      "learning_rate": 1.636950648154529e-09,
+      "logits/chosen": -2.9734344482421875,
+      "logits/rejected": -2.945465087890625,
+      "logps/chosen": -62.07781219482422,
+      "logps/rejected": -62.7835807800293,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07821150124073029,
+      "rewards/margins": 0.03252604603767395,
+      "rewards/rejected": -0.11073753982782364,
+      "step": 7530
+    },
+    {
+      "epoch": 2.598208132322536,
+      "grad_norm": 2.037692070007324,
+      "learning_rate": 1.6097366073607116e-09,
+      "logits/chosen": -3.0244956016540527,
+      "logits/rejected": -2.9907352924346924,
+      "logps/chosen": -62.63225173950195,
+      "logps/rejected": -63.36505889892578,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0753912702202797,
+      "rewards/margins": 0.03147123381495476,
+      "rewards/rejected": -0.10686250030994415,
+      "step": 7540
+    },
+    {
+      "epoch": 2.6016540317022745,
+      "grad_norm": 1.9440016746520996,
+      "learning_rate": 1.5827378498370696e-09,
+      "logits/chosen": -2.963219165802002,
+      "logits/rejected": -2.9418246746063232,
+      "logps/chosen": -61.4139404296875,
+      "logps/rejected": -63.058982849121094,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08572018891572952,
+      "rewards/margins": 0.024764664471149445,
+      "rewards/rejected": -0.11048485338687897,
+      "step": 7550
+    },
+    {
+      "epoch": 2.6050999310820124,
+      "grad_norm": 2.0554587841033936,
+      "learning_rate": 1.5559548096587594e-09,
+      "logits/chosen": -2.9521446228027344,
+      "logits/rejected": -2.933509588241577,
+      "logps/chosen": -64.29824829101562,
+      "logps/rejected": -64.84295654296875,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.08680449426174164,
+      "rewards/margins": 0.02102055959403515,
+      "rewards/rejected": -0.10782505571842194,
+      "step": 7560
+    },
+    {
+      "epoch": 2.6085458304617504,
+      "grad_norm": 2.0225491523742676,
+      "learning_rate": 1.5293879174327456e-09,
+      "logits/chosen": -2.925628900527954,
+      "logits/rejected": -2.9015748500823975,
+      "logps/chosen": -62.69477081298828,
+      "logps/rejected": -64.73983001708984,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.08378428965806961,
+      "rewards/margins": 0.033605050295591354,
+      "rewards/rejected": -0.11738934367895126,
+      "step": 7570
+    },
+    {
+      "epoch": 2.6119917298414888,
+      "grad_norm": 2.123588800430298,
+      "learning_rate": 1.5030376002908233e-09,
+      "logits/chosen": -3.017362594604492,
+      "logits/rejected": -2.995455265045166,
+      "logps/chosen": -61.738250732421875,
+      "logps/rejected": -65.12015533447266,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07836925238370895,
+      "rewards/margins": 0.03269009292125702,
+      "rewards/rejected": -0.11105935275554657,
+      "step": 7580
+    },
+    {
+      "epoch": 2.6154376292212267,
+      "grad_norm": 2.1551673412323,
+      "learning_rate": 1.4769042818828064e-09,
+      "logits/chosen": -2.956270694732666,
+      "logits/rejected": -2.949193239212036,
+      "logps/chosen": -62.92914581298828,
+      "logps/rejected": -65.65613555908203,
+      "loss": 0.683,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.08509718626737595,
+      "rewards/margins": 0.022821497172117233,
+      "rewards/rejected": -0.10791867971420288,
+      "step": 7590
+    },
+    {
+      "epoch": 2.618883528600965,
+      "grad_norm": 2.1350345611572266,
+      "learning_rate": 1.4509883823696756e-09,
+      "logits/chosen": -2.9547903537750244,
+      "logits/rejected": -2.9360547065734863,
+      "logps/chosen": -62.410552978515625,
+      "logps/rejected": -64.22431182861328,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.0829932764172554,
+      "rewards/margins": 0.03046898916363716,
+      "rewards/rejected": -0.11346225440502167,
+      "step": 7600
+    },
+    {
+      "epoch": 2.618883528600965,
+      "eval_logits/chosen": -3.0641889572143555,
+      "eval_logits/rejected": -3.0584418773651123,
+      "eval_logps/chosen": -63.47877883911133,
+      "eval_logps/rejected": -69.3577880859375,
+      "eval_loss": 0.6867708563804626,
+      "eval_rewards/accuracies": 0.5915427803993225,
+      "eval_rewards/chosen": -0.04766882583498955,
+      "eval_rewards/margins": 0.014107778668403625,
+      "eval_rewards/rejected": -0.06177660450339317,
+      "eval_runtime": 382.87,
+      "eval_samples_per_second": 11.241,
+      "eval_steps_per_second": 1.405,
+      "step": 7600
+    },
+    {
+      "epoch": 2.622329427980703,
+      "grad_norm": 1.970401644706726,
+      "learning_rate": 1.4252903184168379e-09,
+      "logits/chosen": -2.9142231941223145,
+      "logits/rejected": -2.8792691230773926,
+      "logps/chosen": -62.303497314453125,
+      "logps/rejected": -62.46787643432617,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.08329756557941437,
+      "rewards/margins": 0.03452523797750473,
+      "rewards/rejected": -0.1178228110074997,
+      "step": 7610
+    },
+    {
+      "epoch": 2.625775327360441,
+      "grad_norm": 2.143105983734131,
+      "learning_rate": 1.399810503187439e-09,
+      "logits/chosen": -2.91473650932312,
+      "logits/rejected": -2.891990900039673,
+      "logps/chosen": -65.17141723632812,
+      "logps/rejected": -65.6788330078125,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.08164992183446884,
+      "rewards/margins": 0.036141157150268555,
+      "rewards/rejected": -0.1177910789847374,
+      "step": 7620
+    },
+    {
+      "epoch": 2.6292212267401793,
+      "grad_norm": 2.2411837577819824,
+      "learning_rate": 1.374549346335691e-09,
+      "logits/chosen": -2.947598457336426,
+      "logits/rejected": -2.9243619441986084,
+      "logps/chosen": -61.01074981689453,
+      "logps/rejected": -63.00620651245117,
+      "loss": 0.6771,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.07530881464481354,
+      "rewards/margins": 0.03449029475450516,
+      "rewards/rejected": -0.1097991093993187,
+      "step": 7630
+    },
+    {
+      "epoch": 2.6326671261199173,
+      "grad_norm": 2.1466968059539795,
+      "learning_rate": 1.3495072540003298e-09,
+      "logits/chosen": -3.0357022285461426,
+      "logits/rejected": -3.004284381866455,
+      "logps/chosen": -62.49274826049805,
+      "logps/rejected": -65.13862609863281,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.0888562947511673,
+      "rewards/margins": 0.037823084741830826,
+      "rewards/rejected": -0.12667937576770782,
+      "step": 7640
+    },
+    {
+      "epoch": 2.6361130254996556,
+      "grad_norm": 1.8608691692352295,
+      "learning_rate": 1.3246846287980306e-09,
+      "logits/chosen": -2.9039101600646973,
+      "logits/rejected": -2.8715453147888184,
+      "logps/chosen": -61.6031608581543,
+      "logps/rejected": -62.659149169921875,
+      "loss": 0.6748,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08239149302244186,
+      "rewards/margins": 0.03951852023601532,
+      "rewards/rejected": -0.12191001325845718,
+      "step": 7650
+    },
+    {
+      "epoch": 2.6395589248793936,
+      "grad_norm": 2.105381488800049,
+      "learning_rate": 1.3000818698169923e-09,
+      "logits/chosen": -2.9945616722106934,
+      "logits/rejected": -2.974489450454712,
+      "logps/chosen": -60.8087043762207,
+      "logps/rejected": -64.46517181396484,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.08131098747253418,
+      "rewards/margins": 0.031762171536684036,
+      "rewards/rejected": -0.11307314783334732,
+      "step": 7660
+    },
+    {
+      "epoch": 2.6430048242591315,
+      "grad_norm": 2.1695470809936523,
+      "learning_rate": 1.2756993726104736e-09,
+      "logits/chosen": -2.934664249420166,
+      "logits/rejected": -2.9153003692626953,
+      "logps/chosen": -63.7085075378418,
+      "logps/rejected": -66.21272277832031,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.08343060314655304,
+      "rewards/margins": 0.028861120343208313,
+      "rewards/rejected": -0.11229170858860016,
+      "step": 7670
+    },
+    {
+      "epoch": 2.64645072363887,
+      "grad_norm": 2.1116952896118164,
+      "learning_rate": 1.2515375291904573e-09,
+      "logits/chosen": -2.902571439743042,
+      "logits/rejected": -2.893882989883423,
+      "logps/chosen": -61.41510772705078,
+      "logps/rejected": -66.25421142578125,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.08394423127174377,
+      "rewards/margins": 0.027783885598182678,
+      "rewards/rejected": -0.11172810941934586,
+      "step": 7680
+    },
+    {
+      "epoch": 2.649896623018608,
+      "grad_norm": 2.0916287899017334,
+      "learning_rate": 1.2275967280213539e-09,
+      "logits/chosen": -3.0038669109344482,
+      "logits/rejected": -2.9761009216308594,
+      "logps/chosen": -63.69536590576172,
+      "logps/rejected": -63.81220245361328,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.08537284284830093,
+      "rewards/margins": 0.032989516854286194,
+      "rewards/rejected": -0.11836235225200653,
+      "step": 7690
+    },
+    {
+      "epoch": 2.6533425223983462,
+      "grad_norm": 2.138741970062256,
+      "learning_rate": 1.203877354013727e-09,
+      "logits/chosen": -3.025852680206299,
+      "logits/rejected": -2.9949421882629395,
+      "logps/chosen": -65.5975341796875,
+      "logps/rejected": -65.45555114746094,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.08112451434135437,
+      "rewards/margins": 0.03549567982554436,
+      "rewards/rejected": -0.11662020534276962,
+      "step": 7700
+    },
+    {
+      "epoch": 2.6533425223983462,
+      "eval_logits/chosen": -3.0640110969543457,
+      "eval_logits/rejected": -3.058209180831909,
+      "eval_logps/chosen": -63.46165466308594,
+      "eval_logps/rejected": -69.36504364013672,
+      "eval_loss": 0.6866519451141357,
+      "eval_rewards/accuracies": 0.5945631861686707,
+      "eval_rewards/chosen": -0.047497596591711044,
+      "eval_rewards/margins": 0.014351638033986092,
+      "eval_rewards/rejected": -0.06184923276305199,
+      "eval_runtime": 383.1133,
+      "eval_samples_per_second": 11.234,
+      "eval_steps_per_second": 1.404,
+      "step": 7700
+    },
+    {
+      "epoch": 2.656788421778084,
+      "grad_norm": 2.070997476577759,
+      "learning_rate": 1.1803797885181443e-09,
+      "logits/chosen": -2.958496332168579,
+      "logits/rejected": -2.9278035163879395,
+      "logps/chosen": -64.34169006347656,
+      "logps/rejected": -64.16641998291016,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.08329670131206512,
+      "rewards/margins": 0.032533999532461166,
+      "rewards/rejected": -0.11583069711923599,
+      "step": 7710
+    },
+    {
+      "epoch": 2.660234321157822,
+      "grad_norm": 2.0432865619659424,
+      "learning_rate": 1.157104409319008e-09,
+      "logits/chosen": -3.085129737854004,
+      "logits/rejected": -3.0545971393585205,
+      "logps/chosen": -64.03402709960938,
+      "logps/rejected": -64.57947540283203,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.07701310515403748,
+      "rewards/margins": 0.03375517204403877,
+      "rewards/rejected": -0.11076828092336655,
+      "step": 7720
+    },
+    {
+      "epoch": 2.66368022053756,
+      "grad_norm": 2.195382833480835,
+      "learning_rate": 1.1340515906285041e-09,
+      "logits/chosen": -2.9089648723602295,
+      "logits/rejected": -2.891738176345825,
+      "logps/chosen": -63.75175094604492,
+      "logps/rejected": -66.0050277709961,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07452965527772903,
+      "rewards/margins": 0.03183392062783241,
+      "rewards/rejected": -0.10636357963085175,
+      "step": 7730
+    },
+    {
+      "epoch": 2.6671261199172984,
+      "grad_norm": 2.012563467025757,
+      "learning_rate": 1.1112217030805864e-09,
+      "logits/chosen": -3.001699447631836,
+      "logits/rejected": -2.9747254848480225,
+      "logps/chosen": -65.54142761230469,
+      "logps/rejected": -64.57428741455078,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.08146289736032486,
+      "rewards/margins": 0.03062131069600582,
+      "rewards/rejected": -0.11208420991897583,
+      "step": 7740
+    },
+    {
+      "epoch": 2.670572019297037,
+      "grad_norm": 1.9579441547393799,
+      "learning_rate": 1.0886151137250022e-09,
+      "logits/chosen": -2.979767084121704,
+      "logits/rejected": -2.9658560752868652,
+      "logps/chosen": -63.1358757019043,
+      "logps/rejected": -63.835227966308594,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.08868329972028732,
+      "rewards/margins": 0.018686974421143532,
+      "rewards/rejected": -0.1073702797293663,
+      "step": 7750
+    },
+    {
+      "epoch": 2.6740179186767747,
+      "grad_norm": 2.133165121078491,
+      "learning_rate": 1.0662321860214052e-09,
+      "logits/chosen": -2.8615498542785645,
+      "logits/rejected": -2.832658290863037,
+      "logps/chosen": -64.00244140625,
+      "logps/rejected": -63.15172576904297,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.07690651714801788,
+      "rewards/margins": 0.03960326313972473,
+      "rewards/rejected": -0.11650979518890381,
+      "step": 7760
+    },
+    {
+      "epoch": 2.6774638180565127,
+      "grad_norm": 2.018717050552368,
+      "learning_rate": 1.0440732798335061e-09,
+      "logits/chosen": -2.9071779251098633,
+      "logits/rejected": -2.88308048248291,
+      "logps/chosen": -61.178077697753906,
+      "logps/rejected": -63.992210388183594,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.08034110814332962,
+      "rewards/margins": 0.033206306397914886,
+      "rewards/rejected": -0.11354740709066391,
+      "step": 7770
+    },
+    {
+      "epoch": 2.6809097174362506,
+      "grad_norm": 2.0166475772857666,
+      "learning_rate": 1.0221387514232897e-09,
+      "logits/chosen": -3.0002598762512207,
+      "logits/rejected": -2.9807779788970947,
+      "logps/chosen": -61.92583084106445,
+      "logps/rejected": -65.00403594970703,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08779053390026093,
+      "rewards/margins": 0.030949950218200684,
+      "rewards/rejected": -0.11874048411846161,
+      "step": 7780
+    },
+    {
+      "epoch": 2.684355616815989,
+      "grad_norm": 2.1261041164398193,
+      "learning_rate": 1.0004289534452804e-09,
+      "logits/chosen": -2.9945363998413086,
+      "logits/rejected": -2.980532169342041,
+      "logps/chosen": -62.60563278198242,
+      "logps/rejected": -65.73075103759766,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.08552855253219604,
+      "rewards/margins": 0.028023535385727882,
+      "rewards/rejected": -0.11355209350585938,
+      "step": 7790
+    },
+    {
+      "epoch": 2.687801516195727,
+      "grad_norm": 2.0022125244140625,
+      "learning_rate": 9.789442349408817e-10,
+      "logits/chosen": -2.9611263275146484,
+      "logits/rejected": -2.9457383155822754,
+      "logps/chosen": -61.539154052734375,
+      "logps/rejected": -63.8754997253418,
+      "loss": 0.6808,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.08337543904781342,
+      "rewards/margins": 0.027251053601503372,
+      "rewards/rejected": -0.11062649637460709,
+      "step": 7800
+    },
+    {
+      "epoch": 2.687801516195727,
+      "eval_logits/chosen": -3.0642244815826416,
+      "eval_logits/rejected": -3.0584356784820557,
+      "eval_logps/chosen": -63.48628616333008,
+      "eval_logps/rejected": -69.37122344970703,
+      "eval_loss": 0.6867465972900391,
+      "eval_rewards/accuracies": 0.591775119304657,
+      "eval_rewards/chosen": -0.04774390161037445,
+      "eval_rewards/margins": 0.014167110435664654,
+      "eval_rewards/rejected": -0.06191101297736168,
+      "eval_runtime": 383.0385,
+      "eval_samples_per_second": 11.236,
+      "eval_steps_per_second": 1.405,
+      "step": 7800
+    },
+    {
+      "epoch": 2.6912474155754653,
+      "grad_norm": 1.9171476364135742,
+      "learning_rate": 9.576849413327576e-10,
+      "logits/chosen": -2.9840142726898193,
+      "logits/rejected": -2.9751133918762207,
+      "logps/chosen": -62.23347091674805,
+      "logps/rejected": -65.10118103027344,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.07706721127033234,
+      "rewards/margins": 0.025799637660384178,
+      "rewards/rejected": -0.10286685079336166,
+      "step": 7810
+    },
+    {
+      "epoch": 2.6946933149552033,
+      "grad_norm": 1.8835673332214355,
+      "learning_rate": 9.36651414419281e-10,
+      "logits/chosen": -2.962472438812256,
+      "logits/rejected": -2.932155132293701,
+      "logps/chosen": -62.508522033691406,
+      "logps/rejected": -63.98848342895508,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.08205662667751312,
+      "rewards/margins": 0.03977404534816742,
+      "rewards/rejected": -0.12183066457509995,
+      "step": 7820
+    },
+    {
+      "epoch": 2.698139214334941,
+      "grad_norm": 2.1322543621063232,
+      "learning_rate": 9.158439923690403e-10,
+      "logits/chosen": -3.0203945636749268,
+      "logits/rejected": -3.0093464851379395,
+      "logps/chosen": -60.87786102294922,
+      "logps/rejected": -64.45945739746094,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.07882450520992279,
+      "rewards/margins": 0.028756093233823776,
+      "rewards/rejected": -0.10758061707019806,
+      "step": 7830
+    },
+    {
+      "epoch": 2.7015851137146796,
+      "grad_norm": 2.248992681503296,
+      "learning_rate": 8.952630097154012e-10,
+      "logits/chosen": -2.8986332416534424,
+      "logits/rejected": -2.8711166381835938,
+      "logps/chosen": -63.5044059753418,
+      "logps/rejected": -65.5551528930664,
+      "loss": 0.6744,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.08697225153446198,
+      "rewards/margins": 0.039866041392087936,
+      "rewards/rejected": -0.12683828175067902,
+      "step": 7840
+    },
+    {
+      "epoch": 2.7050310130944175,
+      "grad_norm": 2.052673816680908,
+      "learning_rate": 8.749087973511221e-10,
+      "logits/chosen": -2.9817519187927246,
+      "logits/rejected": -2.9646289348602295,
+      "logps/chosen": -60.82890701293945,
+      "logps/rejected": -64.52009582519531,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08583492785692215,
+      "rewards/margins": 0.02543085254728794,
+      "rewards/rejected": -0.11126577854156494,
+      "step": 7850
+    },
+    {
+      "epoch": 2.708476912474156,
+      "grad_norm": 2.117123603820801,
+      "learning_rate": 8.547816825230491e-10,
+      "logits/chosen": -2.973619222640991,
+      "logits/rejected": -2.95751690864563,
+      "logps/chosen": -63.94077682495117,
+      "logps/rejected": -65.62185668945312,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.08787936717271805,
+      "rewards/margins": 0.026294738054275513,
+      "rewards/rejected": -0.11417410522699356,
+      "step": 7860
+    },
+    {
+      "epoch": 2.711922811853894,
+      "grad_norm": 2.064377546310425,
+      "learning_rate": 8.348819888268376e-10,
+      "logits/chosen": -3.037029981613159,
+      "logits/rejected": -3.0108654499053955,
+      "logps/chosen": -60.986122131347656,
+      "logps/rejected": -62.56230545043945,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.0823802649974823,
+      "rewards/margins": 0.02913055382668972,
+      "rewards/rejected": -0.11151081323623657,
+      "step": 7870
+    },
+    {
+      "epoch": 2.7153687112336318,
+      "grad_norm": 1.9418176412582397,
+      "learning_rate": 8.152100362017627e-10,
+      "logits/chosen": -2.926182508468628,
+      "logits/rejected": -2.911961317062378,
+      "logps/chosen": -60.71613693237305,
+      "logps/rejected": -64.31734466552734,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.09157772362232208,
+      "rewards/margins": 0.023375337943434715,
+      "rewards/rejected": -0.11495306342840195,
+      "step": 7880
+    },
+    {
+      "epoch": 2.71881461061337,
+      "grad_norm": 2.1349337100982666,
+      "learning_rate": 7.957661409255667e-10,
+      "logits/chosen": -3.0281596183776855,
+      "logits/rejected": -3.0105690956115723,
+      "logps/chosen": -61.37428665161133,
+      "logps/rejected": -63.942405700683594,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.08294235169887543,
+      "rewards/margins": 0.02828875742852688,
+      "rewards/rejected": -0.11123111099004745,
+      "step": 7890
+    },
+    {
+      "epoch": 2.722260509993108,
+      "grad_norm": 2.0175845623016357,
+      "learning_rate": 7.765506156093782e-10,
+      "logits/chosen": -2.9540534019470215,
+      "logits/rejected": -2.921752452850342,
+      "logps/chosen": -63.73915481567383,
+      "logps/rejected": -62.49995803833008,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07595163583755493,
+      "rewards/margins": 0.03223598748445511,
+      "rewards/rejected": -0.10818763077259064,
+      "step": 7900
+    },
+    {
+      "epoch": 2.722260509993108,
+      "eval_logits/chosen": -3.0638577938079834,
+      "eval_logits/rejected": -3.058053731918335,
+      "eval_logps/chosen": -63.490177154541016,
+      "eval_logps/rejected": -69.38743591308594,
+      "eval_loss": 0.6866849660873413,
+      "eval_rewards/accuracies": 0.5924721360206604,
+      "eval_rewards/chosen": -0.04778273403644562,
+      "eval_rewards/margins": 0.014290397986769676,
+      "eval_rewards/rejected": -0.06207313761115074,
+      "eval_runtime": 383.0898,
+      "eval_samples_per_second": 11.235,
+      "eval_steps_per_second": 1.404,
+      "step": 7900
+    },
+    {
+      "epoch": 2.7257064093728465,
+      "grad_norm": 1.9548548460006714,
+      "learning_rate": 7.575637691926862e-10,
+      "logits/chosen": -2.9527695178985596,
+      "logits/rejected": -2.9397573471069336,
+      "logps/chosen": -59.476173400878906,
+      "logps/rejected": -63.79050827026367,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.0926646888256073,
+      "rewards/margins": 0.024859271943569183,
+      "rewards/rejected": -0.11752395331859589,
+      "step": 7910
+    },
+    {
+      "epoch": 2.7291523087525844,
+      "grad_norm": 2.092623710632324,
+      "learning_rate": 7.388059069383739e-10,
+      "logits/chosen": -3.0297412872314453,
+      "logits/rejected": -3.0226123332977295,
+      "logps/chosen": -60.38361358642578,
+      "logps/rejected": -64.63072204589844,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08839724957942963,
+      "rewards/margins": 0.02506449446082115,
+      "rewards/rejected": -0.11346173286437988,
+      "step": 7920
+    },
+    {
+      "epoch": 2.7325982081323223,
+      "grad_norm": 2.1063919067382812,
+      "learning_rate": 7.202773304278081e-10,
+      "logits/chosen": -2.990959882736206,
+      "logits/rejected": -2.964447021484375,
+      "logps/chosen": -61.069091796875,
+      "logps/rejected": -63.280242919921875,
+      "loss": 0.6797,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.07870699465274811,
+      "rewards/margins": 0.02894139662384987,
+      "rewards/rejected": -0.10764838755130768,
+      "step": 7930
+    },
+    {
+      "epoch": 2.7360441075120607,
+      "grad_norm": 2.0567023754119873,
+      "learning_rate": 7.019783375559856e-10,
+      "logits/chosen": -2.9658522605895996,
+      "logits/rejected": -2.9435558319091797,
+      "logps/chosen": -62.06017303466797,
+      "logps/rejected": -63.92023849487305,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.08508308231830597,
+      "rewards/margins": 0.03409438952803612,
+      "rewards/rejected": -0.11917748302221298,
+      "step": 7940
+    },
+    {
+      "epoch": 2.7394900068917987,
+      "grad_norm": 2.0133965015411377,
+      "learning_rate": 6.839092225267611e-10,
+      "logits/chosen": -2.986905813217163,
+      "logits/rejected": -2.9454026222229004,
+      "logps/chosen": -63.72053146362305,
+      "logps/rejected": -62.32769775390625,
+      "loss": 0.671,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.07490957528352737,
+      "rewards/margins": 0.04676508903503418,
+      "rewards/rejected": -0.12167467176914215,
+      "step": 7950
+    },
+    {
+      "epoch": 2.742935906271537,
+      "grad_norm": 2.0102133750915527,
+      "learning_rate": 6.660702758480974e-10,
+      "logits/chosen": -2.91310453414917,
+      "logits/rejected": -2.894657850265503,
+      "logps/chosen": -61.11217498779297,
+      "logps/rejected": -63.186500549316406,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08336718380451202,
+      "rewards/margins": 0.02904461696743965,
+      "rewards/rejected": -0.11241181194782257,
+      "step": 7960
+    },
+    {
+      "epoch": 2.746381805651275,
+      "grad_norm": 2.2364230155944824,
+      "learning_rate": 6.484617843274053e-10,
+      "logits/chosen": -2.905691146850586,
+      "logits/rejected": -2.8785579204559326,
+      "logps/chosen": -64.57199096679688,
+      "logps/rejected": -65.28335571289062,
+      "loss": 0.6763,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.07388728111982346,
+      "rewards/margins": 0.03626793622970581,
+      "rewards/rejected": -0.11015522480010986,
+      "step": 7970
+    },
+    {
+      "epoch": 2.749827705031013,
+      "grad_norm": 1.956066608428955,
+      "learning_rate": 6.310840310669335e-10,
+      "logits/chosen": -3.01426362991333,
+      "logits/rejected": -2.9888076782226562,
+      "logps/chosen": -64.81721496582031,
+      "logps/rejected": -63.649147033691406,
+      "loss": 0.6795,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.07654238492250443,
+      "rewards/margins": 0.030035454779863358,
+      "rewards/rejected": -0.10657783597707748,
+      "step": 7980
+    },
+    {
+      "epoch": 2.7532736044107513,
+      "grad_norm": 2.2849111557006836,
+      "learning_rate": 6.139372954592059e-10,
+      "logits/chosen": -2.940294027328491,
+      "logits/rejected": -2.918808937072754,
+      "logps/chosen": -64.17678833007812,
+      "logps/rejected": -67.3226318359375,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.08292944729328156,
+      "rewards/margins": 0.033299706876277924,
+      "rewards/rejected": -0.11622915416955948,
+      "step": 7990
+    },
+    {
+      "epoch": 2.7567195037904892,
+      "grad_norm": 2.1075446605682373,
+      "learning_rate": 5.97021853182546e-10,
+      "logits/chosen": -3.022444248199463,
+      "logits/rejected": -3.008591890335083,
+      "logps/chosen": -62.58551025390625,
+      "logps/rejected": -65.11709594726562,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.08326215296983719,
+      "rewards/margins": 0.029697049409151077,
+      "rewards/rejected": -0.11295919120311737,
+      "step": 8000
+    },
+    {
+      "epoch": 2.7567195037904892,
+      "eval_logits/chosen": -3.063781976699829,
+      "eval_logits/rejected": -3.0579898357391357,
+      "eval_logps/chosen": -63.50349807739258,
+      "eval_logps/rejected": -69.39224243164062,
+      "eval_loss": 0.6867284178733826,
+      "eval_rewards/accuracies": 0.589684009552002,
+      "eval_rewards/chosen": -0.047916021198034286,
+      "eval_rewards/margins": 0.014205188490450382,
+      "eval_rewards/rejected": -0.062121208757162094,
+      "eval_runtime": 383.1479,
+      "eval_samples_per_second": 11.233,
+      "eval_steps_per_second": 1.404,
+      "step": 8000
+    },
+    {
+      "epoch": 2.7601654031702276,
+      "grad_norm": 2.167107582092285,
+      "learning_rate": 5.803379761966287e-10,
+      "logits/chosen": -3.0690481662750244,
+      "logits/rejected": -3.053863048553467,
+      "logps/chosen": -62.494476318359375,
+      "logps/rejected": -63.94672775268555,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.08322153985500336,
+      "rewards/margins": 0.032021500170230865,
+      "rewards/rejected": -0.11524303257465363,
+      "step": 8010
+    },
+    {
+      "epoch": 2.7636113025499656,
+      "grad_norm": 1.9335951805114746,
+      "learning_rate": 5.638859327381267e-10,
+      "logits/chosen": -3.032121419906616,
+      "logits/rejected": -3.016171932220459,
+      "logps/chosen": -61.2673225402832,
+      "logps/rejected": -63.562461853027344,
+      "loss": 0.6787,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07862520217895508,
+      "rewards/margins": 0.031616099178791046,
+      "rewards/rejected": -0.11024129390716553,
+      "step": 8020
+    },
+    {
+      "epoch": 2.7670572019297035,
+      "grad_norm": 2.129000425338745,
+      "learning_rate": 5.476659873163697e-10,
+      "logits/chosen": -2.981998920440674,
+      "logits/rejected": -2.9631576538085938,
+      "logps/chosen": -62.8355598449707,
+      "logps/rejected": -65.61088562011719,
+      "loss": 0.6774,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.07778668403625488,
+      "rewards/margins": 0.03394133970141411,
+      "rewards/rejected": -0.11172803491353989,
+      "step": 8030
+    },
+    {
+      "epoch": 2.770503101309442,
+      "grad_norm": 2.00679087638855,
+      "learning_rate": 5.316784007091219e-10,
+      "logits/chosen": -2.940821409225464,
+      "logits/rejected": -2.907235860824585,
+      "logps/chosen": -65.14414978027344,
+      "logps/rejected": -64.76019287109375,
+      "loss": 0.6753,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.08056654781103134,
+      "rewards/margins": 0.03819318488240242,
+      "rewards/rejected": -0.11875973641872406,
+      "step": 8040
+    },
+    {
+      "epoch": 2.77394900068918,
+      "grad_norm": 2.0966479778289795,
+      "learning_rate": 5.159234299583742e-10,
+      "logits/chosen": -2.983140468597412,
+      "logits/rejected": -2.9673497676849365,
+      "logps/chosen": -61.3448486328125,
+      "logps/rejected": -64.61131286621094,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08786801993846893,
+      "rewards/margins": 0.031481899321079254,
+      "rewards/rejected": -0.11934993416070938,
+      "step": 8050
+    },
+    {
+      "epoch": 2.777394900068918,
+      "grad_norm": 2.1074142456054688,
+      "learning_rate": 5.004013283662029e-10,
+      "logits/chosen": -2.897190809249878,
+      "logits/rejected": -2.8740172386169434,
+      "logps/chosen": -61.4494743347168,
+      "logps/rejected": -63.81614303588867,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.07603806257247925,
+      "rewards/margins": 0.028843050822615623,
+      "rewards/rejected": -0.10488110780715942,
+      "step": 8060
+    },
+    {
+      "epoch": 2.780840799448656,
+      "grad_norm": 1.9809563159942627,
+      "learning_rate": 4.851123454907269e-10,
+      "logits/chosen": -3.013249397277832,
+      "logits/rejected": -2.988501787185669,
+      "logps/chosen": -61.05573272705078,
+      "logps/rejected": -63.15620040893555,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08469357341527939,
+      "rewards/margins": 0.027358507737517357,
+      "rewards/rejected": -0.1120520830154419,
+      "step": 8070
+    },
+    {
+      "epoch": 2.784286698828394,
+      "grad_norm": 2.085109233856201,
+      "learning_rate": 4.700567271420536e-10,
+      "logits/chosen": -3.037532329559326,
+      "logits/rejected": -3.0124974250793457,
+      "logps/chosen": -62.166770935058594,
+      "logps/rejected": -64.86571502685547,
+      "loss": 0.6753,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": -0.0810675248503685,
+      "rewards/margins": 0.037991948425769806,
+      "rewards/rejected": -0.11905946582555771,
+      "step": 8080
+    },
+    {
+      "epoch": 2.7877325982081325,
+      "grad_norm": 1.8589122295379639,
+      "learning_rate": 4.552347153783681e-10,
+      "logits/chosen": -2.934864044189453,
+      "logits/rejected": -2.899914264678955,
+      "logps/chosen": -64.35227966308594,
+      "logps/rejected": -64.67687225341797,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0795358270406723,
+      "rewards/margins": 0.036931559443473816,
+      "rewards/rejected": -0.11646739393472672,
+      "step": 8090
+    },
+    {
+      "epoch": 2.7911784975878704,
+      "grad_norm": 2.1549365520477295,
+      "learning_rate": 4.4064654850201e-10,
+      "logits/chosen": -2.9288201332092285,
+      "logits/rejected": -2.8916220664978027,
+      "logps/chosen": -61.75331497192383,
+      "logps/rejected": -64.28612518310547,
+      "loss": 0.674,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.07684055715799332,
+      "rewards/margins": 0.040759824216365814,
+      "rewards/rejected": -0.11760036647319794,
+      "step": 8100
+    },
+    {
+      "epoch": 2.7911784975878704,
+      "eval_logits/chosen": -3.063803195953369,
+      "eval_logits/rejected": -3.0580430030822754,
+      "eval_logps/chosen": -63.49922180175781,
+      "eval_logps/rejected": -69.38825988769531,
+      "eval_loss": 0.6867249608039856,
+      "eval_rewards/accuracies": 0.5910780429840088,
+      "eval_rewards/chosen": -0.04787326604127884,
+      "eval_rewards/margins": 0.014208110980689526,
+      "eval_rewards/rejected": -0.06208137422800064,
+      "eval_runtime": 382.7974,
+      "eval_samples_per_second": 11.244,
+      "eval_steps_per_second": 1.405,
+      "step": 8100
+    },
+    {
+      "epoch": 2.794624396967609,
+      "grad_norm": 2.1319384574890137,
+      "learning_rate": 4.262924610556628e-10,
+      "logits/chosen": -2.8872647285461426,
+      "logits/rejected": -2.8691062927246094,
+      "logps/chosen": -61.59931564331055,
+      "logps/rejected": -65.82438659667969,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.0741548165678978,
+      "rewards/margins": 0.03955814987421036,
+      "rewards/rejected": -0.11371296644210815,
+      "step": 8110
+    },
+    {
+      "epoch": 2.7980702963473467,
+      "grad_norm": 2.0711333751678467,
+      "learning_rate": 4.1217268381857236e-10,
+      "logits/chosen": -2.973381519317627,
+      "logits/rejected": -2.9541213512420654,
+      "logps/chosen": -60.149742126464844,
+      "logps/rejected": -63.56388473510742,
+      "loss": 0.6806,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.08591555058956146,
+      "rewards/margins": 0.02720905840396881,
+      "rewards/rejected": -0.11312462389469147,
+      "step": 8120
+    },
+    {
+      "epoch": 2.8015161957270847,
+      "grad_norm": 2.08408784866333,
+      "learning_rate": 3.982874438028377e-10,
+      "logits/chosen": -2.9753928184509277,
+      "logits/rejected": -2.953627109527588,
+      "logps/chosen": -63.28232955932617,
+      "logps/rejected": -66.53197479248047,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08202724158763885,
+      "rewards/margins": 0.03199537843465805,
+      "rewards/rejected": -0.1140226274728775,
+      "step": 8130
+    },
+    {
+      "epoch": 2.804962095106823,
+      "grad_norm": 2.116588592529297,
+      "learning_rate": 3.8463696424976595e-10,
+      "logits/chosen": -3.050222158432007,
+      "logits/rejected": -3.0264439582824707,
+      "logps/chosen": -63.5772705078125,
+      "logps/rejected": -64.74728393554688,
+      "loss": 0.676,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.08089567720890045,
+      "rewards/margins": 0.03671393170952797,
+      "rewards/rejected": -0.11760959774255753,
+      "step": 8140
+    },
+    {
+      "epoch": 2.808407994486561,
+      "grad_norm": 2.2394309043884277,
+      "learning_rate": 3.712214646262768e-10,
+      "logits/chosen": -3.031567335128784,
+      "logits/rejected": -3.015326738357544,
+      "logps/chosen": -61.03345489501953,
+      "logps/rejected": -63.663169860839844,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0819227322936058,
+      "rewards/margins": 0.025047963485121727,
+      "rewards/rejected": -0.10697070509195328,
+      "step": 8150
+    },
+    {
+      "epoch": 2.8118538938662994,
+      "grad_norm": 2.0575995445251465,
+      "learning_rate": 3.5804116062137677e-10,
+      "logits/chosen": -2.946830987930298,
+      "logits/rejected": -2.9335474967956543,
+      "logps/chosen": -61.97414016723633,
+      "logps/rejected": -65.09761047363281,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.08346837013959885,
+      "rewards/margins": 0.020259350538253784,
+      "rewards/rejected": -0.10372772067785263,
+      "step": 8160
+    },
+    {
+      "epoch": 2.8152997932460373,
+      "grad_norm": 1.966666579246521,
+      "learning_rate": 3.4509626414269573e-10,
+      "logits/chosen": -2.9416613578796387,
+      "logits/rejected": -2.9168124198913574,
+      "logps/chosen": -63.408363342285156,
+      "logps/rejected": -64.59962463378906,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.07256388664245605,
+      "rewards/margins": 0.03664146363735199,
+      "rewards/rejected": -0.10920534282922745,
+      "step": 8170
+    },
+    {
+      "epoch": 2.8187456926257752,
+      "grad_norm": 2.0512208938598633,
+      "learning_rate": 3.323869833130743e-10,
+      "logits/chosen": -3.0153419971466064,
+      "logits/rejected": -2.9898173809051514,
+      "logps/chosen": -62.053810119628906,
+      "logps/rejected": -63.88903045654297,
+      "loss": 0.6774,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07939334213733673,
+      "rewards/margins": 0.03388165310025215,
+      "rewards/rejected": -0.11327499151229858,
+      "step": 8180
+    },
+    {
+      "epoch": 2.822191592005513,
+      "grad_norm": 2.2448196411132812,
+      "learning_rate": 3.1991352246721667e-10,
+      "logits/chosen": -2.9603278636932373,
+      "logits/rejected": -2.9444408416748047,
+      "logps/chosen": -62.0151252746582,
+      "logps/rejected": -63.12969970703125,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.07893949747085571,
+      "rewards/margins": 0.026331206783652306,
+      "rewards/rejected": -0.10527070611715317,
+      "step": 8190
+    },
+    {
+      "epoch": 2.8256374913852516,
+      "grad_norm": 2.000413656234741,
+      "learning_rate": 3.076760821484114e-10,
+      "logits/chosen": -2.9046196937561035,
+      "logits/rejected": -2.8842854499816895,
+      "logps/chosen": -61.361968994140625,
+      "logps/rejected": -64.95674133300781,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.07904700934886932,
+      "rewards/margins": 0.03575754165649414,
+      "rewards/rejected": -0.11480454355478287,
+      "step": 8200
+    },
+    {
+      "epoch": 2.8256374913852516,
+      "eval_logits/chosen": -3.063852548599243,
+      "eval_logits/rejected": -3.0581252574920654,
+      "eval_logps/chosen": -63.493812561035156,
+      "eval_logps/rejected": -69.40029907226562,
+      "eval_loss": 0.6866374611854553,
+      "eval_rewards/accuracies": 0.5899163484573364,
+      "eval_rewards/chosen": -0.047819167375564575,
+      "eval_rewards/margins": 0.014382554218173027,
+      "eval_rewards/rejected": -0.062201716005802155,
+      "eval_runtime": 383.0961,
+      "eval_samples_per_second": 11.235,
+      "eval_steps_per_second": 1.404,
+      "step": 8200
+    },
+    {
+      "epoch": 2.82908339076499,
+      "grad_norm": 2.1955642700195312,
+      "learning_rate": 2.9567485910530264e-10,
+      "logits/chosen": -2.9228601455688477,
+      "logits/rejected": -2.899026393890381,
+      "logps/chosen": -62.992454528808594,
+      "logps/rejected": -64.28277587890625,
+      "loss": 0.6783,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08247625082731247,
+      "rewards/margins": 0.032105665653944016,
+      "rewards/rejected": -0.11458192020654678,
+      "step": 8210
+    },
+    {
+      "epoch": 2.832529290144728,
+      "grad_norm": 1.9176273345947266,
+      "learning_rate": 2.8391004628872894e-10,
+      "logits/chosen": -2.911081314086914,
+      "logits/rejected": -2.8978817462921143,
+      "logps/chosen": -61.2845344543457,
+      "logps/rejected": -64.01652526855469,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.08688429743051529,
+      "rewards/margins": 0.02577807940542698,
+      "rewards/rejected": -0.11266238987445831,
+      "step": 8220
+    },
+    {
+      "epoch": 2.835975189524466,
+      "grad_norm": 1.8856381177902222,
+      "learning_rate": 2.7238183284861934e-10,
+      "logits/chosen": -2.9405148029327393,
+      "logits/rejected": -2.924905776977539,
+      "logps/chosen": -61.0767707824707,
+      "logps/rejected": -64.75289154052734,
+      "loss": 0.6765,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.08212117105722427,
+      "rewards/margins": 0.035802870988845825,
+      "rewards/rejected": -0.1179240494966507,
+      "step": 8230
+    },
+    {
+      "epoch": 2.8394210889042037,
+      "grad_norm": 2.1189565658569336,
+      "learning_rate": 2.610904041309525e-10,
+      "logits/chosen": -2.9043619632720947,
+      "logits/rejected": -2.887456178665161,
+      "logps/chosen": -60.037940979003906,
+      "logps/rejected": -65.21144104003906,
+      "loss": 0.6754,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.08182832598686218,
+      "rewards/margins": 0.03817012906074524,
+      "rewards/rejected": -0.11999845504760742,
+      "step": 8240
+    },
+    {
+      "epoch": 2.842866988283942,
+      "grad_norm": 2.0574111938476562,
+      "learning_rate": 2.500359416747805e-10,
+      "logits/chosen": -2.983400821685791,
+      "logits/rejected": -2.9506278038024902,
+      "logps/chosen": -63.04176712036133,
+      "logps/rejected": -61.410743713378906,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.07982859015464783,
+      "rewards/margins": 0.029811274260282516,
+      "rewards/rejected": -0.10963986068964005,
+      "step": 8250
+    },
+    {
+      "epoch": 2.84631288766368,
+      "grad_norm": 2.1163063049316406,
+      "learning_rate": 2.392186232093063e-10,
+      "logits/chosen": -3.048884391784668,
+      "logits/rejected": -3.024604082107544,
+      "logps/chosen": -64.87088012695312,
+      "logps/rejected": -66.00833892822266,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08395379781723022,
+      "rewards/margins": 0.030213261023163795,
+      "rewards/rejected": -0.11416705697774887,
+      "step": 8260
+    },
+    {
+      "epoch": 2.8497587870434185,
+      "grad_norm": 1.9222484827041626,
+      "learning_rate": 2.2863862265102285e-10,
+      "logits/chosen": -2.9435935020446777,
+      "logits/rejected": -2.9199492931365967,
+      "logps/chosen": -62.95438766479492,
+      "logps/rejected": -62.05031204223633,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08340393006801605,
+      "rewards/margins": 0.02691034972667694,
+      "rewards/rejected": -0.1103142723441124,
+      "step": 8270
+    },
+    {
+      "epoch": 2.8532046864231564,
+      "grad_norm": 2.0245275497436523,
+      "learning_rate": 2.1829611010092674e-10,
+      "logits/chosen": -2.9327938556671143,
+      "logits/rejected": -2.912468194961548,
+      "logps/chosen": -62.41936492919922,
+      "logps/rejected": -68.03335571289062,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.084849514067173,
+      "rewards/margins": 0.03654637187719345,
+      "rewards/rejected": -0.12139588594436646,
+      "step": 8280
+    },
+    {
+      "epoch": 2.8566505858028943,
+      "grad_norm": 1.9506707191467285,
+      "learning_rate": 2.081912518417772e-10,
+      "logits/chosen": -2.997532606124878,
+      "logits/rejected": -2.9763944149017334,
+      "logps/chosen": -60.153648376464844,
+      "logps/rejected": -63.301902770996094,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.08040939271450043,
+      "rewards/margins": 0.03193066641688347,
+      "rewards/rejected": -0.1123400554060936,
+      "step": 8290
+    },
+    {
+      "epoch": 2.8600964851826327,
+      "grad_norm": 2.0229573249816895,
+      "learning_rate": 1.9832421033541658e-10,
+      "logits/chosen": -2.925785541534424,
+      "logits/rejected": -2.9074344635009766,
+      "logps/chosen": -64.16853332519531,
+      "logps/rejected": -64.25889587402344,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.08635859936475754,
+      "rewards/margins": 0.024428073316812515,
+      "rewards/rejected": -0.11078667640686035,
+      "step": 8300
+    },
+    {
+      "epoch": 2.8600964851826327,
+      "eval_logits/chosen": -3.063709020614624,
+      "eval_logits/rejected": -3.057910442352295,
+      "eval_logps/chosen": -63.49977493286133,
+      "eval_logps/rejected": -69.39702606201172,
+      "eval_loss": 0.68668532371521,
+      "eval_rewards/accuracies": 0.5889869928359985,
+      "eval_rewards/chosen": -0.04787874594330788,
+      "eval_rewards/margins": 0.014290270395576954,
+      "eval_rewards/rejected": -0.06216902285814285,
+      "eval_runtime": 382.9822,
+      "eval_samples_per_second": 11.238,
+      "eval_steps_per_second": 1.405,
+      "step": 8300
+    },
+    {
+      "epoch": 2.8635423845623706,
+      "grad_norm": 2.1516809463500977,
+      "learning_rate": 1.8869514422017562e-10,
+      "logits/chosen": -2.9164390563964844,
+      "logits/rejected": -2.89357328414917,
+      "logps/chosen": -64.27980041503906,
+      "logps/rejected": -65.00248718261719,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.08439619839191437,
+      "rewards/margins": 0.025900591164827347,
+      "rewards/rejected": -0.11029678583145142,
+      "step": 8310
+    },
+    {
+      "epoch": 2.866988283942109,
+      "grad_norm": 2.088627338409424,
+      "learning_rate": 1.7930420830830571e-10,
+      "logits/chosen": -2.928290843963623,
+      "logits/rejected": -2.8895983695983887,
+      "logps/chosen": -66.83313751220703,
+      "logps/rejected": -63.651878356933594,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.07406539469957352,
+      "rewards/margins": 0.04551464319229126,
+      "rewards/rejected": -0.11958001554012299,
+      "step": 8320
+    },
+    {
+      "epoch": 2.870434183321847,
+      "grad_norm": 2.2679495811462402,
+      "learning_rate": 1.701515535834974e-10,
+      "logits/chosen": -2.930990695953369,
+      "logits/rejected": -2.9178614616394043,
+      "logps/chosen": -62.11572265625,
+      "logps/rejected": -63.32598114013672,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.07914047688245773,
+      "rewards/margins": 0.022064244374632835,
+      "rewards/rejected": -0.10120473057031631,
+      "step": 8330
+    },
+    {
+      "epoch": 2.873880082701585,
+      "grad_norm": 1.8590447902679443,
+      "learning_rate": 1.6123732719845238e-10,
+      "logits/chosen": -2.902958631515503,
+      "logits/rejected": -2.8927102088928223,
+      "logps/chosen": -59.44152069091797,
+      "logps/rejected": -62.86747360229492,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.0926695317029953,
+      "rewards/margins": 0.025111133232712746,
+      "rewards/rejected": -0.1177806630730629,
+      "step": 8340
+    },
+    {
+      "epoch": 2.8773259820813233,
+      "grad_norm": 2.0406289100646973,
+      "learning_rate": 1.525616724725187e-10,
+      "logits/chosen": -3.042212724685669,
+      "logits/rejected": -3.020266056060791,
+      "logps/chosen": -61.5721549987793,
+      "logps/rejected": -63.8056640625,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.0759461373090744,
+      "rewards/margins": 0.0272056944668293,
+      "rewards/rejected": -0.1031518206000328,
+      "step": 8350
+    },
+    {
+      "epoch": 2.8807718814610612,
+      "grad_norm": 1.9178696870803833,
+      "learning_rate": 1.4412472888938431e-10,
+      "logits/chosen": -2.968129873275757,
+      "logits/rejected": -2.939253330230713,
+      "logps/chosen": -61.00701904296875,
+      "logps/rejected": -61.58538818359375,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08411670476198196,
+      "rewards/margins": 0.031833138316869736,
+      "rewards/rejected": -0.1159498319029808,
+      "step": 8360
+    },
+    {
+      "epoch": 2.8842177808407996,
+      "grad_norm": 2.316232442855835,
+      "learning_rate": 1.3592663209483379e-10,
+      "logits/chosen": -2.896716594696045,
+      "logits/rejected": -2.8737173080444336,
+      "logps/chosen": -62.32270431518555,
+      "logps/rejected": -63.394371032714844,
+      "loss": 0.6829,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.08600802719593048,
+      "rewards/margins": 0.022584471851587296,
+      "rewards/rejected": -0.10859249532222748,
+      "step": 8370
+    },
+    {
+      "epoch": 2.8876636802205375,
+      "grad_norm": 2.037475824356079,
+      "learning_rate": 1.279675138945735e-10,
+      "logits/chosen": -2.93186616897583,
+      "logits/rejected": -2.915947914123535,
+      "logps/chosen": -62.780670166015625,
+      "logps/rejected": -67.04425048828125,
+      "loss": 0.6771,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08235308527946472,
+      "rewards/margins": 0.034664928913116455,
+      "rewards/rejected": -0.11701802909374237,
+      "step": 8380
+    },
+    {
+      "epoch": 2.8911095796002755,
+      "grad_norm": 2.3082103729248047,
+      "learning_rate": 1.202475022521032e-10,
+      "logits/chosen": -2.9857611656188965,
+      "logits/rejected": -2.9596362113952637,
+      "logps/chosen": -63.4444580078125,
+      "logps/rejected": -65.36312866210938,
+      "loss": 0.677,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.0802798643708229,
+      "rewards/margins": 0.03464691713452339,
+      "rewards/rejected": -0.114926777780056,
+      "step": 8390
+    },
+    {
+      "epoch": 2.894555478980014,
+      "grad_norm": 2.268125295639038,
+      "learning_rate": 1.1276672128666619e-10,
+      "logits/chosen": -2.9730422496795654,
+      "logits/rejected": -2.9478273391723633,
+      "logps/chosen": -61.3690299987793,
+      "logps/rejected": -62.1486930847168,
+      "loss": 0.6795,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.0806599110364914,
+      "rewards/margins": 0.02996053732931614,
+      "rewards/rejected": -0.11062043905258179,
+      "step": 8400
+    },
+    {
+      "epoch": 2.894555478980014,
+      "eval_logits/chosen": -3.063688039779663,
+      "eval_logits/rejected": -3.0579566955566406,
+      "eval_logps/chosen": -63.49544906616211,
+      "eval_logps/rejected": -69.3868408203125,
+      "eval_loss": 0.6867172122001648,
+      "eval_rewards/accuracies": 0.5903810262680054,
+      "eval_rewards/chosen": -0.047835495322942734,
+      "eval_rewards/margins": 0.014231679029762745,
+      "eval_rewards/rejected": -0.06206716597080231,
+      "eval_runtime": 383.1194,
+      "eval_samples_per_second": 11.234,
+      "eval_steps_per_second": 1.404,
+      "step": 8400
+    },
+    {
+      "epoch": 2.898001378359752,
+      "grad_norm": 2.0325965881347656,
+      "learning_rate": 1.0552529127125064e-10,
+      "logits/chosen": -2.9690990447998047,
+      "logits/rejected": -2.9420087337493896,
+      "logps/chosen": -61.78255081176758,
+      "logps/rejected": -61.640281677246094,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.07507432997226715,
+      "rewards/margins": 0.03316715359687805,
+      "rewards/rejected": -0.1082414984703064,
+      "step": 8410
+    },
+    {
+      "epoch": 2.90144727773949,
+      "grad_norm": 2.1202170848846436,
+      "learning_rate": 9.852332863065471e-11,
+      "logits/chosen": -2.9899940490722656,
+      "logits/rejected": -2.9650206565856934,
+      "logps/chosen": -62.15747833251953,
+      "logps/rejected": -65.27293395996094,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.08593356609344482,
+      "rewards/margins": 0.03423687070608139,
+      "rewards/rejected": -0.12017042934894562,
+      "step": 8420
+    },
+    {
+      "epoch": 2.904893177119228,
+      "grad_norm": 1.9820818901062012,
+      "learning_rate": 9.176094593962125e-11,
+      "logits/chosen": -2.982215642929077,
+      "logits/rejected": -2.942953586578369,
+      "logps/chosen": -63.40718460083008,
+      "logps/rejected": -61.37022018432617,
+      "loss": 0.678,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.08138169348239899,
+      "rewards/margins": 0.03287549689412117,
+      "rewards/rejected": -0.11425720155239105,
+      "step": 8430
+    },
+    {
+      "epoch": 2.908339076498966,
+      "grad_norm": 2.0494680404663086,
+      "learning_rate": 8.523825192101597e-11,
+      "logits/chosen": -3.0455162525177,
+      "logits/rejected": -3.0237479209899902,
+      "logps/chosen": -59.611732482910156,
+      "logps/rejected": -63.030418395996094,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08587189018726349,
+      "rewards/margins": 0.029600840061903,
+      "rewards/rejected": -0.11547273397445679,
+      "step": 8440
+    },
+    {
+      "epoch": 2.9117849758787044,
+      "grad_norm": 2.022709846496582,
+      "learning_rate": 7.895535144409548e-11,
+      "logits/chosen": -2.9769985675811768,
+      "logits/rejected": -2.96103835105896,
+      "logps/chosen": -63.152687072753906,
+      "logps/rejected": -63.297821044921875,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.09083075076341629,
+      "rewards/margins": 0.021183013916015625,
+      "rewards/rejected": -0.11201377213001251,
+      "step": 8450
+    },
+    {
+      "epoch": 2.9152308752584424,
+      "grad_norm": 1.8870978355407715,
+      "learning_rate": 7.2912345522802e-11,
+      "logits/chosen": -2.9887826442718506,
+      "logits/rejected": -2.9673519134521484,
+      "logps/chosen": -61.3896598815918,
+      "logps/rejected": -64.93919372558594,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.08222813159227371,
+      "rewards/margins": 0.029316555708646774,
+      "rewards/rejected": -0.111544668674469,
+      "step": 8460
+    },
+    {
+      "epoch": 2.9186767746381808,
+      "grad_norm": 2.1745338439941406,
+      "learning_rate": 6.71093313141563e-11,
+      "logits/chosen": -3.013063430786133,
+      "logits/rejected": -2.991659164428711,
+      "logps/chosen": -64.53852844238281,
+      "logps/rejected": -64.00894927978516,
+      "loss": 0.678,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.07416824996471405,
+      "rewards/margins": 0.03233487531542778,
+      "rewards/rejected": -0.10650312900543213,
+      "step": 8470
+    },
+    {
+      "epoch": 2.9221226740179187,
+      "grad_norm": 2.1404759883880615,
+      "learning_rate": 6.154640211668727e-11,
+      "logits/chosen": -2.953394889831543,
+      "logits/rejected": -2.9412128925323486,
+      "logps/chosen": -60.77058792114258,
+      "logps/rejected": -66.36811828613281,
+      "loss": 0.6792,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.08206119388341904,
+      "rewards/margins": 0.030566278845071793,
+      "rewards/rejected": -0.11262746900320053,
+      "step": 8480
+    },
+    {
+      "epoch": 2.9255685733976566,
+      "grad_norm": 2.093405246734619,
+      "learning_rate": 5.622364736893481e-11,
+      "logits/chosen": -3.0309131145477295,
+      "logits/rejected": -3.0056862831115723,
+      "logps/chosen": -62.84321975708008,
+      "logps/rejected": -65.09196472167969,
+      "loss": 0.676,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.08165743947029114,
+      "rewards/margins": 0.03672707825899124,
+      "rewards/rejected": -0.11838452517986298,
+      "step": 8490
+    },
+    {
+      "epoch": 2.929014472777395,
+      "grad_norm": 2.0061511993408203,
+      "learning_rate": 5.1141152648007666e-11,
+      "logits/chosen": -3.0523035526275635,
+      "logits/rejected": -3.0281991958618164,
+      "logps/chosen": -62.279998779296875,
+      "logps/rejected": -63.6883430480957,
+      "loss": 0.679,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.07944914698600769,
+      "rewards/margins": 0.03057517483830452,
+      "rewards/rejected": -0.11002431064844131,
+      "step": 8500
+    },
+    {
+      "epoch": 2.929014472777395,
+      "eval_logits/chosen": -3.0637094974517822,
+      "eval_logits/rejected": -3.0579497814178467,
+      "eval_logps/chosen": -63.49948501586914,
+      "eval_logps/rejected": -69.39813995361328,
+      "eval_loss": 0.6866816282272339,
+      "eval_rewards/accuracies": 0.5924721360206604,
+      "eval_rewards/chosen": -0.04787592217326164,
+      "eval_rewards/margins": 0.014304240234196186,
+      "eval_rewards/rejected": -0.06218015402555466,
+      "eval_runtime": 383.0545,
+      "eval_samples_per_second": 11.236,
+      "eval_steps_per_second": 1.404,
+      "step": 8500
+    },
+    {
+      "epoch": 2.932460372157133,
+      "grad_norm": 2.0353848934173584,
+      "learning_rate": 4.6298999668212824e-11,
+      "logits/chosen": -2.9379279613494873,
+      "logits/rejected": -2.93033766746521,
+      "logps/chosen": -62.128562927246094,
+      "logps/rejected": -65.45433044433594,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.08974123001098633,
+      "rewards/margins": 0.023062895983457565,
+      "rewards/rejected": -0.11280412971973419,
+      "step": 8510
+    },
+    {
+      "epoch": 2.9359062715368713,
+      "grad_norm": 2.509343147277832,
+      "learning_rate": 4.16972662797399e-11,
+      "logits/chosen": -3.0194308757781982,
+      "logits/rejected": -2.9956936836242676,
+      "logps/chosen": -64.87427520751953,
+      "logps/rejected": -63.29524612426758,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08701174706220627,
+      "rewards/margins": 0.025242164731025696,
+      "rewards/rejected": -0.11225390434265137,
+      "step": 8520
+    },
+    {
+      "epoch": 2.9393521709166093,
+      "grad_norm": 1.9690372943878174,
+      "learning_rate": 3.7336026467405524e-11,
+      "logits/chosen": -3.0140509605407715,
+      "logits/rejected": -2.9873809814453125,
+      "logps/chosen": -63.52933883666992,
+      "logps/rejected": -65.43867492675781,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.08610363304615021,
+      "rewards/margins": 0.026120662689208984,
+      "rewards/rejected": -0.11222430318593979,
+      "step": 8530
+    },
+    {
+      "epoch": 2.942798070296347,
+      "grad_norm": 2.179032564163208,
+      "learning_rate": 3.321535034947087e-11,
+      "logits/chosen": -2.9117836952209473,
+      "logits/rejected": -2.8833229541778564,
+      "logps/chosen": -62.50554656982422,
+      "logps/rejected": -64.86080169677734,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.08089722692966461,
+      "rewards/margins": 0.03771393373608589,
+      "rewards/rejected": -0.1186111718416214,
+      "step": 8540
+    },
+    {
+      "epoch": 2.9462439696760856,
+      "grad_norm": 2.187864303588867,
+      "learning_rate": 2.9335304176509314e-11,
+      "logits/chosen": -2.931490659713745,
+      "logits/rejected": -2.907622814178467,
+      "logps/chosen": -62.852622985839844,
+      "logps/rejected": -64.65711975097656,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.08195560425519943,
+      "rewards/margins": 0.03601902723312378,
+      "rewards/rejected": -0.11797462403774261,
+      "step": 8550
+    },
+    {
+      "epoch": 2.9496898690558235,
+      "grad_norm": 1.994933009147644,
+      "learning_rate": 2.569595033034222e-11,
+      "logits/chosen": -2.986529588699341,
+      "logits/rejected": -2.9673736095428467,
+      "logps/chosen": -62.722740173339844,
+      "logps/rejected": -63.26072311401367,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.09294059872627258,
+      "rewards/margins": 0.024714117869734764,
+      "rewards/rejected": -0.1176547259092331,
+      "step": 8560
+    },
+    {
+      "epoch": 2.953135768435562,
+      "grad_norm": 2.0444486141204834,
+      "learning_rate": 2.2297347323039782e-11,
+      "logits/chosen": -2.922816038131714,
+      "logits/rejected": -2.8971104621887207,
+      "logps/chosen": -61.705177307128906,
+      "logps/rejected": -63.867279052734375,
+      "loss": 0.677,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.0757937952876091,
+      "rewards/margins": 0.03462393954396248,
+      "rewards/rejected": -0.11041773855686188,
+      "step": 8570
+    },
+    {
+      "epoch": 2.9565816678153,
+      "grad_norm": 1.9794280529022217,
+      "learning_rate": 1.91395497959701e-11,
+      "logits/chosen": -3.0625908374786377,
+      "logits/rejected": -3.0407299995422363,
+      "logps/chosen": -60.069580078125,
+      "logps/rejected": -63.95684814453125,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07948606461286545,
+      "rewards/margins": 0.039867181330919266,
+      "rewards/rejected": -0.11935325711965561,
+      "step": 8580
+    },
+    {
+      "epoch": 2.960027567195038,
+      "grad_norm": 2.034372568130493,
+      "learning_rate": 1.622260851893653e-11,
+      "logits/chosen": -3.0658843517303467,
+      "logits/rejected": -3.0454087257385254,
+      "logps/chosen": -63.855384826660156,
+      "logps/rejected": -64.94879913330078,
+      "loss": 0.684,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.0874631255865097,
+      "rewards/margins": 0.020751936361193657,
+      "rewards/rejected": -0.10821505635976791,
+      "step": 8590
+    },
+    {
+      "epoch": 2.963473466574776,
+      "grad_norm": 2.1895265579223633,
+      "learning_rate": 1.3546570389345035e-11,
+      "logits/chosen": -2.957758903503418,
+      "logits/rejected": -2.9512016773223877,
+      "logps/chosen": -60.950279235839844,
+      "logps/rejected": -64.09395599365234,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08055905997753143,
+      "rewards/margins": 0.025479119271039963,
+      "rewards/rejected": -0.10603819042444229,
+      "step": 8600
+    },
+    {
+      "epoch": 2.963473466574776,
+      "eval_logits/chosen": -3.0636613368988037,
+      "eval_logits/rejected": -3.057868480682373,
+      "eval_logps/chosen": -63.490692138671875,
+      "eval_logps/rejected": -69.3946304321289,
+      "eval_loss": 0.6866512298583984,
+      "eval_rewards/accuracies": 0.5922397971153259,
+      "eval_rewards/chosen": -0.04778796061873436,
+      "eval_rewards/margins": 0.014357124455273151,
+      "eval_rewards/rejected": -0.06214507669210434,
+      "eval_runtime": 382.877,
+      "eval_samples_per_second": 11.241,
+      "eval_steps_per_second": 1.405,
+      "step": 8600
+    },
+    {
+      "epoch": 2.966919365954514,
+      "grad_norm": 2.2051644325256348,
+      "learning_rate": 1.111147843146143e-11,
+      "logits/chosen": -2.9392690658569336,
+      "logits/rejected": -2.936722993850708,
+      "logps/chosen": -62.7488899230957,
+      "logps/rejected": -67.1611099243164,
+      "loss": 0.6806,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.08900688588619232,
+      "rewards/margins": 0.02782275713980198,
+      "rewards/rejected": -0.11682965606451035,
+      "step": 8610
+    },
+    {
+      "epoch": 2.9703652653342525,
+      "grad_norm": 2.273514986038208,
+      "learning_rate": 8.917371795713613e-12,
+      "logits/chosen": -2.9783403873443604,
+      "logits/rejected": -2.976510524749756,
+      "logps/chosen": -61.298583984375,
+      "logps/rejected": -65.40287780761719,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.081795834004879,
+      "rewards/margins": 0.021864496171474457,
+      "rewards/rejected": -0.10366033017635345,
+      "step": 8620
+    },
+    {
+      "epoch": 2.9738111647139904,
+      "grad_norm": 2.0920350551605225,
+      "learning_rate": 6.9642857580687325e-12,
+      "logits/chosen": -2.908949851989746,
+      "logits/rejected": -2.888460636138916,
+      "logps/chosen": -61.871315002441406,
+      "logps/rejected": -64.46957397460938,
+      "loss": 0.6754,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": -0.08098328113555908,
+      "rewards/margins": 0.03785686939954758,
+      "rewards/rejected": -0.11884014308452606,
+      "step": 8630
+    },
+    {
+      "epoch": 2.9772570640937284,
+      "grad_norm": 2.061936616897583,
+      "learning_rate": 5.252251719456979e-12,
+      "logits/chosen": -2.955897331237793,
+      "logits/rejected": -2.9279160499572754,
+      "logps/chosen": -63.2880744934082,
+      "logps/rejected": -64.03325653076172,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.08193423599004745,
+      "rewards/margins": 0.027203386649489403,
+      "rewards/rejected": -0.1091376319527626,
+      "step": 8640
+    },
+    {
+      "epoch": 2.9807029634734663,
+      "grad_norm": 2.156332492828369,
+      "learning_rate": 3.781297205271983e-12,
+      "logits/chosen": -2.998988628387451,
+      "logits/rejected": -2.9828782081604004,
+      "logps/chosen": -65.4218521118164,
+      "logps/rejected": -65.43940734863281,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.08261623233556747,
+      "rewards/margins": 0.023654678836464882,
+      "rewards/rejected": -0.10627090930938721,
+      "step": 8650
+    },
+    {
+      "epoch": 2.9841488628532047,
+      "grad_norm": 2.0988051891326904,
+      "learning_rate": 2.5514458649311675e-12,
+      "logits/chosen": -2.9806160926818848,
+      "logits/rejected": -2.958583354949951,
+      "logps/chosen": -63.239830017089844,
+      "logps/rejected": -63.61602783203125,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08698787540197372,
+      "rewards/margins": 0.027879631146788597,
+      "rewards/rejected": -0.11486752331256866,
+      "step": 8660
+    },
+    {
+      "epoch": 2.987594762232943,
+      "grad_norm": 1.9743732213974,
+      "learning_rate": 1.562717471487729e-12,
+      "logits/chosen": -2.9887750148773193,
+      "logits/rejected": -2.9638569355010986,
+      "logps/chosen": -61.57611083984375,
+      "logps/rejected": -63.081993103027344,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.07708355039358139,
+      "rewards/margins": 0.03751971200108528,
+      "rewards/rejected": -0.11460325866937637,
+      "step": 8670
+    },
+    {
+      "epoch": 2.991040661612681,
+      "grad_norm": 1.929993987083435,
+      "learning_rate": 8.151279213208795e-13,
+      "logits/chosen": -3.012930154800415,
+      "logits/rejected": -2.9947848320007324,
+      "logps/chosen": -62.7015266418457,
+      "logps/rejected": -65.101318359375,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.0871218889951706,
+      "rewards/margins": 0.02738434448838234,
+      "rewards/rejected": -0.11450624465942383,
+      "step": 8680
+    },
+    {
+      "epoch": 2.994486560992419,
+      "grad_norm": 2.3326921463012695,
+      "learning_rate": 3.0868923387439294e-13,
+      "logits/chosen": -2.967649221420288,
+      "logits/rejected": -2.956500291824341,
+      "logps/chosen": -61.452049255371094,
+      "logps/rejected": -65.99691009521484,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08869382739067078,
+      "rewards/margins": 0.021128688007593155,
+      "rewards/rejected": -0.10982249677181244,
+      "step": 8690
+    },
+    {
+      "epoch": 2.997932460372157,
+      "grad_norm": 2.1507482528686523,
+      "learning_rate": 4.34095514667554e-14,
+      "logits/chosen": -2.985356569290161,
+      "logits/rejected": -2.9610397815704346,
+      "logps/chosen": -62.43767166137695,
+      "logps/rejected": -63.207801818847656,
+      "loss": 0.6751,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": -0.07855219393968582,
+      "rewards/margins": 0.03873778507113457,
+      "rewards/rejected": -0.11728998273611069,
+      "step": 8700
+    },
+    {
+      "epoch": 2.997932460372157,
+      "eval_logits/chosen": -3.063735008239746,
+      "eval_logits/rejected": -3.0579655170440674,
+      "eval_logps/chosen": -63.48755645751953,
+      "eval_logps/rejected": -69.37789154052734,
+      "eval_loss": 0.6867183446884155,
+      "eval_rewards/accuracies": 0.5936338305473328,
+      "eval_rewards/chosen": -0.04775654152035713,
+      "eval_rewards/margins": 0.01422115322202444,
+      "eval_rewards/rejected": -0.061977699398994446,
+      "eval_runtime": 383.9833,
+      "eval_samples_per_second": 11.209,
+      "eval_steps_per_second": 1.401,
+      "step": 8700
+    },
+    {
+      "epoch": 3.0,
+      "step": 8706,
+      "total_flos": 0.0,
+      "train_loss": 0.6843596647285796,
+      "train_runtime": 104656.3305,
+      "train_samples_per_second": 2.662,
+      "train_steps_per_second": 0.083
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 8706,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}